1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SelectionDAGNodes.h"
32#include "llvm/CodeGen/TargetFrameLowering.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/CodeGenTypes/MachineValueType.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Compiler.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/MathExtras.h"
49#include "llvm/Support/raw_ostream.h"
50#include "llvm/Target/TargetMachine.h"
51#include "llvm/Target/TargetOptions.h"
52#include <cassert>
53#include <cstdint>
54#include <tuple>
55#include <utility>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "legalizedag"
60
61namespace {
62
63/// Keeps track of state when getting the sign of a floating-point value as an
64/// integer.
65struct FloatSignAsInt {
66 EVT FloatVT;
67 SDValue Chain;
68 SDValue FloatPtr;
69 SDValue IntPtr;
70 MachinePointerInfo IntPointerInfo;
71 MachinePointerInfo FloatPointerInfo;
72 SDValue IntValue;
73 APInt SignMask;
74 uint8_t SignBit;
75};
76
77//===----------------------------------------------------------------------===//
78/// This takes an arbitrary SelectionDAG as input and
79/// hacks on it until the target machine can handle it. This involves
80/// eliminating value sizes the machine cannot handle (promoting small sizes to
81/// large sizes or splitting up large values into small values) as well as
82/// eliminating operations the machine cannot handle.
83///
84/// This code also does a small amount of optimization and recognition of idioms
85/// as part of its processing. For example, if a target does not support a
86/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
87/// will attempt merge setcc and brc instructions into brcc's.
88class SelectionDAGLegalize {
89 const TargetMachine &TM;
90 const TargetLowering &TLI;
91 SelectionDAG &DAG;
92
93 /// The set of nodes which have already been legalized. We hold a
94 /// reference to it in order to update as necessary on node deletion.
95 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
96
97 /// A set of all the nodes updated during legalization.
98 SmallSetVector<SDNode *, 16> *UpdatedNodes;
99
100 EVT getSetCCResultType(EVT VT) const {
101 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
102 }
103
104 // Libcall insertion helpers.
105
106public:
107 SelectionDAGLegalize(SelectionDAG &DAG,
108 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
109 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
110 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
112
113 /// Legalizes the given operation.
114 void LegalizeOp(SDNode *Node);
115
116private:
117 SDValue OptimizeFloatStore(StoreSDNode *ST);
118
119 void LegalizeLoadOps(SDNode *Node);
120 void LegalizeStoreOps(SDNode *Node);
121
122 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
123
124 /// Return a vector shuffle operation which
125 /// performs the same shuffe in terms of order or result bytes, but on a type
126 /// whose vector element type is narrower than the original shuffle type.
127 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
128 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
129 SDValue N1, SDValue N2,
130 ArrayRef<int> Mask) const;
131
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 TargetLowering::ArgListTy &&Args,
134 bool IsSigned, EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
136
137 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 SmallVectorImpl<SDValue> &Results);
139
140 void
141 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
142 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
143 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
144 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
145 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
146 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
147 SmallVectorImpl<SDValue> &Results);
148
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
150 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
151 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
152 void ExpandArgFPLibCall(SDNode *Node,
153 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
154 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
155 RTLIB::Libcall Call_PPCF128,
156 SmallVectorImpl<SDValue> &Results);
157 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
158 RTLIB::Libcall CallI64,
159 RTLIB::Libcall CallI128);
160 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161
162 SDValue ExpandSincosStretLibCall(SDNode *Node) const;
163
164 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
165 const SDLoc &dl);
166 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
167 const SDLoc &dl, SDValue ChainIn);
168 SDValue ExpandBUILD_VECTOR(SDNode *Node);
169 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
170 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
171 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
172 SmallVectorImpl<SDValue> &Results);
173 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
174 SDValue Value) const;
175 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
176 SDValue NewIntValue) const;
177 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
178 SDValue ExpandFABS(SDNode *Node) const;
179 SDValue ExpandFNEG(SDNode *Node) const;
180 SDValue expandLdexp(SDNode *Node) const;
181 SDValue expandFrexp(SDNode *Node) const;
182 SDValue expandModf(SDNode *Node) const;
183
184 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
185 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
186 SmallVectorImpl<SDValue> &Results);
187 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
188 SmallVectorImpl<SDValue> &Results);
189 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
190
191 /// Implements vector reduce operation promotion.
192 ///
193 /// All vector operands are promoted to a vector type with larger element
194 /// type, and the start value is promoted to a larger scalar type. Then the
195 /// result is truncated back to the original scalar type.
196 SDValue PromoteReduction(SDNode *Node);
197
198 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
199
200 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
201 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
202 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
203 SDValue ExpandConcatVectors(SDNode *Node);
204
205 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
206 SDValue ExpandConstant(ConstantSDNode *CP);
207
208 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
209 bool ExpandNode(SDNode *Node);
210 void ConvertNodeToLibcall(SDNode *Node);
211 void PromoteNode(SDNode *Node);
212
213public:
214 // Node replacement helpers
215
216 void ReplacedNode(SDNode *N) {
217 LegalizedNodes.erase(Ptr: N);
218 if (UpdatedNodes)
219 UpdatedNodes->insert(X: N);
220 }
221
222 void ReplaceNode(SDNode *Old, SDNode *New) {
223 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
224 dbgs() << " with: "; New->dump(&DAG));
225
226 assert(Old->getNumValues() == New->getNumValues() &&
227 "Replacing one node with another that produces a different number "
228 "of values!");
229 DAG.ReplaceAllUsesWith(From: Old, To: New);
230 if (UpdatedNodes)
231 UpdatedNodes->insert(X: New);
232 ReplacedNode(N: Old);
233 }
234
235 void ReplaceNode(SDValue Old, SDValue New) {
236 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
237 dbgs() << " with: "; New->dump(&DAG));
238
239 DAG.ReplaceAllUsesWith(From: Old, To: New);
240 if (UpdatedNodes)
241 UpdatedNodes->insert(X: New.getNode());
242 ReplacedNode(N: Old.getNode());
243 }
244
245 void ReplaceNode(SDNode *Old, const SDValue *New) {
246 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
247
248 DAG.ReplaceAllUsesWith(From: Old, To: New);
249 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
250 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
251 New[i]->dump(&DAG));
252 if (UpdatedNodes)
253 UpdatedNodes->insert(X: New[i].getNode());
254 }
255 ReplacedNode(N: Old);
256 }
257
258 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
259 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
260 dbgs() << " with: "; New->dump(&DAG));
261
262 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
263 if (UpdatedNodes)
264 UpdatedNodes->insert(X: New.getNode());
265 ReplacedNode(N: Old.getNode());
266 }
267};
268
269} // end anonymous namespace
270
271// Helper function that generates an MMO that considers the alignment of the
272// stack, and the size of the stack object
273static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
274 MachineFunction &MF,
275 bool isObjectScalable) {
276 auto &MFI = MF.getFrameInfo();
277 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
278 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
279 LocationSize ObjectSize = isObjectScalable
280 ? LocationSize::beforeOrAfterPointer()
281 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
282 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
283 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
284}
285
286/// Return a vector shuffle operation which
287/// performs the same shuffle in terms of order or result bytes, but on a type
288/// whose vector element type is narrower than the original shuffle type.
289/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
290SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
291 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
292 ArrayRef<int> Mask) const {
293 unsigned NumMaskElts = VT.getVectorNumElements();
294 unsigned NumDestElts = NVT.getVectorNumElements();
295 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
296
297 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
298
299 if (NumEltsGrowth == 1)
300 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
301
302 SmallVector<int, 8> NewMask;
303 for (unsigned i = 0; i != NumMaskElts; ++i) {
304 int Idx = Mask[i];
305 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
306 if (Idx < 0)
307 NewMask.push_back(Elt: -1);
308 else
309 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
310 }
311 }
312 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
313 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
314 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
315}
316
317/// Expands the ConstantFP node to an integer constant or
318/// a load from the constant pool.
319SDValue
320SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
321 bool Extend = false;
322 SDLoc dl(CFP);
323
324 // If a FP immediate is precise when represented as a float and if the
325 // target can do an extending load from float to double, we put it into
326 // the constant pool as a float, even if it's is statically typed as a
327 // double. This shrinks FP constants and canonicalizes them for targets where
328 // an FP extending load is the same cost as a normal load (such as on the x87
329 // fp stack or PPC FP unit).
330 EVT VT = CFP->getValueType(ResNo: 0);
331 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
332 if (!UseCP) {
333 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
334 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
335 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
336 }
337
338 APFloat APF = CFP->getValueAPF();
339 EVT OrigVT = VT;
340 EVT SVT = VT;
341
342 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
343 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
344 if (!APF.isSignaling()) {
345 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
346 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
347 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
348 // Only do this if the target has a native EXTLOAD instruction from
349 // smaller type.
350 TLI.isLoadLegal(
351 ValVT: OrigVT, MemVT: SVT,
352 Alignment: Align(DAG.getDataLayout().getPrefTypeAlign(
353 Ty: SVT.getTypeForEVT(Context&: *DAG.getContext()))),
354 AddrSpace: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction())
355 .getAddrSpace(),
356 ExtType: ISD::EXTLOAD, Atomic: false) &&
357 TLI.ShouldShrinkFPConstant(OrigVT)) {
358 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
359 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
360 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
361 VT = SVT;
362 Extend = true;
363 }
364 }
365 }
366
367 SDValue CPIdx =
368 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
369 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
370 if (Extend) {
371 SDValue Result = DAG.getExtLoad(
372 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
373 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
374 Alignment);
375 return Result;
376 }
377 SDValue Result = DAG.getLoad(
378 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
379 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
380 return Result;
381}
382
383/// Expands the Constant node to a load from the constant pool.
384SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
385 SDLoc dl(CP);
386 EVT VT = CP->getValueType(ResNo: 0);
387 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
388 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
389 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
390 SDValue Result = DAG.getLoad(
391 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
392 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
393 return Result;
394}
395
396SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
397 SDValue Vec = Op.getOperand(i: 0);
398 SDValue Val = Op.getOperand(i: 1);
399 SDValue Idx = Op.getOperand(i: 2);
400 SDLoc dl(Op);
401
402 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
403 // SCALAR_TO_VECTOR requires that the type of the value being inserted
404 // match the element type of the vector being created, except for
405 // integers in which case the inserted value can be over width.
406 EVT EltVT = Vec.getValueType().getVectorElementType();
407 if (Val.getValueType() == EltVT ||
408 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
409 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
410 VT: Vec.getValueType(), Operand: Val);
411
412 unsigned NumElts = Vec.getValueType().getVectorNumElements();
413 // We generate a shuffle of InVec and ScVec, so the shuffle mask
414 // should be 0,1,2,3,4,5... with the appropriate element replaced with
415 // elt 0 of the RHS.
416 SmallVector<int, 8> ShufOps;
417 for (unsigned i = 0; i != NumElts; ++i)
418 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
419
420 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
421 }
422 }
423 return ExpandInsertToVectorThroughStack(Op);
424}
425
426SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
427 if (!ISD::isNormalStore(N: ST))
428 return SDValue();
429
430 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
431 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
432 // FIXME: move this to the DAG Combiner! Note that we can't regress due
433 // to phase ordering between legalized code and the dag combiner. This
434 // probably means that we need to integrate dag combiner and legalizer
435 // together.
436 // We generally can't do this one for long doubles.
437 SDValue Chain = ST->getChain();
438 SDValue Ptr = ST->getBasePtr();
439 SDValue Value = ST->getValue();
440 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
441 AAMDNodes AAInfo = ST->getAAInfo();
442 SDLoc dl(ST);
443
444 // Don't optimise TargetConstantFP
445 if (Value.getOpcode() == ISD::TargetConstantFP)
446 return SDValue();
447
448 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
449 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
450 TLI.isTypeLegal(VT: MVT::i32)) {
451 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
452 bitcastToAPInt().zextOrTrunc(width: 32),
453 DL: SDLoc(CFP), VT: MVT::i32);
454 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
455 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
456 }
457
458 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
459 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
460 // If this target supports 64-bit registers, do a single 64-bit store.
461 if (TLI.isTypeLegal(VT: MVT::i64)) {
462 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
463 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
464 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
465 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
466 }
467
468 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
469 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
470 // stores. If the target supports neither 32- nor 64-bits, this
471 // xform is certainly not worth it.
472 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
473 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
474 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
475 if (DAG.getDataLayout().isBigEndian())
476 std::swap(a&: Lo, b&: Hi);
477
478 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
479 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
480 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
481 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
482 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
483 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
484
485 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
486 }
487 }
488 }
489 return SDValue();
490}
491
492void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
493 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
494 SDValue Chain = ST->getChain();
495 SDValue Ptr = ST->getBasePtr();
496 SDLoc dl(Node);
497
498 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
499 AAMDNodes AAInfo = ST->getAAInfo();
500
501 if (!ST->isTruncatingStore()) {
502 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
503 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
504 ReplaceNode(Old: ST, New: OptStore);
505 return;
506 }
507
508 SDValue Value = ST->getValue();
509 MVT VT = Value.getSimpleValueType();
510 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
511 default: llvm_unreachable("This action is not supported yet!");
512 case TargetLowering::Legal: {
513 // If this is an unaligned store and the target doesn't support it,
514 // expand it.
515 EVT MemVT = ST->getMemoryVT();
516 const DataLayout &DL = DAG.getDataLayout();
517 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
518 MMO: *ST->getMemOperand())) {
519 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
520 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
521 ReplaceNode(Old: SDValue(ST, 0), New: Result);
522 } else
523 LLVM_DEBUG(dbgs() << "Legal store\n");
524 break;
525 }
526 case TargetLowering::Custom: {
527 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
528 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
529 if (Res && Res != SDValue(Node, 0))
530 ReplaceNode(Old: SDValue(Node, 0), New: Res);
531 return;
532 }
533 case TargetLowering::Promote: {
534 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
535 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
536 "Can only promote stores to same size type");
537 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
538 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
539 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
540 ReplaceNode(Old: SDValue(Node, 0), New: Result);
541 break;
542 }
543 }
544 return;
545 }
546
547 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
548 SDValue Value = ST->getValue();
549 EVT StVT = ST->getMemoryVT();
550 TypeSize StWidth = StVT.getSizeInBits();
551 TypeSize StSize = StVT.getStoreSizeInBits();
552 auto &DL = DAG.getDataLayout();
553
554 if (StWidth != StSize) {
555 // Promote to a byte-sized store with upper bits zero if not
556 // storing an integral number of bytes. For example, promote
557 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
558 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
559 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
560 SDValue Result =
561 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
562 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
563 ReplaceNode(Old: SDValue(Node, 0), New: Result);
564 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
565 // If not storing a power-of-2 number of bits, expand as two stores.
566 assert(!StVT.isVector() && "Unsupported truncstore!");
567 unsigned StWidthBits = StWidth.getFixedValue();
568 unsigned LogStWidth = Log2_32(Value: StWidthBits);
569 assert(LogStWidth < 32);
570 unsigned RoundWidth = 1 << LogStWidth;
571 assert(RoundWidth < StWidthBits);
572 unsigned ExtraWidth = StWidthBits - RoundWidth;
573 assert(ExtraWidth < RoundWidth);
574 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
575 "Store size not an integral number of bytes!");
576 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
577 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
578 SDValue Lo, Hi;
579 unsigned IncrementSize;
580
581 if (DL.isLittleEndian()) {
582 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
583 // Store the bottom RoundWidth bits.
584 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
585 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
586
587 // Store the remaining ExtraWidth bits.
588 IncrementSize = RoundWidth / 8;
589 Ptr =
590 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
591 Hi = DAG.getNode(
592 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
593 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Value.getValueType(), DL: dl));
594 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
595 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
596 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
597 } else {
598 // Big endian - avoid unaligned stores.
599 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
600 // Store the top RoundWidth bits.
601 Hi = DAG.getNode(
602 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
603 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Value.getValueType(), DL: dl));
604 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
605 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
606
607 // Store the remaining ExtraWidth bits.
608 IncrementSize = RoundWidth / 8;
609 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
610 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
611 VT: Ptr.getValueType()));
612 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
613 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
614 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
615 }
616
617 // The order of the stores doesn't matter.
618 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
619 ReplaceNode(Old: SDValue(Node, 0), New: Result);
620 } else {
621 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT,
622 Alignment: ST->getAlign(), AddrSpace: ST->getAddressSpace())) {
623 default:
624 llvm_unreachable("This action is not supported yet!");
625 case TargetLowering::Legal: {
626 EVT MemVT = ST->getMemoryVT();
627 // If this is an unaligned store and the target doesn't support it,
628 // expand it.
629 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
630 MMO: *ST->getMemOperand())) {
631 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
632 ReplaceNode(Old: SDValue(ST, 0), New: Result);
633 }
634 break;
635 }
636 case TargetLowering::Custom: {
637 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
638 if (Res && Res != SDValue(Node, 0))
639 ReplaceNode(Old: SDValue(Node, 0), New: Res);
640 return;
641 }
642 case TargetLowering::Expand:
643 assert(!StVT.isVector() &&
644 "Vector Stores are handled in LegalizeVectorOps");
645
646 SDValue Result;
647
648 // TRUNCSTORE:i16 i32 -> STORE i16
649 if (TLI.isTypeLegal(VT: StVT)) {
650 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
651 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
652 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
653 } else {
654 // The in-memory type isn't legal. Truncate to the type it would promote
655 // to, and then do a truncstore.
656 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
657 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
658 Operand: Value);
659 Result = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
660 SVT: StVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
661 }
662
663 ReplaceNode(Old: SDValue(Node, 0), New: Result);
664 break;
665 }
666 }
667}
668
669void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
670 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
671 SDValue Chain = LD->getChain(); // The chain.
672 SDValue Ptr = LD->getBasePtr(); // The base pointer.
673 SDValue Value; // The value returned by the load op.
674 SDLoc dl(Node);
675
676 ISD::LoadExtType ExtType = LD->getExtensionType();
677 if (ExtType == ISD::NON_EXTLOAD) {
678 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
679 MVT VT = Node->getSimpleValueType(ResNo: 0);
680 SDValue RVal = SDValue(Node, 0);
681 SDValue RChain = SDValue(Node, 1);
682
683 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
684 default: llvm_unreachable("This action is not supported yet!");
685 case TargetLowering::Legal: {
686 EVT MemVT = LD->getMemoryVT();
687 const DataLayout &DL = DAG.getDataLayout();
688 // If this is an unaligned load and the target doesn't support it,
689 // expand it.
690 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
691 MMO: *LD->getMemOperand())) {
692 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
693 }
694 break;
695 }
696 case TargetLowering::Custom:
697 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
698 RVal = Res;
699 RChain = Res.getValue(R: 1);
700 }
701 break;
702
703 case TargetLowering::Promote: {
704 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
705 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
706 "Can only promote loads to same size type");
707
708 // If the range metadata type does not match the legalized memory
709 // operation type, remove the range metadata.
710 if (const MDNode *MD = LD->getRanges()) {
711 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
712 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
713 !NVT.isInteger())
714 LD->getMemOperand()->clearRanges();
715 }
716 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
717 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
718 RChain = Res.getValue(R: 1);
719 break;
720 }
721 }
722 if (RChain.getNode() != Node) {
723 assert(RVal.getNode() != Node && "Load must be completely replaced");
724 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
725 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
726 if (UpdatedNodes) {
727 UpdatedNodes->insert(X: RVal.getNode());
728 UpdatedNodes->insert(X: RChain.getNode());
729 }
730 ReplacedNode(N: Node);
731 }
732 return;
733 }
734
735 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
736 EVT SrcVT = LD->getMemoryVT();
737 TypeSize SrcWidth = SrcVT.getSizeInBits();
738 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
739 AAMDNodes AAInfo = LD->getAAInfo();
740
741 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
742 // Some targets pretend to have an i1 loading operation, and actually
743 // load an i8. This trick is correct for ZEXTLOAD because the top 7
744 // bits are guaranteed to be zero; it helps the optimizers understand
745 // that these bits are zero. It is also useful for EXTLOAD, since it
746 // tells the optimizers that those bits are undefined. It would be
747 // nice to have an effective generic way of getting these benefits...
748 // Until such a way is found, don't insist on promoting i1 here.
749 (SrcVT != MVT::i1 ||
750 TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1, Alignment: LD->getAlign(),
751 AddrSpace: LD->getAddressSpace(), ExtType,
752 Atomic: false) == TargetLowering::Promote)) {
753 // Promote to a byte-sized load if not loading an integral number of
754 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
755 unsigned NewWidth = SrcVT.getStoreSizeInBits();
756 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
757 SDValue Ch;
758
759 // The extra bits are guaranteed to be zero, since we stored them that
760 // way. A zext load from NVT thus automatically gives zext from SrcVT.
761
762 ISD::LoadExtType NewExtType =
763 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
764
765 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
766 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
767 Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
768
769 Ch = Result.getValue(R: 1); // The chain.
770
771 if (ExtType == ISD::SEXTLOAD)
772 // Having the top bits zero doesn't help when sign extending.
773 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
774 VT: Result.getValueType(),
775 N1: Result, N2: DAG.getValueType(SrcVT));
776 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
777 // All the top bits are guaranteed to be zero - inform the optimizers.
778 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
779 VT: Result.getValueType(), N1: Result,
780 N2: DAG.getValueType(SrcVT));
781
782 Value = Result;
783 Chain = Ch;
784 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
785 // If not loading a power-of-2 number of bits, expand as two loads.
786 assert(!SrcVT.isVector() && "Unsupported extload!");
787 unsigned SrcWidthBits = SrcWidth.getFixedValue();
788 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
789 assert(LogSrcWidth < 32);
790 unsigned RoundWidth = 1 << LogSrcWidth;
791 assert(RoundWidth < SrcWidthBits);
792 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
793 assert(ExtraWidth < RoundWidth);
794 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
795 "Load size not an integral number of bytes!");
796 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
797 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
798 SDValue Lo, Hi, Ch;
799 unsigned IncrementSize;
800 auto &DL = DAG.getDataLayout();
801
802 if (DL.isLittleEndian()) {
803 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
804 // Load the bottom RoundWidth bits.
805 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
806 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
807 MMOFlags, Metadata: AAInfo);
808
809 // Load the remaining ExtraWidth bits.
810 IncrementSize = RoundWidth / 8;
811 Ptr =
812 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
813 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
814 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
815 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
816
817 // Build a factor node to remember that this load is independent of
818 // the other one.
819 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
820 N2: Hi.getValue(R: 1));
821
822 // Move the top bits to the right place.
823 Hi = DAG.getNode(
824 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
825 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Hi.getValueType(), DL: dl));
826
827 // Join the hi and lo parts.
828 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
829 } else {
830 // Big endian - avoid unaligned loads.
831 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
832 // Load the top RoundWidth bits.
833 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
834 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
835 MMOFlags, Metadata: AAInfo);
836
837 // Load the remaining ExtraWidth bits.
838 IncrementSize = RoundWidth / 8;
839 Ptr =
840 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
841 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
842 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
843 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
844
845 // Build a factor node to remember that this load is independent of
846 // the other one.
847 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
848 N2: Hi.getValue(R: 1));
849
850 // Move the top bits to the right place.
851 Hi = DAG.getNode(
852 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
853 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Hi.getValueType(), DL: dl));
854
855 // Join the hi and lo parts.
856 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
857 }
858
859 Chain = Ch;
860 } else {
861 bool isCustom = false;
862 switch (TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: SrcVT.getSimpleVT(),
863 Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(), ExtType,
864 Atomic: false)) {
865 default:
866 llvm_unreachable("This action is not supported yet!");
867 case TargetLowering::Custom:
868 isCustom = true;
869 [[fallthrough]];
870 case TargetLowering::Legal:
871 Value = SDValue(Node, 0);
872 Chain = SDValue(Node, 1);
873
874 if (isCustom) {
875 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
876 Value = Res;
877 Chain = Res.getValue(R: 1);
878 }
879 } else {
880 // If this is an unaligned load and the target doesn't support it,
881 // expand it.
882 EVT MemVT = LD->getMemoryVT();
883 const DataLayout &DL = DAG.getDataLayout();
884 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
885 MMO: *LD->getMemOperand())) {
886 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
887 }
888 }
889 break;
890
891 case TargetLowering::Expand: {
892 EVT DestVT = Node->getValueType(ResNo: 0);
893 if (!TLI.isLoadLegal(ValVT: DestVT, MemVT: SrcVT, Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(),
894 ExtType: ISD::EXTLOAD, Atomic: false)) {
895 // If the source type is not legal, see if there is a legal extload to
896 // an intermediate type that we can then extend further.
897 EVT LoadVT =
898 TLI.getRegisterType(Context&: *DAG.getContext(), VT: SrcVT.getSimpleVT());
899 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
900 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
901 TLI.isLoadLegal(ValVT: LoadVT, MemVT: SrcVT, Alignment: LD->getAlign(),
902 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false))) {
903 // If we are loading a legal type, this is a non-extload followed by a
904 // full extend.
905 ISD::LoadExtType MidExtType =
906 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
907
908 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
909 MemVT: SrcVT, MMO: LD->getMemOperand());
910 unsigned ExtendOp =
911 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
912 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
913 Chain = Load.getValue(R: 1);
914 break;
915 }
916
917 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
918 // normal undefined upper bits behavior to allow using an in-reg extend
919 // with the illegal FP type, so load as an integer and do the
920 // from-integer conversion.
921 EVT SVT = SrcVT.getScalarType();
922 if (SVT == MVT::f16 || SVT == MVT::bf16) {
923 EVT ISrcVT = SrcVT.changeTypeToInteger();
924 EVT IDestVT = DestVT.changeTypeToInteger();
925 EVT ILoadVT =
926 TLI.getRegisterType(Context&: *DAG.getContext(), VT: IDestVT.getSimpleVT());
927
928 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
929 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
930 Value =
931 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
932 DL: dl, VT: DestVT, Operand: Result);
933 Chain = Result.getValue(R: 1);
934 break;
935 }
936 }
937
938 assert(!SrcVT.isVector() &&
939 "Vector Loads are handled in LegalizeVectorOps");
940
941 // FIXME: This does not work for vectors on most targets. Sign-
942 // and zero-extend operations are currently folded into extending
943 // loads, whether they are legal or not, and then we end up here
944 // without any support for legalizing them.
945 assert(ExtType != ISD::EXTLOAD &&
946 "EXTLOAD should always be supported!");
947 // Turn the unsupported load into an EXTLOAD followed by an
948 // explicit zero/sign extend inreg.
949 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
950 VT: Node->getValueType(ResNo: 0),
951 Chain, Ptr, MemVT: SrcVT,
952 MMO: LD->getMemOperand());
953 SDValue ValRes;
954 if (ExtType == ISD::SEXTLOAD)
955 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
956 VT: Result.getValueType(),
957 N1: Result, N2: DAG.getValueType(SrcVT));
958 else
959 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
960 Value = ValRes;
961 Chain = Result.getValue(R: 1);
962 break;
963 }
964 }
965 }
966
967 // Since loads produce two values, make sure to remember that we legalized
968 // both of them.
969 if (Chain.getNode() != Node) {
970 assert(Value.getNode() != Node && "Load must be completely replaced");
971 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
972 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
973 if (UpdatedNodes) {
974 UpdatedNodes->insert(X: Value.getNode());
975 UpdatedNodes->insert(X: Chain.getNode());
976 }
977 ReplacedNode(N: Node);
978 }
979}
980
981/// Return a legal replacement for the given operation, with all legal operands.
982void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
983 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
984
985 // Allow illegal target nodes and illegal registers.
986 if (Node->getOpcode() == ISD::TargetConstant ||
987 Node->getOpcode() == ISD::Register)
988 return;
989
990#ifndef NDEBUG
991 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
992 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
993 TargetLowering::TypeLegal &&
994 "Unexpected illegal type!");
995
996 for (const SDValue &Op : Node->op_values())
997 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
998 TargetLowering::TypeLegal ||
999 Op.getOpcode() == ISD::TargetConstant ||
1000 Op.getOpcode() == ISD::Register) &&
1001 "Unexpected illegal type!");
1002#endif
1003
1004 // Figure out the correct action; the way to query this varies by opcode
1005 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1006 bool SimpleFinishLegalizing = true;
1007 switch (Node->getOpcode()) {
1008 case ISD::INTRINSIC_W_CHAIN:
1009 case ISD::INTRINSIC_WO_CHAIN:
1010 case ISD::INTRINSIC_VOID:
1011 case ISD::STACKSAVE:
1012 case ISD::STACKADDRESS:
1013 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1014 break;
1015 case ISD::GET_DYNAMIC_AREA_OFFSET:
1016 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1017 VT: Node->getValueType(ResNo: 0));
1018 break;
1019 case ISD::VAARG:
1020 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1021 VT: Node->getValueType(ResNo: 0));
1022 if (Action != TargetLowering::Promote)
1023 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1024 break;
1025 case ISD::SET_FPENV:
1026 case ISD::SET_FPMODE:
1027 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1028 VT: Node->getOperand(Num: 1).getValueType());
1029 break;
1030 case ISD::FP_TO_FP16:
1031 case ISD::FP_TO_BF16:
1032 case ISD::SINT_TO_FP:
1033 case ISD::UINT_TO_FP:
1034 case ISD::EXTRACT_VECTOR_ELT:
1035 case ISD::LROUND:
1036 case ISD::LLROUND:
1037 case ISD::LRINT:
1038 case ISD::LLRINT:
1039 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1040 VT: Node->getOperand(Num: 0).getValueType());
1041 break;
1042 case ISD::STRICT_FP_TO_FP16:
1043 case ISD::STRICT_FP_TO_BF16:
1044 case ISD::STRICT_SINT_TO_FP:
1045 case ISD::STRICT_UINT_TO_FP:
1046 case ISD::STRICT_LRINT:
1047 case ISD::STRICT_LLRINT:
1048 case ISD::STRICT_LROUND:
1049 case ISD::STRICT_LLROUND:
1050 // These pseudo-ops are the same as the other STRICT_ ops except
1051 // they are registered with setOperationAction() using the input type
1052 // instead of the output type.
1053 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1054 VT: Node->getOperand(Num: 1).getValueType());
1055 break;
1056 case ISD::SIGN_EXTEND_INREG: {
1057 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1058 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1059 break;
1060 }
1061 case ISD::ATOMIC_STORE:
1062 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1063 VT: Node->getOperand(Num: 1).getValueType());
1064 break;
1065 case ISD::SELECT_CC:
1066 case ISD::STRICT_FSETCC:
1067 case ISD::STRICT_FSETCCS:
1068 case ISD::SETCC:
1069 case ISD::SETCCCARRY:
1070 case ISD::BR_CC: {
1071 unsigned Opc = Node->getOpcode();
1072 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1073 : Opc == ISD::STRICT_FSETCC ? 3
1074 : Opc == ISD::STRICT_FSETCCS ? 3
1075 : Opc == ISD::SETCCCARRY ? 3
1076 : Opc == ISD::SETCC ? 2
1077 : 1;
1078 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1079 : Opc == ISD::STRICT_FSETCC ? 1
1080 : Opc == ISD::STRICT_FSETCCS ? 1
1081 : 0;
1082 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1083 ISD::CondCode CCCode =
1084 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1085 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1086 if (Action == TargetLowering::Legal) {
1087 if (Node->getOpcode() == ISD::SELECT_CC)
1088 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1089 VT: Node->getValueType(ResNo: 0));
1090 else
1091 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1092 }
1093 break;
1094 }
1095 case ISD::LOAD:
1096 case ISD::STORE:
1097 // FIXME: Model these properly. LOAD and STORE are complicated, and
1098 // STORE expects the unlegalized operand in some cases.
1099 SimpleFinishLegalizing = false;
1100 break;
1101 case ISD::CALLSEQ_START:
1102 case ISD::CALLSEQ_END:
1103 // FIXME: This shouldn't be necessary. These nodes have special properties
1104 // dealing with the recursive nature of legalization. Removing this
1105 // special case should be done as part of making LegalizeDAG non-recursive.
1106 SimpleFinishLegalizing = false;
1107 break;
1108 case ISD::EXTRACT_ELEMENT:
1109 case ISD::GET_ROUNDING:
1110 case ISD::MERGE_VALUES:
1111 case ISD::EH_RETURN:
1112 case ISD::FRAME_TO_ARGS_OFFSET:
1113 case ISD::EH_DWARF_CFA:
1114 case ISD::EH_SJLJ_SETJMP:
1115 case ISD::EH_SJLJ_LONGJMP:
1116 case ISD::EH_SJLJ_SETUP_DISPATCH:
1117 // These operations lie about being legal: when they claim to be legal,
1118 // they should actually be expanded.
1119 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1120 if (Action == TargetLowering::Legal)
1121 Action = TargetLowering::Expand;
1122 break;
1123 case ISD::INIT_TRAMPOLINE:
1124 case ISD::ADJUST_TRAMPOLINE:
1125 case ISD::FRAMEADDR:
1126 case ISD::RETURNADDR:
1127 case ISD::ADDROFRETURNADDR:
1128 case ISD::SPONENTRY:
1129 // These operations lie about being legal: when they claim to be legal,
1130 // they should actually be custom-lowered.
1131 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1132 if (Action == TargetLowering::Legal)
1133 Action = TargetLowering::Custom;
1134 break;
1135 case ISD::CLEAR_CACHE:
1136 // This operation is typically going to be LibCall unless the target wants
1137 // something differrent.
1138 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1139 break;
1140 case ISD::READCYCLECOUNTER:
1141 case ISD::READSTEADYCOUNTER:
1142 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1143 // legalization might have expanded that to several smaller types.
1144 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1145 break;
1146 case ISD::READ_REGISTER:
1147 case ISD::WRITE_REGISTER:
1148 // Named register is legal in the DAG, but blocked by register name
1149 // selection if not implemented by target (to chose the correct register)
1150 // They'll be converted to Copy(To/From)Reg.
1151 Action = TargetLowering::Legal;
1152 break;
1153 case ISD::UBSANTRAP:
1154 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1155 if (Action == TargetLowering::Expand) {
1156 // replace ISD::UBSANTRAP with ISD::TRAP
1157 SDValue NewVal;
1158 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1159 N: Node->getOperand(Num: 0));
1160 ReplaceNode(Old: Node, New: NewVal.getNode());
1161 LegalizeOp(Node: NewVal.getNode());
1162 return;
1163 }
1164 break;
1165 case ISD::DEBUGTRAP:
1166 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1167 if (Action == TargetLowering::Expand) {
1168 // replace ISD::DEBUGTRAP with ISD::TRAP
1169 SDValue NewVal;
1170 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1171 N: Node->getOperand(Num: 0));
1172 ReplaceNode(Old: Node, New: NewVal.getNode());
1173 LegalizeOp(Node: NewVal.getNode());
1174 return;
1175 }
1176 break;
1177 case ISD::SADDSAT:
1178 case ISD::UADDSAT:
1179 case ISD::SSUBSAT:
1180 case ISD::USUBSAT:
1181 case ISD::SSHLSAT:
1182 case ISD::USHLSAT:
1183 case ISD::SCMP:
1184 case ISD::UCMP:
1185 case ISD::FP_TO_SINT_SAT:
1186 case ISD::FP_TO_UINT_SAT:
1187 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1188 break;
1189 case ISD::SMULFIX:
1190 case ISD::SMULFIXSAT:
1191 case ISD::UMULFIX:
1192 case ISD::UMULFIXSAT:
1193 case ISD::SDIVFIX:
1194 case ISD::SDIVFIXSAT:
1195 case ISD::UDIVFIX:
1196 case ISD::UDIVFIXSAT: {
1197 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1198 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1199 VT: Node->getValueType(ResNo: 0), Scale);
1200 break;
1201 }
1202 case ISD::MSCATTER:
1203 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1204 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1205 break;
1206 case ISD::MSTORE:
1207 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1208 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1209 break;
1210 case ISD::VP_SCATTER:
1211 Action = TLI.getOperationAction(
1212 Op: Node->getOpcode(),
1213 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1214 break;
1215 case ISD::VP_STORE:
1216 Action = TLI.getOperationAction(
1217 Op: Node->getOpcode(),
1218 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1219 break;
1220 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1221 Action = TLI.getOperationAction(
1222 Op: Node->getOpcode(),
1223 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1224 break;
1225 case ISD::VECREDUCE_FADD:
1226 case ISD::VECREDUCE_FMUL:
1227 case ISD::VECREDUCE_ADD:
1228 case ISD::VECREDUCE_MUL:
1229 case ISD::VECREDUCE_AND:
1230 case ISD::VECREDUCE_OR:
1231 case ISD::VECREDUCE_XOR:
1232 case ISD::VECREDUCE_SMAX:
1233 case ISD::VECREDUCE_SMIN:
1234 case ISD::VECREDUCE_UMAX:
1235 case ISD::VECREDUCE_UMIN:
1236 case ISD::VECREDUCE_FMAX:
1237 case ISD::VECREDUCE_FMIN:
1238 case ISD::VECREDUCE_FMAXIMUM:
1239 case ISD::VECREDUCE_FMINIMUM:
1240 case ISD::VECREDUCE_FMAXIMUMNUM:
1241 case ISD::VECREDUCE_FMINIMUMNUM:
1242 case ISD::IS_FPCLASS:
1243 Action = TLI.getOperationAction(
1244 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1245 break;
1246 case ISD::VECREDUCE_SEQ_FADD:
1247 case ISD::VECREDUCE_SEQ_FMUL:
1248 case ISD::VP_REDUCE_FADD:
1249 case ISD::VP_REDUCE_FMUL:
1250 case ISD::VP_REDUCE_ADD:
1251 case ISD::VP_REDUCE_MUL:
1252 case ISD::VP_REDUCE_AND:
1253 case ISD::VP_REDUCE_OR:
1254 case ISD::VP_REDUCE_XOR:
1255 case ISD::VP_REDUCE_SMAX:
1256 case ISD::VP_REDUCE_SMIN:
1257 case ISD::VP_REDUCE_UMAX:
1258 case ISD::VP_REDUCE_UMIN:
1259 case ISD::VP_REDUCE_FMAX:
1260 case ISD::VP_REDUCE_FMIN:
1261 case ISD::VP_REDUCE_FMAXIMUM:
1262 case ISD::VP_REDUCE_FMINIMUM:
1263 case ISD::VP_REDUCE_SEQ_FADD:
1264 case ISD::VP_REDUCE_SEQ_FMUL:
1265 Action = TLI.getOperationAction(
1266 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1267 break;
1268 case ISD::CTTZ_ELTS:
1269 case ISD::CTTZ_ELTS_ZERO_POISON:
1270 case ISD::VP_CTTZ_ELTS:
1271 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
1272 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1273 VT: Node->getOperand(Num: 0).getValueType());
1274 break;
1275 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1276 Action = TLI.getOperationAction(
1277 Op: Node->getOpcode(),
1278 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1279 break;
1280 default:
1281 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1282 Action = TLI.getCustomOperationAction(Op&: *Node);
1283 } else {
1284 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1285 }
1286 break;
1287 }
1288
1289 if (SimpleFinishLegalizing) {
1290 SDNode *NewNode = Node;
1291 switch (Node->getOpcode()) {
1292 default: break;
1293 case ISD::SHL:
1294 case ISD::SRL:
1295 case ISD::SRA:
1296 case ISD::ROTL:
1297 case ISD::ROTR:
1298 case ISD::SSHLSAT:
1299 case ISD::USHLSAT: {
1300 // Legalizing shifts/rotates requires adjusting the shift amount
1301 // to the appropriate width.
1302 SDValue Op0 = Node->getOperand(Num: 0);
1303 SDValue Op1 = Node->getOperand(Num: 1);
1304 if (!Op1.getValueType().isVector()) {
1305 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1306 // The getShiftAmountOperand() may create a new operand node or
1307 // return the existing one. If new operand is created we need
1308 // to update the parent node.
1309 // Do not try to legalize SAO here! It will be automatically legalized
1310 // in the next round.
1311 if (SAO != Op1)
1312 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1313 }
1314 break;
1315 }
1316 case ISD::FSHL:
1317 case ISD::FSHR:
1318 case ISD::SRL_PARTS:
1319 case ISD::SRA_PARTS:
1320 case ISD::SHL_PARTS: {
1321 // Legalizing shifts/rotates requires adjusting the shift amount
1322 // to the appropriate width.
1323 SDValue Op0 = Node->getOperand(Num: 0);
1324 SDValue Op1 = Node->getOperand(Num: 1);
1325 SDValue Op2 = Node->getOperand(Num: 2);
1326 if (!Op2.getValueType().isVector()) {
1327 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1328 // The getShiftAmountOperand() may create a new operand node or
1329 // return the existing one. If new operand is created we need
1330 // to update the parent node.
1331 if (SAO != Op2)
1332 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1333 }
1334 break;
1335 }
1336 }
1337
1338 if (NewNode != Node) {
1339 ReplaceNode(Old: Node, New: NewNode);
1340 Node = NewNode;
1341 }
1342 switch (Action) {
1343 case TargetLowering::Legal:
1344 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1345 return;
1346 case TargetLowering::Custom:
1347 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1348 // FIXME: The handling for custom lowering with multiple results is
1349 // a complete mess.
1350 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1351 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1352 return;
1353
1354 if (Node->getNumValues() == 1) {
1355 // Verify the new types match the original. Glue is waived because
1356 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1357 assert((Res.getValueType() == Node->getValueType(0) ||
1358 Node->getValueType(0) == MVT::Glue) &&
1359 "Type mismatch for custom legalized operation");
1360 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1361 // We can just directly replace this node with the lowered value.
1362 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1363 return;
1364 }
1365
1366 SmallVector<SDValue, 8> ResultVals;
1367 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1368 // Verify the new types match the original. Glue is waived because
1369 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1370 assert((Res->getValueType(i) == Node->getValueType(i) ||
1371 Node->getValueType(i) == MVT::Glue) &&
1372 "Type mismatch for custom legalized operation");
1373 ResultVals.push_back(Elt: Res.getValue(R: i));
1374 }
1375 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1376 ReplaceNode(Old: Node, New: ResultVals.data());
1377 return;
1378 }
1379 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1380 [[fallthrough]];
1381 case TargetLowering::Expand:
1382 if (ExpandNode(Node))
1383 return;
1384 [[fallthrough]];
1385 case TargetLowering::LibCall:
1386 ConvertNodeToLibcall(Node);
1387 return;
1388 case TargetLowering::Promote:
1389 PromoteNode(Node);
1390 return;
1391 }
1392 }
1393
1394 switch (Node->getOpcode()) {
1395 default:
1396#ifndef NDEBUG
1397 dbgs() << "NODE: ";
1398 Node->dump( &DAG);
1399 dbgs() << "\n";
1400#endif
1401 llvm_unreachable("Do not know how to legalize this operator!");
1402
1403 case ISD::CALLSEQ_START:
1404 case ISD::CALLSEQ_END:
1405 break;
1406 case ISD::LOAD:
1407 return LegalizeLoadOps(Node);
1408 case ISD::STORE:
1409 return LegalizeStoreOps(Node);
1410 }
1411}
1412
1413SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1414 SDValue Vec = Op.getOperand(i: 0);
1415 SDValue Idx = Op.getOperand(i: 1);
1416 SDLoc dl(Op);
1417
1418 // Before we generate a new store to a temporary stack slot, see if there is
1419 // already one that we can use. There often is because when we scalarize
1420 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1421 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1422 // the vector. If all are expanded here, we don't want one store per vector
1423 // element.
1424
1425 // Caches for hasPredecessorHelper
1426 SmallPtrSet<const SDNode *, 32> Visited;
1427 SmallVector<const SDNode *, 16> Worklist;
1428 Visited.insert(Ptr: Op.getNode());
1429 Worklist.push_back(Elt: Idx.getNode());
1430 SDValue StackPtr, Ch;
1431 for (SDNode *User : Vec.getNode()->users()) {
1432 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1433 if (ST->isIndexed() || ST->isTruncatingStore() ||
1434 ST->getValue() != Vec)
1435 continue;
1436
1437 // Make sure that nothing else could have stored into the destination of
1438 // this store.
1439 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1440 continue;
1441
1442 // If the index is dependent on the store we will introduce a cycle when
1443 // creating the load (the load uses the index, and by replacing the chain
1444 // we will make the index dependent on the load). Also, the store might be
1445 // dependent on the extractelement and introduce a cycle when creating
1446 // the load.
1447 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1448 ST->hasPredecessor(N: Op.getNode()))
1449 continue;
1450
1451 StackPtr = ST->getBasePtr();
1452 Ch = SDValue(ST, 0);
1453 break;
1454 }
1455 }
1456
1457 EVT VecVT = Vec.getValueType();
1458
1459 if (!Ch.getNode()) {
1460 // Store the value to a temporary stack slot, then LOAD the returned part.
1461 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1462 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1463 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1464 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1465 }
1466
1467 SDValue NewLoad;
1468 Align ElementAlignment =
1469 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1470 b: DAG.getDataLayout().getPrefTypeAlign(
1471 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1472
1473 if (Op.getValueType().isVector()) {
1474 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1475 SubVecVT: Op.getValueType(), Index: Idx);
1476 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1477 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1478 } else {
1479 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1480 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1481 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1482 Alignment: ElementAlignment);
1483 }
1484
1485 // Replace the chain going out of the store, by the one out of the load.
1486 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1487
1488 // We introduced a cycle though, so update the loads operands, making sure
1489 // to use the original store's chain as an incoming chain.
1490 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1491 NewLoadOperands[0] = Ch;
1492 NewLoad =
1493 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1494 return NewLoad;
1495}
1496
1497SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1498 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1499
1500 SDValue Vec = Op.getOperand(i: 0);
1501 SDValue Part = Op.getOperand(i: 1);
1502 SDValue Idx = Op.getOperand(i: 2);
1503 SDLoc dl(Op);
1504
1505 // Store the value to a temporary stack slot, then LOAD the returned part.
1506 EVT VecVT = Vec.getValueType();
1507 EVT PartVT = Part.getValueType();
1508 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1509 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1510 MachinePointerInfo PtrInfo =
1511 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1512
1513 // First store the whole vector.
1514 Align BaseVecAlignment =
1515 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1516 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1517 Alignment: BaseVecAlignment);
1518
1519 // Freeze the index so we don't poison the clamping code we're about to emit.
1520 Idx = DAG.getFreeze(V: Idx);
1521
1522 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1523 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1524
1525 // Then store the inserted part.
1526 if (PartVT.isVector()) {
1527 SDValue SubStackPtr =
1528 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1529
1530 // Store the subvector.
1531 Ch = DAG.getStore(
1532 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1533 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1534 Alignment: PartAlignment);
1535 } else {
1536 SDValue SubStackPtr =
1537 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1538
1539 // Store the scalar value.
1540 Ch = DAG.getTruncStore(
1541 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1542 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1543 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1544 }
1545
1546 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1547 "ElementAlignment does not match!");
1548
1549 // Finally, load the updated vector.
1550 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1551 Alignment: BaseVecAlignment);
1552}
1553
1554SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1555 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1556 SDLoc DL(Node);
1557 SmallVector<SDValue, 16> Ops;
1558 unsigned NumOperands = Node->getNumOperands();
1559 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1560 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1561 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1562 EVT ElementValueType = TLI.getTypeToTransformTo(
1563 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1564 for (unsigned I = 0; I < NumOperands; ++I) {
1565 SDValue SubOp = Node->getOperand(Num: I);
1566 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1567 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1568 N1: SubOp,
1569 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1570 }
1571 }
1572 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1573}
1574
1575SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1576 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1577 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1578 "Unexpected opcode!");
1579
1580 // We can't handle this case efficiently. Allocate a sufficiently
1581 // aligned object on the stack, store each operand into it, then load
1582 // the result as a vector.
1583 // Create the stack frame object.
1584 EVT VT = Node->getValueType(ResNo: 0);
1585 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1586 : Node->getOperand(Num: 0).getValueType();
1587 SDLoc dl(Node);
1588 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1589 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1590 MachinePointerInfo PtrInfo =
1591 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1592
1593 // Emit a store of each element to the stack slot.
1594 SmallVector<SDValue, 8> Stores;
1595 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1596 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1597
1598 // If the destination vector element type of a BUILD_VECTOR is narrower than
1599 // the source element type, only store the bits necessary.
1600 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1601 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1602
1603 // Store (in the right endianness) the elements to memory.
1604 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1605 // Ignore undef elements.
1606 if (Node->getOperand(Num: i).isUndef()) continue;
1607
1608 unsigned Offset = TypeByteSize*i;
1609
1610 SDValue Idx =
1611 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1612
1613 if (Truncate)
1614 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1615 Val: Node->getOperand(Num: i), Ptr: Idx,
1616 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1617 else
1618 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1619 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1620 }
1621
1622 SDValue StoreChain;
1623 if (!Stores.empty()) // Not all undef elements?
1624 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1625 else
1626 StoreChain = DAG.getEntryNode();
1627
1628 // Result is a load from the stack slot.
1629 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1630}
1631
1632/// Bitcast a floating-point value to an integer value. Only bitcast the part
1633/// containing the sign bit if the target has no integer value capable of
1634/// holding all bits of the floating-point value.
1635void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1636 const SDLoc &DL,
1637 SDValue Value) const {
1638 EVT FloatVT = Value.getValueType();
1639 unsigned NumBits = FloatVT.getScalarSizeInBits();
1640 State.FloatVT = FloatVT;
1641 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1642 // Convert to an integer of the same size.
1643 if (TLI.isTypeLegal(VT: IVT)) {
1644 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1645 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1646 State.SignBit = NumBits - 1;
1647 return;
1648 }
1649
1650 auto &DataLayout = DAG.getDataLayout();
1651 // Store the float to memory, then load the sign part out as an integer.
1652 MVT LoadTy = TLI.getRegisterType(Context&: *DAG.getContext(), VT: MVT::i8);
1653 // First create a temporary that is aligned for both the load and store.
1654 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1655 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1656 // Then store the float to it.
1657 State.FloatPtr = StackPtr;
1658 MachineFunction &MF = DAG.getMachineFunction();
1659 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1660 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1661 PtrInfo: State.FloatPointerInfo);
1662
1663 SDValue IntPtr;
1664 if (DataLayout.isBigEndian()) {
1665 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1666 // Load out a legal integer with the same sign bit as the float.
1667 IntPtr = StackPtr;
1668 State.IntPointerInfo = State.FloatPointerInfo;
1669 } else {
1670 // Advance the pointer so that the loaded byte will contain the sign bit.
1671 unsigned ByteOffset = (NumBits / 8) - 1;
1672 IntPtr =
1673 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1674 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1675 Offset: ByteOffset);
1676 }
1677
1678 State.IntPtr = IntPtr;
1679 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1680 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1681 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1682 State.SignBit = 7;
1683}
1684
1685/// Replace the integer value produced by getSignAsIntValue() with a new value
1686/// and cast the result back to a floating-point type.
1687SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1688 const SDLoc &DL,
1689 SDValue NewIntValue) const {
1690 if (!State.Chain)
1691 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1692
1693 // Override the part containing the sign bit in the value stored on the stack.
1694 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1695 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1696 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1697 PtrInfo: State.FloatPointerInfo);
1698}
1699
1700SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1701 SDLoc DL(Node);
1702 SDValue Mag = Node->getOperand(Num: 0);
1703 SDValue Sign = Node->getOperand(Num: 1);
1704
1705 if (Sign.getValueType().isVector())
1706 return DAG.UnrollVectorOp(N: Node);
1707
1708 // Get sign bit into an integer value.
1709 FloatSignAsInt SignAsInt;
1710 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1711
1712 EVT IntVT = SignAsInt.IntValue.getValueType();
1713 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1714 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1715 N2: SignMask);
1716
1717 // If FABS is legal transform
1718 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1719 EVT FloatVT = Mag.getValueType();
1720 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1721 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1722 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1723 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1724 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1725 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1726 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1727 }
1728
1729 // Transform Mag value to integer, and clear the sign bit.
1730 FloatSignAsInt MagAsInt;
1731 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1732 EVT MagVT = MagAsInt.IntValue.getValueType();
1733 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1734 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1735 N2: ClearSignMask);
1736
1737 // Get the signbit at the right position for MagAsInt.
1738 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1739 EVT ShiftVT = IntVT;
1740 if (SignBit.getScalarValueSizeInBits() <
1741 ClearedSign.getScalarValueSizeInBits()) {
1742 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1743 ShiftVT = MagVT;
1744 }
1745 if (ShiftAmount > 0) {
1746 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1747 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1748 } else if (ShiftAmount < 0) {
1749 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1750 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1751 }
1752 if (SignBit.getScalarValueSizeInBits() >
1753 ClearedSign.getScalarValueSizeInBits()) {
1754 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1755 }
1756
1757 // Store the part with the modified sign and convert back to float.
1758 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1759 Flags: SDNodeFlags::Disjoint);
1760
1761 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1762}
1763
1764SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1765 // Get the sign bit as an integer.
1766 SDLoc DL(Node);
1767 if (Node->getValueType(ResNo: 0).isVector())
1768 return DAG.UnrollVectorOp(N: Node);
1769
1770 FloatSignAsInt SignAsInt;
1771 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1772 EVT IntVT = SignAsInt.IntValue.getValueType();
1773
1774 // Flip the sign.
1775 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1776 SDValue SignFlip =
1777 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1778
1779 // Convert back to float.
1780 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1781}
1782
1783SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1784 SDLoc DL(Node);
1785 SDValue Value = Node->getOperand(Num: 0);
1786
1787 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1788 EVT FloatVT = Value.getValueType();
1789 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1790 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1791 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1792 }
1793
1794 if (FloatVT.isVector())
1795 return DAG.UnrollVectorOp(N: Node);
1796
1797 // Transform value to integer, clear the sign bit and transform back.
1798 FloatSignAsInt ValueAsInt;
1799 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1800 EVT IntVT = ValueAsInt.IntValue.getValueType();
1801 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1802 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1803 N2: ClearSignMask);
1804 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1805}
1806
1807void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1808 SmallVectorImpl<SDValue> &Results) {
1809 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1810 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1811 " not tell us which reg is the stack pointer!");
1812 SDLoc dl(Node);
1813 EVT VT = Node->getValueType(ResNo: 0);
1814 SDValue Tmp1 = SDValue(Node, 0);
1815 SDValue Tmp2 = SDValue(Node, 1);
1816 SDValue Tmp3 = Node->getOperand(Num: 2);
1817 SDValue Chain = Tmp1.getOperand(i: 0);
1818
1819 // Chain the dynamic stack allocation so that it doesn't modify the stack
1820 // pointer when other instructions are using the stack.
1821 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1822
1823 SDValue Size = Tmp2.getOperand(i: 1);
1824 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1825 Chain = SP.getValue(R: 1);
1826 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1827 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1828 unsigned Opc =
1829 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1830 ISD::ADD : ISD::SUB;
1831
1832 Align StackAlign = TFL->getStackAlign();
1833 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1834 if (Alignment > StackAlign)
1835 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1836 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1837 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1838
1839 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1840
1841 Results.push_back(Elt: Tmp1);
1842 Results.push_back(Elt: Tmp2);
1843}
1844
1845/// Emit a store/load combination to the stack. This stores
1846/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1847/// a load from the stack slot to DestVT, extending it if needed.
1848/// The resultant code need not be legal.
1849SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1850 EVT DestVT, const SDLoc &dl) {
1851 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1852}
1853
1854SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1855 EVT DestVT, const SDLoc &dl,
1856 SDValue Chain) {
1857 EVT SrcVT = SrcOp.getValueType();
1858 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1859 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1860
1861 // Don't convert with stack if the load/store is expensive.
1862 if ((SrcVT.bitsGT(VT: SlotVT) && !TLI.isTruncStoreLegalOrCustom(
1863 ValVT: SrcOp.getValueType(), MemVT: SlotVT, Alignment: DestAlign,
1864 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace())) ||
1865 (SlotVT.bitsLT(VT: DestVT) &&
1866 !TLI.isLoadLegalOrCustom(ValVT: DestVT, MemVT: SlotVT, Alignment: DestAlign,
1867 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace(),
1868 ExtType: ISD::EXTLOAD, Atomic: false)))
1869 return SDValue();
1870
1871 // Create the stack frame object.
1872 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1873 Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1874 SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1875
1876 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1877 int SPFI = StackPtrFI->getIndex();
1878 MachinePointerInfo PtrInfo =
1879 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1880
1881 // Emit a store to the stack slot. Use a truncstore if the input value is
1882 // later than DestVT.
1883 SDValue Store;
1884
1885 if (SrcVT.bitsGT(VT: SlotVT))
1886 Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1887 SVT: SlotVT, Alignment: SrcAlign);
1888 else {
1889 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1890 Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1891 }
1892
1893 // Result is a load from the stack slot.
1894 if (SlotVT.bitsEq(VT: DestVT))
1895 return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1896
1897 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1898 return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1899 Alignment: DestAlign);
1900}
1901
1902SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1903 SDLoc dl(Node);
1904 // Create a vector sized/aligned stack slot, store the value to element #0,
1905 // then load the whole vector back out.
1906 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1907
1908 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1909 int SPFI = StackPtrFI->getIndex();
1910
1911 SDValue Ch = DAG.getTruncStore(
1912 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1913 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1914 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1915 return DAG.getLoad(
1916 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1917 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1918}
1919
1920static bool
1921ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1922 const TargetLowering &TLI, SDValue &Res) {
1923 unsigned NumElems = Node->getNumOperands();
1924 SDLoc dl(Node);
1925 EVT VT = Node->getValueType(ResNo: 0);
1926
1927 // Try to group the scalars into pairs, shuffle the pairs together, then
1928 // shuffle the pairs of pairs together, etc. until the vector has
1929 // been built. This will work only if all of the necessary shuffle masks
1930 // are legal.
1931
1932 // We do this in two phases; first to check the legality of the shuffles,
1933 // and next, assuming that all shuffles are legal, to create the new nodes.
1934 for (int Phase = 0; Phase < 2; ++Phase) {
1935 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1936 NewIntermedVals;
1937 for (unsigned i = 0; i < NumElems; ++i) {
1938 SDValue V = Node->getOperand(Num: i);
1939 if (V.isUndef())
1940 continue;
1941
1942 SDValue Vec;
1943 if (Phase)
1944 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1945 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1946 }
1947
1948 while (IntermedVals.size() > 2) {
1949 NewIntermedVals.clear();
1950 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1951 // This vector and the next vector are shuffled together (simply to
1952 // append the one to the other).
1953 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1954
1955 SmallVector<int, 16> FinalIndices;
1956 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1957 IntermedVals[i+1].second.size());
1958
1959 int k = 0;
1960 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1961 ++j, ++k) {
1962 ShuffleVec[k] = j;
1963 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1964 }
1965 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1966 ++j, ++k) {
1967 ShuffleVec[k] = NumElems + j;
1968 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1969 }
1970
1971 SDValue Shuffle;
1972 if (Phase)
1973 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1974 N2: IntermedVals[i+1].first,
1975 Mask: ShuffleVec);
1976 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1977 return false;
1978 NewIntermedVals.push_back(
1979 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1980 }
1981
1982 // If we had an odd number of defined values, then append the last
1983 // element to the array of new vectors.
1984 if ((IntermedVals.size() & 1) != 0)
1985 NewIntermedVals.push_back(Elt: IntermedVals.back());
1986
1987 IntermedVals.swap(RHS&: NewIntermedVals);
1988 }
1989
1990 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1991 "Invalid number of intermediate vectors");
1992 SDValue Vec1 = IntermedVals[0].first;
1993 SDValue Vec2;
1994 if (IntermedVals.size() > 1)
1995 Vec2 = IntermedVals[1].first;
1996 else if (Phase)
1997 Vec2 = DAG.getPOISON(VT);
1998
1999 SmallVector<int, 16> ShuffleVec(NumElems, -1);
2000 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
2001 ShuffleVec[IntermedVals[0].second[i]] = i;
2002 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
2003 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
2004
2005 if (Phase)
2006 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2007 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
2008 return false;
2009 }
2010
2011 return true;
2012}
2013
2014/// Expand a BUILD_VECTOR node on targets that don't
2015/// support the operation, but do support the resultant vector type.
2016SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2017 unsigned NumElems = Node->getNumOperands();
2018 SDValue Value1, Value2;
2019 SDLoc dl(Node);
2020 EVT VT = Node->getValueType(ResNo: 0);
2021 EVT OpVT = Node->getOperand(Num: 0).getValueType();
2022 EVT EltVT = VT.getVectorElementType();
2023
2024 // If the only non-undef value is the low element, turn this into a
2025 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
2026 bool isOnlyLowElement = true;
2027 bool MoreThanTwoValues = false;
2028 bool isConstant = true;
2029 for (unsigned i = 0; i < NumElems; ++i) {
2030 SDValue V = Node->getOperand(Num: i);
2031 if (V.isUndef())
2032 continue;
2033 if (i > 0)
2034 isOnlyLowElement = false;
2035 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2036 isConstant = false;
2037
2038 if (!Value1.getNode()) {
2039 Value1 = V;
2040 } else if (!Value2.getNode()) {
2041 if (V != Value1)
2042 Value2 = V;
2043 } else if (V != Value1 && V != Value2) {
2044 MoreThanTwoValues = true;
2045 }
2046 }
2047
2048 if (!Value1.getNode())
2049 return DAG.getUNDEF(VT);
2050
2051 if (isOnlyLowElement)
2052 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2053
2054 // If all elements are constants, create a load from the constant pool.
2055 if (isConstant) {
2056 SmallVector<Constant*, 16> CV;
2057 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2058 if (ConstantFPSDNode *V =
2059 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2060 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2061 } else if (ConstantSDNode *V =
2062 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2063 if (OpVT==EltVT)
2064 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2065 else {
2066 // If OpVT and EltVT don't match, EltVT is not legal and the
2067 // element values have been promoted/truncated earlier. Undo this;
2068 // we don't want a v16i8 to become a v16i32 for example.
2069 const ConstantInt *CI = V->getConstantIntValue();
2070 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2071 V: CI->getZExtValue(), /*IsSigned=*/false,
2072 /*ImplicitTrunc=*/true));
2073 }
2074 } else {
2075 assert(Node->getOperand(i).isUndef());
2076 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2077 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2078 }
2079 }
2080 Constant *CP = ConstantVector::get(V: CV);
2081 SDValue CPIdx =
2082 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2083 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2084 return DAG.getLoad(
2085 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2086 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2087 Alignment);
2088 }
2089
2090 SmallSet<SDValue, 16> DefinedValues;
2091 for (unsigned i = 0; i < NumElems; ++i) {
2092 if (Node->getOperand(Num: i).isUndef())
2093 continue;
2094 DefinedValues.insert(V: Node->getOperand(Num: i));
2095 }
2096
2097 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2098 if (!MoreThanTwoValues) {
2099 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2100 for (unsigned i = 0; i < NumElems; ++i) {
2101 SDValue V = Node->getOperand(Num: i);
2102 if (V.isUndef())
2103 continue;
2104 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2105 }
2106 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2107 // Get the splatted value into the low element of a vector register.
2108 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2109 SDValue Vec2;
2110 if (Value2.getNode())
2111 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2112 else
2113 Vec2 = DAG.getPOISON(VT);
2114
2115 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2116 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2117 }
2118 } else {
2119 SDValue Res;
2120 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2121 return Res;
2122 }
2123 }
2124
2125 // Otherwise, we can't handle this case efficiently.
2126 return ExpandVectorBuildThroughStack(Node);
2127}
2128
2129SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2130 SDLoc DL(Node);
2131 EVT VT = Node->getValueType(ResNo: 0);
2132 SDValue SplatVal = Node->getOperand(Num: 0);
2133
2134 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2135}
2136
2137// Expand a node into a call to a libcall, returning the value as the first
2138// result and the chain as the second. If the result value does not fit into a
2139// register, return the lo part and set the hi part to the by-reg argument in
2140// the first. If it does fit into a single register, return the result and
2141// leave the Hi part unset.
2142std::pair<SDValue, SDValue>
2143SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2144 TargetLowering::ArgListTy &&Args,
2145 bool IsSigned, EVT RetVT) {
2146 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2147 SDValue Callee;
2148 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2149 if (LCImpl != RTLIB::Unsupported)
2150 Callee = DAG.getExternalSymbol(LCImpl, VT: CodePtrTy);
2151 else {
2152 Callee = DAG.getPOISON(VT: CodePtrTy);
2153 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2154 Node->getOperationName(G: &DAG));
2155 }
2156
2157 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2158
2159 // By default, the input chain to this libcall is the entry node of the
2160 // function. If the libcall is going to be emitted as a tail call then
2161 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2162 // node which is being folded has a non-entry input chain.
2163 SDValue InChain = DAG.getEntryNode();
2164
2165 // isTailCall may be true since the callee does not reference caller stack
2166 // frame. Check if it's in the right position and that the return types match.
2167 SDValue TCChain = InChain;
2168 const Function &F = DAG.getMachineFunction().getFunction();
2169 bool isTailCall =
2170 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2171 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()) &&
2172 // Lowering doesn't support tail calling inside a function with
2173 // a swifterror argument yet.
2174 !DAG.hasSwiftErrorArg();
2175 if (isTailCall)
2176 InChain = TCChain;
2177
2178 TargetLowering::CallLoweringInfo CLI(DAG);
2179 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2180 CLI.setDebugLoc(SDLoc(Node))
2181 .setChain(InChain)
2182 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
2183 Target: Callee, ArgsList: std::move(Args))
2184 .setTailCall(isTailCall)
2185 .setSExtResult(signExtend)
2186 .setZExtResult(!signExtend)
2187 .setIsPostTypeLegalization(true);
2188
2189 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2190
2191 if (!CallInfo.second.getNode()) {
2192 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2193 // It's a tailcall, return the chain (which is the DAG root).
2194 return {DAG.getRoot(), DAG.getRoot()};
2195 }
2196
2197 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2198 return CallInfo;
2199}
2200
2201std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2202 bool isSigned) {
2203 TargetLowering::ArgListTy Args;
2204 for (const SDValue &Op : Node->op_values()) {
2205 EVT ArgVT = Op.getValueType();
2206 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2207 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2208 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2209 Entry.IsZExt = !Entry.IsSExt;
2210 Args.push_back(x: Entry);
2211 }
2212
2213 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2214 RetVT: Node->getValueType(ResNo: 0));
2215}
2216
2217void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2218 RTLIB::Libcall LC,
2219 SmallVectorImpl<SDValue> &Results) {
2220 if (LC == RTLIB::UNKNOWN_LIBCALL)
2221 llvm_unreachable("Can't create an unknown libcall!");
2222
2223 if (Node->isStrictFPOpcode()) {
2224 EVT RetVT = Node->getValueType(ResNo: 0);
2225 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2226 if (LCImpl == RTLIB::Unsupported) {
2227 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2228 Node->getOperationName(G: &DAG));
2229 Results.push_back(Elt: DAG.getPOISON(VT: RetVT));
2230 Results.push_back(Elt: Node->getOperand(Num: 0));
2231 return;
2232 }
2233 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2234 TargetLowering::MakeLibCallOptions CallOptions;
2235 CallOptions.IsPostTypeLegalization = true;
2236 // FIXME: This doesn't support tail calls.
2237 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2238 DAG, LibcallImpl: LCImpl, RetVT, Ops, CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
2239 Results.push_back(Elt: Tmp.first);
2240 Results.push_back(Elt: Tmp.second);
2241 } else {
2242 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2243 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2244 Results.push_back(Elt: Tmp);
2245 }
2246}
2247
2248/// Expand the node to a libcall based on the result type.
2249void SelectionDAGLegalize::ExpandFastFPLibCall(
2250 SDNode *Node, bool IsFast,
2251 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2252 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2253 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2254 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2255 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2256 SmallVectorImpl<SDValue> &Results) {
2257
2258 EVT VT = Node->getSimpleValueType(ResNo: 0);
2259
2260 RTLIB::Libcall LC;
2261
2262 // FIXME: Probably should define fast to respect nan/inf and only be
2263 // approximate functions.
2264
2265 if (IsFast) {
2266 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.first, Call_F64: Call_F64.first, Call_F80: Call_F80.first,
2267 Call_F128: Call_F128.first, Call_PPCF128: Call_PPCF128.first);
2268 }
2269
2270 if (!IsFast || DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
2271 // Fall back if we don't have a fast implementation.
2272 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.second, Call_F64: Call_F64.second,
2273 Call_F80: Call_F80.second, Call_F128: Call_F128.second,
2274 Call_PPCF128: Call_PPCF128.second);
2275 }
2276
2277 ExpandFPLibCall(Node, LC, Results);
2278}
2279
2280SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2281 RTLIB::Libcall Call_I8,
2282 RTLIB::Libcall Call_I16,
2283 RTLIB::Libcall Call_I32,
2284 RTLIB::Libcall Call_I64,
2285 RTLIB::Libcall Call_I128) {
2286 RTLIB::Libcall LC;
2287 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2288 default: llvm_unreachable("Unexpected request for libcall!");
2289 case MVT::i8: LC = Call_I8; break;
2290 case MVT::i16: LC = Call_I16; break;
2291 case MVT::i32: LC = Call_I32; break;
2292 case MVT::i64: LC = Call_I64; break;
2293 case MVT::i128: LC = Call_I128; break;
2294 }
2295 return ExpandLibCall(LC, Node, isSigned).first;
2296}
2297
2298/// Expand the node to a libcall based on first argument type (for instance
2299/// lround and its variant).
2300void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2301 RTLIB::Libcall Call_F32,
2302 RTLIB::Libcall Call_F64,
2303 RTLIB::Libcall Call_F80,
2304 RTLIB::Libcall Call_F128,
2305 RTLIB::Libcall Call_PPCF128,
2306 SmallVectorImpl<SDValue> &Results) {
2307 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2308 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2309 Call_F32, Call_F64, Call_F80,
2310 Call_F128, Call_PPCF128);
2311 ExpandFPLibCall(Node, LC, Results);
2312}
2313
2314SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2315 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2316 RTLIB::Libcall CallI128) {
2317 RTLIB::Libcall LC;
2318 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2319 default:
2320 llvm_unreachable("Unexpected request for libcall!");
2321 case MVT::i32:
2322 LC = CallI32;
2323 break;
2324 case MVT::i64:
2325 LC = CallI64;
2326 break;
2327 case MVT::i128:
2328 LC = CallI128;
2329 break;
2330 }
2331
2332 // Bit-counting libcalls have one unsigned argument and return `int`.
2333 // Note that `int` may be illegal on this target; ExpandLibCall will
2334 // take care of promoting it to a legal type.
2335 SDValue Op = Node->getOperand(Num: 0);
2336 EVT IntVT =
2337 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2338
2339 EVT ArgVT = Op.getValueType();
2340 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2341 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2342 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2343 Arg.IsZExt = !Arg.IsSExt;
2344
2345 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2346 /*IsSigned=*/true, RetVT: IntVT)
2347 .first;
2348
2349 // If ExpandLibCall created a tail call, the result was already
2350 // of the correct type. Otherwise, we need to sign extend it.
2351 if (Res.getValueType() != MVT::Other)
2352 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2353 return Res;
2354}
2355
2356/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2357void
2358SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2359 SmallVectorImpl<SDValue> &Results) {
2360 unsigned Opcode = Node->getOpcode();
2361 bool isSigned = Opcode == ISD::SDIVREM;
2362
2363 RTLIB::Libcall LC;
2364 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2365 default: llvm_unreachable("Unexpected request for libcall!");
2366 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2367 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2368 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2369 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2370 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2371 }
2372
2373 // The input chain to this libcall is the entry node of the function.
2374 // Legalizing the call will automatically add the previous call to the
2375 // dependence.
2376 SDValue InChain = DAG.getEntryNode();
2377
2378 EVT RetVT = Node->getValueType(ResNo: 0);
2379 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2380
2381 TargetLowering::ArgListTy Args;
2382 for (const SDValue &Op : Node->op_values()) {
2383 EVT ArgVT = Op.getValueType();
2384 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2385 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2386 Entry.IsSExt = isSigned;
2387 Entry.IsZExt = !isSigned;
2388 Args.push_back(x: Entry);
2389 }
2390
2391 // Also pass the return address of the remainder.
2392 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2393 TargetLowering::ArgListEntry Entry(
2394 FIPtr, PointerType::getUnqual(C&: RetTy->getContext()));
2395 Entry.IsSExt = isSigned;
2396 Entry.IsZExt = !isSigned;
2397 Args.push_back(x: Entry);
2398
2399 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2400 if (LibcallImpl == RTLIB::Unsupported) {
2401 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2402 Node->getOperationName(G: &DAG));
2403 SDValue Poison = DAG.getPOISON(VT: RetVT);
2404 Results.push_back(Elt: Poison);
2405 Results.push_back(Elt: Poison);
2406 return;
2407 }
2408
2409 SDValue Callee =
2410 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2411
2412 SDLoc dl(Node);
2413 TargetLowering::CallLoweringInfo CLI(DAG);
2414 CLI.setDebugLoc(dl)
2415 .setChain(InChain)
2416 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
2417 ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2418 .setSExtResult(isSigned)
2419 .setZExtResult(!isSigned);
2420
2421 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2422
2423 // Remainder is loaded back from the stack frame.
2424 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
2425 MachinePointerInfo PtrInfo =
2426 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2427
2428 SDValue Rem = DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo);
2429 Results.push_back(Elt: CallInfo.first);
2430 Results.push_back(Elt: Rem);
2431}
2432
2433/// Return true if sincos or __sincos_stret libcall is available.
2434static bool isSinCosLibcallAvailable(SDNode *Node,
2435 const LibcallLoweringInfo &Libcalls) {
2436 MVT::SimpleValueType VT = Node->getSimpleValueType(ResNo: 0).SimpleTy;
2437 return Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS(VT)) != RTLIB::Unsupported ||
2438 Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS_STRET(VT)) !=
2439 RTLIB::Unsupported;
2440}
2441
2442/// Only issue sincos libcall if both sin and cos are needed.
2443static bool useSinCos(SDNode *Node) {
2444 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2445 ? ISD::FCOS : ISD::FSIN;
2446
2447 SDValue Op0 = Node->getOperand(Num: 0);
2448 for (const SDNode *User : Op0.getNode()->users()) {
2449 if (User == Node)
2450 continue;
2451 // The other user might have been turned into sincos already.
2452 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2453 return true;
2454 }
2455 return false;
2456}
2457
2458SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2459 // For iOS, we want to call an alternative entry point: __sincos_stret,
2460 // which returns the values in two S / D registers.
2461 SDLoc dl(Node);
2462 SDValue Arg = Node->getOperand(Num: 0);
2463 EVT ArgVT = Arg.getValueType();
2464 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(VT: ArgVT);
2465 RTLIB::LibcallImpl SincosStret = DAG.getLibcalls().getLibcallImpl(Call: LC);
2466 if (SincosStret == RTLIB::Unsupported)
2467 return SDValue();
2468
2469 /// There are 3 different ABI cases to handle:
2470 /// - Direct return of separate fields in registers
2471 /// - Single return as vector elements
2472 /// - sret struct
2473
2474 const RTLIB::RuntimeLibcallsInfo &CallsInfo = TLI.getRuntimeLibcallsInfo();
2475
2476 const DataLayout &DL = DAG.getDataLayout();
2477
2478 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2479 Ctx&: *DAG.getContext(), TT: TM.getTargetTriple(), DL, LibcallImpl: SincosStret);
2480
2481 Type *SincosStretRetTy = FuncTy->getReturnType();
2482 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(Call: SincosStret);
2483
2484 SDValue Callee =
2485 DAG.getExternalSymbol(LCImpl: SincosStret, VT: TLI.getProgramPointerTy(DL));
2486
2487 TargetLowering::ArgListTy Args;
2488 SDValue SRet;
2489
2490 int FrameIdx;
2491 if (FuncTy->getParamType(i: 0)->isPointerTy()) {
2492 // Uses sret
2493 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2494
2495 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(ArgNo: 0);
2496 Type *StructTy = PtrAttrs.getStructRetType();
2497 const uint64_t ByteSize = DL.getTypeAllocSize(Ty: StructTy);
2498 const Align StackAlign = DL.getPrefTypeAlign(Ty: StructTy);
2499
2500 FrameIdx = MFI.CreateStackObject(Size: ByteSize, Alignment: StackAlign, isSpillSlot: false);
2501 SRet = DAG.getFrameIndex(FI: FrameIdx, VT: TLI.getFrameIndexTy(DL));
2502
2503 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(i: 0));
2504 Entry.IsSRet = true;
2505 Entry.IndirectType = StructTy;
2506 Entry.Alignment = StackAlign;
2507
2508 Args.push_back(x: Entry);
2509 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 1));
2510 } else {
2511 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 0));
2512 }
2513
2514 TargetLowering::CallLoweringInfo CLI(DAG);
2515 CLI.setDebugLoc(dl)
2516 .setChain(DAG.getEntryNode())
2517 .setLibCallee(CC: CallConv, ResultType: SincosStretRetTy, Target: Callee, ArgsList: std::move(Args))
2518 .setIsPostTypeLegalization();
2519
2520 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2521
2522 if (SRet) {
2523 MachinePointerInfo PtrInfo =
2524 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2525 SDValue LoadSin = DAG.getLoad(VT: ArgVT, dl, Chain: CallResult.second, Ptr: SRet, PtrInfo);
2526
2527 TypeSize StoreSize = ArgVT.getStoreSize();
2528
2529 // Address of cos field.
2530 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr: SRet, Offset: StoreSize);
2531 SDValue LoadCos = DAG.getLoad(VT: ArgVT, dl, Chain: LoadSin.getValue(R: 1), Ptr: Add,
2532 PtrInfo: PtrInfo.getWithOffset(O: StoreSize));
2533
2534 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2535 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: LoadSin.getValue(R: 0),
2536 N2: LoadCos.getValue(R: 0));
2537 }
2538
2539 if (!CallResult.first.getValueType().isVector())
2540 return CallResult.first;
2541
2542 SDValue SinVal =
2543 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2544 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2545 SDValue CosVal =
2546 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2547 N2: DAG.getVectorIdxConstant(Val: 1, DL: dl));
2548 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2549 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: SinVal, N2: CosVal);
2550}
2551
2552SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2553 SDLoc dl(Node);
2554 EVT VT = Node->getValueType(ResNo: 0);
2555 SDValue X = Node->getOperand(Num: 0);
2556 SDValue N = Node->getOperand(Num: 1);
2557 EVT ExpVT = N.getValueType();
2558 EVT AsIntVT = VT.changeTypeToInteger();
2559 if (AsIntVT == EVT()) // TODO: How to handle f80?
2560 return SDValue();
2561
2562 // The expansion works through the integer-equivalent type; if that is not
2563 // legal, bail out and let the caller use a libcall (or diagnose a missing
2564 // one).
2565 if (!TLI.isTypeLegal(VT: AsIntVT))
2566 return SDValue();
2567
2568 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2569 return SDValue();
2570
2571 SDNodeFlags NSW;
2572 NSW.setNoSignedWrap(true);
2573 SDNodeFlags NUW_NSW;
2574 NUW_NSW.setNoUnsignedWrap(true);
2575 NUW_NSW.setNoSignedWrap(true);
2576
2577 EVT SetCCVT =
2578 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2579 const fltSemantics &FltSem = VT.getFltSemantics();
2580
2581 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2582 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2583 const int Precision = APFloat::semanticsPrecision(FltSem);
2584
2585 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2586 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2587
2588 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2589
2590 const APFloat One(FltSem, "1.0");
2591 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2592
2593 // Offset by precision to avoid denormal range.
2594 APFloat ScaleDownK =
2595 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2596
2597 // TODO: Should really introduce control flow and use a block for the >
2598 // MaxExp, < MinExp cases
2599
2600 // First, handle exponents Exp > MaxExp and scale down.
2601 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2602
2603 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2604 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2605 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2606 SDValue DecN1 =
2607 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2608
2609 SDValue ScaleUpTwice =
2610 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2611
2612 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2613 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2614 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2615
2616 SDValue SelectN_Big =
2617 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2618 SDValue SelectX_Big =
2619 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2620
2621 // Now handle exponents Exp < MinExp
2622 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2623
2624 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2625 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2626
2627 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2628
2629 SDValue ClampMinVal =
2630 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2631 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2632 SDValue IncN1 =
2633 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2634
2635 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2636 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2637 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2638
2639 SDValue ScaleDownTwice = DAG.getSetCC(
2640 DL: dl, VT: SetCCVT, LHS: N,
2641 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2642
2643 SDValue SelectN_Small =
2644 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2645 SDValue SelectX_Small =
2646 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2647
2648 // Now combine the two out of range exponent handling cases with the base
2649 // case.
2650 SDValue NewX = DAG.getNode(
2651 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2652 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2653
2654 SDValue NewN = DAG.getNode(
2655 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2656 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2657
2658 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2659
2660 SDValue ExponentShiftAmt =
2661 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2662 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2663
2664 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2665 N2: ExponentShiftAmt, Flags: NUW_NSW);
2666 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2667 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2668}
2669
2670SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2671 SDLoc dl(Node);
2672 SDValue Val = Node->getOperand(Num: 0);
2673 EVT VT = Val.getValueType();
2674 EVT ExpVT = Node->getValueType(ResNo: 1);
2675 EVT AsIntVT = VT.changeTypeToInteger();
2676 if (AsIntVT == EVT()) // TODO: How to handle f80?
2677 return SDValue();
2678
2679 // The expansion works through the integer-equivalent type; if that is not
2680 // legal, bail out and let the caller use a libcall (or diagnose a missing
2681 // one).
2682 if (!TLI.isTypeLegal(VT: AsIntVT))
2683 return SDValue();
2684
2685 const fltSemantics &FltSem = VT.getFltSemantics();
2686 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2687 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2688 const unsigned BitSize = VT.getScalarSizeInBits();
2689
2690 // TODO: Could introduce control flow and skip over the denormal handling.
2691
2692 // scale_up = fmul value, scalbn(1.0, precision + 1)
2693 // extracted_exp = (bitcast value to uint) >> precision - 1
2694 // biased_exp = extracted_exp + min_exp
2695 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2696 //
2697 // is_denormal = val < smallest_normalized
2698 // computed_fract = is_denormal ? scale_up : extracted_fract
2699 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2700 //
2701 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2702 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2703
2704 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2705 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2706 VT: AsIntVT);
2707
2708 SDValue SmallestNormalizedInt = DAG.getConstant(
2709 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2710 VT: AsIntVT);
2711
2712 // Masks out the exponent bits.
2713 SDValue ExpMask =
2714 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2715
2716 // Mask out the exponent part of the value.
2717 //
2718 // e.g, for f32 FractSignMaskVal = 0x807fffff
2719 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2720 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2721
2722 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2723 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2724
2725 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2726
2727 const APFloat One(FltSem, "1.0");
2728 // Scale a possible denormal input.
2729 // e.g., for f64, 0x1p+54
2730 APFloat ScaleUpKVal =
2731 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2732
2733 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2734 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2735
2736 EVT SetCCVT =
2737 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2738
2739 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2740
2741 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2742
2743 SDValue AddNegSmallestNormal =
2744 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2745 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2746 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2747
2748 SDValue IsDenormal =
2749 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2750
2751 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2752 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2753
2754 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2755 SDValue ScaledSelect =
2756 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2757
2758 SDValue ExpMaskScaled =
2759 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2760
2761 SDValue ScaledValue =
2762 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2763
2764 // Extract the exponent bits.
2765 SDValue ExponentShiftAmt =
2766 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2767 SDValue ShiftedExp =
2768 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2769 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2770
2771 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2772 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2773 SDValue DenormalExpBias =
2774 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2775
2776 SDValue MaskedFractAsInt =
2777 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2778 const APFloat Half(FltSem, "0.5");
2779 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2780 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2781 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2782
2783 SDValue ComputedExp =
2784 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2785
2786 SDValue Result0 =
2787 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2788
2789 SDValue Result1 =
2790 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2791
2792 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2793}
2794
2795SDValue SelectionDAGLegalize::expandModf(SDNode *Node) const {
2796 SDLoc dl(Node);
2797 SDValue Val = Node->getOperand(Num: 0);
2798 EVT VT = Val.getValueType();
2799 SDNodeFlags Flags = Node->getFlags();
2800
2801 SDValue IntPart = DAG.getNode(Opcode: ISD::FTRUNC, DL: dl, VT, Operand: Val, Flags);
2802 SDValue FracPart = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT, N1: Val, N2: IntPart, Flags);
2803
2804 SDValue FracToUse;
2805 if (Flags.hasNoInfs()) {
2806 FracToUse = FracPart;
2807 } else {
2808 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val, Flags);
2809 SDValue Inf =
2810 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
2811 EVT SetCCVT =
2812 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2813 SDValue IsInf = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: Inf, Cond: ISD::SETOEQ);
2814 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: dl, VT);
2815 FracToUse = DAG.getSelect(DL: dl, VT, Cond: IsInf, LHS: Zero, RHS: FracPart);
2816 }
2817
2818 SDValue ResultFrac =
2819 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT, N1: FracToUse, N2: Val, Flags);
2820 return DAG.getMergeValues(Ops: {ResultFrac, IntPart}, dl);
2821}
2822
2823/// This function is responsible for legalizing a
2824/// INT_TO_FP operation of the specified operand when the target requests that
2825/// we expand it. At this point, we know that the result and operand types are
2826/// legal for the target.
2827SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2828 SDValue &Chain) {
2829 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2830 Node->getOpcode() == ISD::SINT_TO_FP);
2831 EVT DestVT = Node->getValueType(ResNo: 0);
2832 SDLoc dl(Node);
2833 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2834 SDValue Op0 = Node->getOperand(Num: OpNo);
2835 EVT SrcVT = Op0.getValueType();
2836
2837 // TODO: Should any fast-math-flags be set for the created nodes?
2838 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2839 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2840 (DestVT.bitsLE(VT: MVT::f64) ||
2841 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2842 : ISD::FP_EXTEND,
2843 VT: DestVT))) {
2844 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2845 "expansion\n");
2846
2847 // Get the stack frame index of a 8 byte buffer.
2848 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2849
2850 SDValue Lo = Op0;
2851 // if signed map to unsigned space
2852 if (isSigned) {
2853 // Invert sign bit (signed to unsigned mapping).
2854 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2855 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2856 }
2857 // Initial hi portion of constructed double.
2858 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2859
2860 // If this a big endian target, swap the lo and high data.
2861 if (DAG.getDataLayout().isBigEndian())
2862 std::swap(a&: Lo, b&: Hi);
2863
2864 SDValue MemChain = DAG.getEntryNode();
2865
2866 // Store the lo of the constructed double.
2867 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2868 PtrInfo: MachinePointerInfo());
2869 // Store the hi of the constructed double.
2870 SDValue HiPtr =
2871 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2872 SDValue Store2 =
2873 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2874 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2875
2876 // load the constructed double
2877 SDValue Load =
2878 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2879 // FP constant to bias correct the final result
2880 SDValue Bias = DAG.getConstantFP(
2881 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2882 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2883 DL: dl, VT: MVT::f64);
2884 // Subtract the bias and get the final result.
2885 SDValue Sub;
2886 SDValue Result;
2887 if (Node->isStrictFPOpcode()) {
2888 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2889 Ops: {Node->getOperand(Num: 0), Load, Bias});
2890 Chain = Sub.getValue(R: 1);
2891 if (DestVT != Sub.getValueType()) {
2892 std::pair<SDValue, SDValue> ResultPair;
2893 ResultPair =
2894 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2895 Result = ResultPair.first;
2896 Chain = ResultPair.second;
2897 }
2898 else
2899 Result = Sub;
2900 } else {
2901 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2902 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2903 }
2904 return Result;
2905 }
2906
2907 if (isSigned)
2908 return SDValue();
2909
2910 // TODO: Generalize this for use with other types.
2911 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2912 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2913 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2914 // For unsigned conversions, convert them to signed conversions using the
2915 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2916 // should be valid for i32->f32 as well.
2917
2918 // More generally this transform should be valid if there are 3 more bits
2919 // in the integer type than the significand. Rounding uses the first bit
2920 // after the width of the significand and the OR of all bits after that. So
2921 // we need to be able to OR the shifted out bit into one of the bits that
2922 // participate in the OR.
2923
2924 // TODO: This really should be implemented using a branch rather than a
2925 // select. We happen to get lucky and machinesink does the right
2926 // thing most of the time. This would be a good candidate for a
2927 // pseudo-op, or, even better, for whole-function isel.
2928 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2929
2930 SDValue SignBitTest = DAG.getSetCC(
2931 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2932
2933 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL: dl);
2934 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2935 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2936 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2937 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2938
2939 SDValue Slow, Fast;
2940 if (Node->isStrictFPOpcode()) {
2941 // In strict mode, we must avoid spurious exceptions, and therefore
2942 // must make sure to only emit a single STRICT_SINT_TO_FP.
2943 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2944 // The STRICT_SINT_TO_FP inherits the exception mode from the
2945 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2946 // never raise any exception.
2947 SDNodeFlags Flags;
2948 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2949 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: {DestVT, MVT::Other},
2950 Ops: {Node->getOperand(Num: 0), InCvt}, Flags);
2951 Flags.setNoFPExcept(true);
2952 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: {DestVT, MVT::Other},
2953 Ops: {Fast.getValue(R: 1), Fast, Fast}, Flags);
2954 Chain = Slow.getValue(R: 1);
2955 } else {
2956 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2957 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2958 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2959 }
2960
2961 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2962 }
2963
2964 // Don't expand it if there isn't cheap fadd.
2965 if (!TLI.isOperationLegalOrCustom(
2966 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2967 return SDValue();
2968
2969 // The following optimization is valid only if every value in SrcVT (when
2970 // treated as signed) is representable in DestVT. Check that the mantissa
2971 // size of DestVT is >= than the number of bits in SrcVT -1.
2972 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2973 SrcVT.getSizeInBits() - 1 &&
2974 "Cannot perform lossless SINT_TO_FP!");
2975
2976 SDValue Tmp1;
2977 if (Node->isStrictFPOpcode()) {
2978 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2979 Ops: { Node->getOperand(Num: 0), Op0 });
2980 } else
2981 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2982
2983 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2984 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2985 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2986 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2987 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2988 Cond: SignSet, LHS: Four, RHS: Zero);
2989
2990 // If the sign bit of the integer is set, the large number will be treated
2991 // as a negative number. To counteract this, the dynamic code adds an
2992 // offset depending on the data type.
2993 uint64_t FF;
2994 switch (SrcVT.getSimpleVT().SimpleTy) {
2995 default:
2996 return SDValue();
2997 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2998 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2999 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
3000 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
3001 }
3002 if (DAG.getDataLayout().isLittleEndian())
3003 FF <<= 32;
3004 Constant *FudgeFactor = ConstantInt::get(
3005 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
3006
3007 SDValue CPIdx =
3008 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3009 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
3010 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
3011 Alignment = commonAlignment(A: Alignment, Offset: 4);
3012 SDValue FudgeInReg;
3013 if (DestVT == MVT::f32)
3014 FudgeInReg = DAG.getLoad(
3015 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
3016 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
3017 Alignment);
3018 else {
3019 SDValue Load = DAG.getExtLoad(
3020 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
3021 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
3022 Alignment);
3023 HandleSDNode Handle(Load);
3024 LegalizeOp(Node: Load.getNode());
3025 FudgeInReg = Handle.getValue();
3026 }
3027
3028 if (Node->isStrictFPOpcode()) {
3029 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
3030 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
3031 Chain = Result.getValue(R: 1);
3032 return Result;
3033 }
3034
3035 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
3036}
3037
3038/// This function is responsible for legalizing a
3039/// *INT_TO_FP operation of the specified operand when the target requests that
3040/// we promote it. At this point, we know that the result and operand types are
3041/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
3042/// operation that takes a larger input.
3043void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
3044 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
3045 bool IsStrict = N->isStrictFPOpcode();
3046 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
3047 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
3048 EVT DestVT = N->getValueType(ResNo: 0);
3049 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3050 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
3051 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
3052
3053 // First step, figure out the appropriate *INT_TO_FP operation to use.
3054 EVT NewInTy = LegalOp.getValueType();
3055
3056 unsigned OpToUse = 0;
3057
3058 // Scan for the appropriate larger type to use.
3059 while (true) {
3060 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
3061 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3062
3063 // If the target supports SINT_TO_FP of this type, use it.
3064 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
3065 OpToUse = SIntOp;
3066 break;
3067 }
3068 if (IsSigned)
3069 continue;
3070
3071 // If the target supports UINT_TO_FP of this type, use it.
3072 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
3073 OpToUse = UIntOp;
3074 break;
3075 }
3076
3077 // Otherwise, try a larger type.
3078 }
3079
3080 // Okay, we found the operation and type to use. Zero extend our input to the
3081 // desired type then run the operation on it.
3082 if (IsStrict) {
3083 SDValue Res =
3084 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
3085 Ops: {N->getOperand(Num: 0),
3086 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3087 DL: dl, VT: NewInTy, Operand: LegalOp)});
3088 Results.push_back(Elt: Res);
3089 Results.push_back(Elt: Res.getValue(R: 1));
3090 return;
3091 }
3092
3093 Results.push_back(
3094 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
3095 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3096 DL: dl, VT: NewInTy, Operand: LegalOp)));
3097}
3098
3099/// This function is responsible for legalizing a
3100/// FP_TO_*INT operation of the specified operand when the target requests that
3101/// we promote it. At this point, we know that the result and operand types are
3102/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
3103/// operation that returns a larger result.
3104void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3105 SmallVectorImpl<SDValue> &Results) {
3106 bool IsStrict = N->isStrictFPOpcode();
3107 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3108 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3109 EVT DestVT = N->getValueType(ResNo: 0);
3110 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3111 // First step, figure out the appropriate FP_TO*INT operation to use.
3112 EVT NewOutTy = DestVT;
3113
3114 unsigned OpToUse = 0;
3115
3116 // Scan for the appropriate larger type to use.
3117 while (true) {
3118 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
3119 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3120
3121 // A larger signed type can hold all unsigned values of the requested type,
3122 // so using FP_TO_SINT is valid
3123 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
3124 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3125 break;
3126
3127 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
3128 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
3129 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3130 break;
3131
3132 // Otherwise, try a larger type.
3133 }
3134
3135 // Okay, we found the operation and type to use.
3136 SDValue Operation;
3137 if (IsStrict) {
3138 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
3139 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
3140 } else
3141 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
3142
3143 // Truncate the result of the extended FP_TO_*INT operation to the desired
3144 // size.
3145 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
3146 Results.push_back(Elt: Trunc);
3147 if (IsStrict)
3148 Results.push_back(Elt: Operation.getValue(R: 1));
3149}
3150
3151/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
3152/// the result and operand types are legal and there must be a legal
3153/// FP_TO_*INT_SAT operation for a larger result type.
3154SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3155 const SDLoc &dl) {
3156 unsigned Opcode = Node->getOpcode();
3157
3158 // Scan for the appropriate larger type to use.
3159 EVT NewOutTy = Node->getValueType(ResNo: 0);
3160 while (true) {
3161 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
3162 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3163
3164 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
3165 break;
3166 }
3167
3168 // Saturation width is determined by second operand, so we don't have to
3169 // perform any fixup and can directly truncate the result.
3170 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
3171 N2: Node->getOperand(Num: 1));
3172 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
3173}
3174
3175/// Open code the operations for PARITY of the specified operation.
3176SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3177 EVT VT = Op.getValueType();
3178 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3179 unsigned Sz = VT.getScalarSizeInBits();
3180
3181 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3182 SDValue Result;
3183 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3184 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3185 } else {
3186 Result = Op;
3187 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3188 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3189 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3190 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3191 }
3192 }
3193
3194 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3195}
3196
3197SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3198 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3199 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3200 : Node->getOperand(Num: 0).getSimpleValueType();
3201 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3202 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3203 MVT NewScalarVT = NewVecVT.getVectorElementType();
3204
3205 SDLoc DL(Node);
3206 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3207
3208 // FIXME: Support integer.
3209 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3210 "Only FP promotion is supported");
3211
3212 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3213 if (Node->getOperand(Num: j).getValueType().isVector() &&
3214 !(IsVPOpcode &&
3215 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3216 // promote the vector operand.
3217 // FIXME: Support integer.
3218 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3219 "Only FP promotion is supported");
3220 Operands[j] =
3221 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3222 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3223 // promote the initial value.
3224 Operands[j] =
3225 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3226 } else {
3227 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3228 }
3229
3230 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3231 Flags: Node->getFlags());
3232
3233 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3234 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3235 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3236}
3237
3238bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3239 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3240 SmallVector<SDValue, 8> Results;
3241 SDLoc dl(Node);
3242 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3243 bool NeedInvert;
3244 switch (Node->getOpcode()) {
3245 case ISD::ABS:
3246 case ISD::ABS_MIN_POISON:
3247 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3248 Results.push_back(Elt: Tmp1);
3249 break;
3250 case ISD::ABDS:
3251 case ISD::ABDU:
3252 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3253 Results.push_back(Elt: Tmp1);
3254 break;
3255 case ISD::AVGCEILS:
3256 case ISD::AVGCEILU:
3257 case ISD::AVGFLOORS:
3258 case ISD::AVGFLOORU:
3259 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3260 Results.push_back(Elt: Tmp1);
3261 break;
3262 case ISD::CTPOP:
3263 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3264 Results.push_back(Elt: Tmp1);
3265 break;
3266 case ISD::CTLZ:
3267 case ISD::CTLZ_ZERO_POISON:
3268 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3269 Results.push_back(Elt: Tmp1);
3270 break;
3271 case ISD::CTLS:
3272 if ((Tmp1 = TLI.expandCTLS(N: Node, DAG)))
3273 Results.push_back(Elt: Tmp1);
3274 break;
3275 case ISD::CTTZ:
3276 case ISD::CTTZ_ZERO_POISON:
3277 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3278 Results.push_back(Elt: Tmp1);
3279 break;
3280 case ISD::BITREVERSE:
3281 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3282 Results.push_back(Elt: Tmp1);
3283 break;
3284 case ISD::BSWAP:
3285 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3286 Results.push_back(Elt: Tmp1);
3287 break;
3288 case ISD::PARITY:
3289 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3290 break;
3291 case ISD::FRAMEADDR:
3292 case ISD::RETURNADDR:
3293 case ISD::FRAME_TO_ARGS_OFFSET:
3294 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3295 break;
3296 case ISD::EH_DWARF_CFA: {
3297 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3298 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3299 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3300 VT: CfaArg.getValueType(),
3301 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3302 VT: CfaArg.getValueType()),
3303 N2: CfaArg);
3304 SDValue FA = DAG.getNode(
3305 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3306 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3307 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3308 N1: FA, N2: Offset));
3309 break;
3310 }
3311 case ISD::GET_ROUNDING:
3312 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3313 Results.push_back(Elt: Node->getOperand(Num: 0));
3314 break;
3315 case ISD::EH_RETURN:
3316 case ISD::EH_LABEL:
3317 case ISD::PREFETCH:
3318 case ISD::VAEND:
3319 case ISD::EH_SJLJ_LONGJMP:
3320 // If the target didn't expand these, there's nothing to do, so just
3321 // preserve the chain and be done.
3322 Results.push_back(Elt: Node->getOperand(Num: 0));
3323 break;
3324 case ISD::READCYCLECOUNTER:
3325 case ISD::READSTEADYCOUNTER:
3326 // If the target didn't expand this, just return 'zero' and preserve the
3327 // chain.
3328 Results.append(NumInputs: Node->getNumValues() - 1,
3329 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3330 Results.push_back(Elt: Node->getOperand(Num: 0));
3331 break;
3332 case ISD::EH_SJLJ_SETJMP:
3333 // If the target didn't expand this, just return 'zero' and preserve the
3334 // chain.
3335 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3336 Results.push_back(Elt: Node->getOperand(Num: 0));
3337 break;
3338 case ISD::ATOMIC_LOAD: {
3339 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3340 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3341 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3342 SDValue Swap = DAG.getAtomicCmpSwap(
3343 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3344 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3345 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3346 Results.push_back(Elt: Swap.getValue(R: 0));
3347 Results.push_back(Elt: Swap.getValue(R: 1));
3348 break;
3349 }
3350 case ISD::ATOMIC_STORE: {
3351 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3352 SDValue Swap = DAG.getAtomic(
3353 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3354 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3355 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3356 Results.push_back(Elt: Swap.getValue(R: 1));
3357 break;
3358 }
3359 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3360 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3361 // splits out the success value as a comparison. Expanding the resulting
3362 // ATOMIC_CMP_SWAP will produce a libcall.
3363 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3364 SDValue Res = DAG.getAtomicCmpSwap(
3365 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3366 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3367 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3368
3369 SDValue ExtRes = Res;
3370 SDValue LHS = Res;
3371 SDValue RHS = Node->getOperand(Num: 1);
3372
3373 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3374 EVT OuterType = Node->getValueType(ResNo: 0);
3375 switch (TLI.getExtendForAtomicOps()) {
3376 case ISD::SIGN_EXTEND:
3377 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3378 N2: DAG.getValueType(AtomicType));
3379 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3380 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3381 ExtRes = LHS;
3382 break;
3383 case ISD::ZERO_EXTEND:
3384 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3385 N2: DAG.getValueType(AtomicType));
3386 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3387 ExtRes = LHS;
3388 break;
3389 case ISD::ANY_EXTEND:
3390 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3391 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3392 break;
3393 default:
3394 llvm_unreachable("Invalid atomic op extension");
3395 }
3396
3397 SDValue Success =
3398 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3399
3400 Results.push_back(Elt: ExtRes.getValue(R: 0));
3401 Results.push_back(Elt: Success);
3402 Results.push_back(Elt: Res.getValue(R: 1));
3403 break;
3404 }
3405 case ISD::ATOMIC_LOAD_SUB: {
3406 SDLoc DL(Node);
3407 EVT VT = Node->getValueType(ResNo: 0);
3408 SDValue RHS = Node->getOperand(Num: 2);
3409 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3410 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3411 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3412 RHS = RHS->getOperand(Num: 0);
3413 SDValue NewRHS =
3414 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3415 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3416 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3417 Val: NewRHS, MMO: AN->getMemOperand());
3418 Results.push_back(Elt: Res);
3419 Results.push_back(Elt: Res.getValue(R: 1));
3420 break;
3421 }
3422 case ISD::DYNAMIC_STACKALLOC:
3423 ExpandDYNAMIC_STACKALLOC(Node, Results);
3424 break;
3425 case ISD::MERGE_VALUES:
3426 for (unsigned i = 0; i < Node->getNumValues(); i++)
3427 Results.push_back(Elt: Node->getOperand(Num: i));
3428 break;
3429 case ISD::POISON:
3430 case ISD::UNDEF: {
3431 EVT VT = Node->getValueType(ResNo: 0);
3432 if (VT.isInteger())
3433 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3434 else {
3435 assert(VT.isFloatingPoint() && "Unknown value type!");
3436 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3437 }
3438 break;
3439 }
3440 case ISD::STRICT_FP_ROUND:
3441 // When strict mode is enforced we can't do expansion because it
3442 // does not honor the "strict" properties. Only libcall is allowed.
3443 if (TLI.isStrictFPEnabled())
3444 break;
3445 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3446 // since this operation is more efficient than stack operation.
3447 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3448 VT: Node->getValueType(ResNo: 0))
3449 == TargetLowering::Legal)
3450 break;
3451 // We fall back to use stack operation when the FP_ROUND operation
3452 // isn't available.
3453 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3454 DestVT: Node->getValueType(ResNo: 0), dl,
3455 Chain: Node->getOperand(Num: 0)))) {
3456 ReplaceNode(Old: Node, New: Tmp1.getNode());
3457 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3458 return true;
3459 }
3460 break;
3461 case ISD::FP_ROUND: {
3462 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3463 Results.push_back(Elt: Tmp1);
3464 break;
3465 }
3466
3467 [[fallthrough]];
3468 }
3469 case ISD::BITCAST:
3470 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3471 DestVT: Node->getValueType(ResNo: 0), dl)))
3472 Results.push_back(Elt: Tmp1);
3473 break;
3474 case ISD::STRICT_FP_EXTEND:
3475 // When strict mode is enforced we can't do expansion because it
3476 // does not honor the "strict" properties. Only libcall is allowed.
3477 if (TLI.isStrictFPEnabled())
3478 break;
3479 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3480 // since this operation is more efficient than stack operation.
3481 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3482 VT: Node->getValueType(ResNo: 0))
3483 == TargetLowering::Legal)
3484 break;
3485 // We fall back to use stack operation when the FP_EXTEND operation
3486 // isn't available.
3487 if ((Tmp1 = EmitStackConvert(
3488 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3489 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3490 ReplaceNode(Old: Node, New: Tmp1.getNode());
3491 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3492 return true;
3493 }
3494 break;
3495 case ISD::FP_EXTEND: {
3496 SDValue Op = Node->getOperand(Num: 0);
3497 EVT SrcVT = Op.getValueType();
3498 EVT DstVT = Node->getValueType(ResNo: 0);
3499 if (SrcVT.getScalarType() == MVT::bf16) {
3500 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3501 break;
3502 }
3503
3504 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3505 Results.push_back(Elt: Tmp1);
3506 break;
3507 }
3508 case ISD::BF16_TO_FP: {
3509 // Always expand bf16 to f32 casts, they lower to ext + shift.
3510 //
3511 // Note that the operand of this code can be bf16 or an integer type in case
3512 // bf16 is not supported on the target and was softened.
3513 SDValue Op = Node->getOperand(Num: 0);
3514 if (Op.getValueType() == MVT::bf16) {
3515 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3516 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3517 } else {
3518 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3519 }
3520 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3521 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3522 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3523 // Add fp_extend in case the output is bigger than f32.
3524 if (Node->getValueType(ResNo: 0) != MVT::f32)
3525 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3526 Results.push_back(Elt: Op);
3527 break;
3528 }
3529 case ISD::FP_TO_BF16: {
3530 SDValue Op = Node->getOperand(Num: 0);
3531 if (Op.getValueType() != MVT::f32)
3532 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3533 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3534 // Certain SNaNs will turn into infinities if we do a simple shift right.
3535 if (!DAG.isKnownNeverSNaN(Op)) {
3536 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3537 }
3538 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
3539 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3540 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3541 // The result of this node can be bf16 or an integer type in case bf16 is
3542 // not supported on the target and was softened to i16 for storage.
3543 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3544 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3545 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3546 } else {
3547 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3548 }
3549 Results.push_back(Elt: Op);
3550 break;
3551 }
3552 case ISD::CONVERT_FROM_ARBITRARY_FP: {
3553 // Expand conversion from arbitrary FP format stored in an integer to a
3554 // native IEEE float type using integer bit manipulation.
3555 //
3556 // TODO: currently only conversions from FP4, FP6 and FP8 formats from OCP
3557 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3558 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3559 // Float8E8M0FNU.
3560 EVT DstVT = Node->getValueType(ResNo: 0);
3561 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
3562 Results.push_back(Elt: Expanded);
3563 else
3564 Results.push_back(Elt: DAG.getPOISON(VT: DstVT));
3565 break;
3566 }
3567 case ISD::CONVERT_TO_ARBITRARY_FP: {
3568 // Expand conversion from a native IEEE float type to an arbitrary FP
3569 // format, returning the result as an integer using bit manipulation.
3570 //
3571 // TODO: currently only conversions to FP4, FP6 and FP8 formats from OCP
3572 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3573 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3574 // Float8E8M0FNU.
3575 EVT ResVT = Node->getValueType(ResNo: 0);
3576 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
3577 Results.push_back(Elt: Expanded);
3578 else
3579 Results.push_back(Elt: DAG.getPOISON(VT: ResVT));
3580 break;
3581 }
3582 case ISD::FCANONICALIZE: {
3583 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
3584 Results.push_back(Elt: Mul);
3585 break;
3586 }
3587 case ISD::SIGN_EXTEND_INREG: {
3588 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3589 EVT VT = Node->getValueType(ResNo: 0);
3590
3591 // An in-register sign-extend of a boolean is a negation:
3592 // 'true' (1) sign-extended is -1.
3593 // 'false' (0) sign-extended is 0.
3594 // However, we must mask the high bits of the source operand because the
3595 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3596
3597 // TODO: Do this for vectors too?
3598 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3599 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3600 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3601 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3602 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3603 Results.push_back(Elt: Neg);
3604 break;
3605 }
3606
3607 // NOTE: we could fall back on load/store here too for targets without
3608 // SRA. However, it is doubtful that any exist.
3609 unsigned BitsDiff = VT.getScalarSizeInBits() -
3610 ExtraVT.getScalarSizeInBits();
3611 SDValue ShiftCst = DAG.getShiftAmountConstant(Val: BitsDiff, VT, DL: dl);
3612 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: ShiftCst);
3613 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Tmp1, N2: ShiftCst);
3614 Results.push_back(Elt: Tmp1);
3615 break;
3616 }
3617 case ISD::UINT_TO_FP:
3618 case ISD::STRICT_UINT_TO_FP:
3619 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3620 Results.push_back(Elt: Tmp1);
3621 if (Node->isStrictFPOpcode())
3622 Results.push_back(Elt: Tmp2);
3623 break;
3624 }
3625 [[fallthrough]];
3626 case ISD::SINT_TO_FP:
3627 case ISD::STRICT_SINT_TO_FP:
3628 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3629 Results.push_back(Elt: Tmp1);
3630 if (Node->isStrictFPOpcode())
3631 Results.push_back(Elt: Tmp2);
3632 }
3633 break;
3634 case ISD::FP_TO_SINT:
3635 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3636 Results.push_back(Elt: Tmp1);
3637 break;
3638 case ISD::STRICT_FP_TO_SINT:
3639 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3640 ReplaceNode(Old: Node, New: Tmp1.getNode());
3641 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3642 return true;
3643 }
3644 break;
3645 case ISD::FP_TO_UINT:
3646 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3647 Results.push_back(Elt: Tmp1);
3648 break;
3649 case ISD::STRICT_FP_TO_UINT:
3650 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3651 // Relink the chain.
3652 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3653 // Replace the new UINT result.
3654 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3655 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3656 return true;
3657 }
3658 break;
3659 case ISD::FP_TO_SINT_SAT:
3660 case ISD::FP_TO_UINT_SAT:
3661 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3662 break;
3663 case ISD::LROUND:
3664 case ISD::LLROUND: {
3665 SDValue Arg = Node->getOperand(Num: 0);
3666 EVT ArgVT = Arg.getValueType();
3667 EVT ResVT = Node->getValueType(ResNo: 0);
3668 SDLoc dl(Node);
3669 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3670 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3671 break;
3672 }
3673 case ISD::VAARG:
3674 Results.push_back(Elt: DAG.expandVAArg(Node));
3675 Results.push_back(Elt: Results[0].getValue(R: 1));
3676 break;
3677 case ISD::VACOPY:
3678 Results.push_back(Elt: DAG.expandVACopy(Node));
3679 break;
3680 case ISD::EXTRACT_VECTOR_ELT:
3681 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3682 // This must be an access of the only element. Return it.
3683 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3684 Operand: Node->getOperand(Num: 0));
3685 else
3686 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3687 Results.push_back(Elt: Tmp1);
3688 break;
3689 case ISD::EXTRACT_SUBVECTOR:
3690 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3691 break;
3692 case ISD::INSERT_SUBVECTOR:
3693 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3694 break;
3695 case ISD::CONCAT_VECTORS:
3696 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3697 VectorValueType.isScalableVector() ||
3698 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3699 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3700 else
3701 Results.push_back(Elt: ExpandConcatVectors(Node));
3702 break;
3703 case ISD::SCALAR_TO_VECTOR:
3704 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3705 break;
3706 case ISD::INSERT_VECTOR_ELT:
3707 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3708 break;
3709 case ISD::VECTOR_SHUFFLE: {
3710 SmallVector<int, 32> NewMask;
3711 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3712
3713 EVT VT = Node->getValueType(ResNo: 0);
3714 EVT EltVT = VT.getVectorElementType();
3715 SDValue Op0 = Node->getOperand(Num: 0);
3716 SDValue Op1 = Node->getOperand(Num: 1);
3717 if (!TLI.isTypeLegal(VT: EltVT)) {
3718 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3719
3720 // BUILD_VECTOR operands are allowed to be wider than the element type.
3721 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3722 // it.
3723 if (NewEltVT.bitsLT(VT: EltVT)) {
3724 // Convert shuffle node.
3725 // If original node was v4i64 and the new EltVT is i32,
3726 // cast operands to v8i32 and re-build the mask.
3727
3728 // Calculate new VT, the size of the new VT should be equal to original.
3729 EVT NewVT =
3730 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3731 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3732 assert(NewVT.bitsEq(VT));
3733
3734 // cast operands to new VT
3735 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3736 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3737
3738 // Convert the shuffle mask
3739 unsigned int factor =
3740 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3741
3742 // EltVT gets smaller
3743 assert(factor > 0);
3744
3745 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3746 if (Mask[i] < 0) {
3747 for (unsigned fi = 0; fi < factor; ++fi)
3748 NewMask.push_back(Elt: Mask[i]);
3749 }
3750 else {
3751 for (unsigned fi = 0; fi < factor; ++fi)
3752 NewMask.push_back(Elt: Mask[i]*factor+fi);
3753 }
3754 }
3755 Mask = NewMask;
3756 VT = NewVT;
3757 }
3758 EltVT = NewEltVT;
3759 }
3760 unsigned NumElems = VT.getVectorNumElements();
3761 SmallVector<SDValue, 16> Ops;
3762 for (unsigned i = 0; i != NumElems; ++i) {
3763 if (Mask[i] < 0) {
3764 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3765 continue;
3766 }
3767 unsigned Idx = Mask[i];
3768 if (Idx < NumElems)
3769 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3770 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3771 else
3772 Ops.push_back(
3773 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3774 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3775 }
3776
3777 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3778 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3779 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3780 Results.push_back(Elt: Tmp1);
3781 break;
3782 }
3783 case ISD::VECTOR_SPLICE_LEFT:
3784 case ISD::VECTOR_SPLICE_RIGHT: {
3785 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3786 break;
3787 }
3788 case ISD::VECTOR_DEINTERLEAVE: {
3789 unsigned Factor = Node->getNumOperands();
3790 if (Factor <= 2 || Factor % 2 != 0)
3791 break;
3792 SmallVector<SDValue, 8> Ops(Node->ops());
3793 EVT VecVT = Node->getValueType(ResNo: 0);
3794 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3795 // Deinterleave at Factor/2 so each result contains two factors interleaved:
3796 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
3797 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3798 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
3799 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3800 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
3801 Results.resize(N: Factor);
3802 // Deinterleave the 2 factors out:
3803 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
3804 for (unsigned I = 0; I < Factor / 2; I++) {
3805 SDValue Deinterleave =
3806 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3807 Ops: {L.getValue(R: I), R.getValue(R: I)});
3808 Results[I] = Deinterleave.getValue(R: 0);
3809 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
3810 }
3811 break;
3812 }
3813 case ISD::VECTOR_INTERLEAVE: {
3814 unsigned Factor = Node->getNumOperands();
3815 if (Factor <= 2 || Factor % 2 != 0)
3816 break;
3817 EVT VecVT = Node->getValueType(ResNo: 0);
3818 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3819 SmallVector<SDValue, 8> LOps, ROps;
3820 // Interleave so we have 2 factors per result:
3821 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
3822 for (unsigned I = 0; I < Factor / 2; I++) {
3823 SDValue Interleave =
3824 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3825 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
3826 LOps.push_back(Elt: Interleave.getValue(R: 0));
3827 ROps.push_back(Elt: Interleave.getValue(R: 1));
3828 }
3829 // Interleave at Factor/2:
3830 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
3831 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: LOps);
3832 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: ROps);
3833 for (unsigned I = 0; I < Factor / 2; I++)
3834 Results.push_back(Elt: L.getValue(R: I));
3835 for (unsigned I = 0; I < Factor / 2; I++)
3836 Results.push_back(Elt: R.getValue(R: I));
3837 break;
3838 }
3839 case ISD::EXTRACT_ELEMENT: {
3840 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3841 if (Node->getConstantOperandVal(Num: 1)) {
3842 // 1 -> Hi
3843 Tmp1 = DAG.getNode(
3844 Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3845 N2: DAG.getShiftAmountConstant(Val: OpTy.getSizeInBits() / 2, VT: OpTy, DL: dl));
3846 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3847 } else {
3848 // 0 -> Lo
3849 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3850 Operand: Node->getOperand(Num: 0));
3851 }
3852 Results.push_back(Elt: Tmp1);
3853 break;
3854 }
3855 case ISD::STACKADDRESS:
3856 case ISD::STACKSAVE:
3857 // Expand to CopyFromReg if the target set
3858 // StackPointerRegisterToSaveRestore.
3859 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3860 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3861 VT: Node->getValueType(ResNo: 0)));
3862 Results.push_back(Elt: Results[0].getValue(R: 1));
3863 } else {
3864 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3865 Results.push_back(Elt: Node->getOperand(Num: 0));
3866
3867 StringRef IntrinsicName = Node->getOpcode() == ISD::STACKADDRESS
3868 ? "llvm.stackaddress"
3869 : "llvm.stacksave";
3870 DAG.getContext()->diagnose(DI: DiagnosticInfoLegalizationFailure(
3871 Twine(IntrinsicName) + " is not supported on this target.",
3872 DAG.getMachineFunction().getFunction(), dl.getDebugLoc()));
3873 }
3874 break;
3875 case ISD::STACKRESTORE:
3876 // Expand to CopyToReg if the target set
3877 // StackPointerRegisterToSaveRestore.
3878 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3879 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3880 N: Node->getOperand(Num: 1)));
3881 } else {
3882 Results.push_back(Elt: Node->getOperand(Num: 0));
3883 }
3884 break;
3885 case ISD::GET_DYNAMIC_AREA_OFFSET:
3886 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3887 Results.push_back(Elt: Results[0].getValue(R: 0));
3888 break;
3889 case ISD::FCOPYSIGN:
3890 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3891 break;
3892 case ISD::FNEG:
3893 Results.push_back(Elt: ExpandFNEG(Node));
3894 break;
3895 case ISD::FABS:
3896 Results.push_back(Elt: ExpandFABS(Node));
3897 break;
3898 case ISD::IS_FPCLASS: {
3899 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3900 if (SDValue Expanded =
3901 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3902 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3903 Results.push_back(Elt: Expanded);
3904 break;
3905 }
3906 case ISD::SMIN:
3907 case ISD::SMAX:
3908 case ISD::UMIN:
3909 case ISD::UMAX: {
3910 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3911 ISD::CondCode Pred;
3912 switch (Node->getOpcode()) {
3913 default: llvm_unreachable("How did we get here?");
3914 case ISD::SMAX: Pred = ISD::SETGT; break;
3915 case ISD::SMIN: Pred = ISD::SETLT; break;
3916 case ISD::UMAX: Pred = ISD::SETUGT; break;
3917 case ISD::UMIN: Pred = ISD::SETULT; break;
3918 }
3919 Tmp1 = Node->getOperand(Num: 0);
3920 Tmp2 = Node->getOperand(Num: 1);
3921 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3922 Results.push_back(Elt: Tmp1);
3923 break;
3924 }
3925 case ISD::FMINNUM:
3926 case ISD::FMAXNUM: {
3927 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3928 Results.push_back(Elt: Expanded);
3929 break;
3930 }
3931 case ISD::FMINIMUM:
3932 case ISD::FMAXIMUM: {
3933 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
3934 Results.push_back(Elt: Expanded);
3935 break;
3936 }
3937 case ISD::FMINIMUMNUM:
3938 case ISD::FMAXIMUMNUM: {
3939 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
3940 break;
3941 }
3942 case ISD::FSIN:
3943 case ISD::FCOS: {
3944 EVT VT = Node->getValueType(ResNo: 0);
3945 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3946 // fcos which share the same operand and both are used.
3947 if ((TLI.isOperationLegal(Op: ISD::FSINCOS, VT) ||
3948 isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls())) &&
3949 useSinCos(Node)) {
3950 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3951 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3952 if (Node->getOpcode() == ISD::FCOS)
3953 Tmp1 = Tmp1.getValue(R: 1);
3954 Results.push_back(Elt: Tmp1);
3955 }
3956 break;
3957 }
3958 case ISD::FLDEXP:
3959 case ISD::STRICT_FLDEXP: {
3960 EVT VT = Node->getValueType(ResNo: 0);
3961 RTLIB::Libcall LC = RTLIB::getLDEXP(VT);
3962 // Use the LibCall instead, it is very likely faster
3963 // FIXME: Use separate LibCall action.
3964 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3965 break;
3966
3967 if (SDValue Expanded = expandLdexp(Node)) {
3968 Results.push_back(Elt: Expanded);
3969 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3970 Results.push_back(Elt: Expanded.getValue(R: 1));
3971 }
3972
3973 break;
3974 }
3975 case ISD::FFREXP: {
3976 RTLIB::Libcall LC = RTLIB::getFREXP(VT: Node->getValueType(ResNo: 0));
3977 // Use the LibCall instead, it is very likely faster
3978 // FIXME: Use separate LibCall action.
3979 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3980 break;
3981
3982 if (SDValue Expanded = expandFrexp(Node)) {
3983 Results.push_back(Elt: Expanded);
3984 Results.push_back(Elt: Expanded.getValue(R: 1));
3985 }
3986 break;
3987 }
3988 case ISD::FMODF: {
3989 RTLIB::Libcall LC = RTLIB::getMODF(VT: Node->getValueType(ResNo: 0));
3990 // Use the LibCall instead, it is very likely faster
3991 // FIXME: Use separate LibCall action.
3992 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3993 break;
3994
3995 if (SDValue Expanded = expandModf(Node)) {
3996 Results.push_back(Elt: Expanded);
3997 Results.push_back(Elt: Expanded.getValue(R: 1));
3998 }
3999 break;
4000 }
4001 case ISD::FSINCOS: {
4002 if (isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls()))
4003 break;
4004 EVT VT = Node->getValueType(ResNo: 0);
4005 SDValue Op = Node->getOperand(Num: 0);
4006 SDNodeFlags Flags = Node->getFlags();
4007 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
4008 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
4009 Results.append(IL: {Tmp1, Tmp2});
4010 break;
4011 }
4012 case ISD::FMAD:
4013 llvm_unreachable("Illegal fmad should never be formed");
4014
4015 case ISD::FP16_TO_FP:
4016 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4017 // We can extend to types bigger than f32 in two steps without changing
4018 // the result. Since "f16 -> f32" is much more commonly available, give
4019 // CodeGen the option of emitting that before resorting to a libcall.
4020 SDValue Res =
4021 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
4022 Results.push_back(
4023 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
4024 }
4025 break;
4026 case ISD::STRICT_BF16_TO_FP:
4027 case ISD::STRICT_FP16_TO_FP:
4028 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4029 // We can extend to types bigger than f32 in two steps without changing
4030 // the result. Since "f16 -> f32" is much more commonly available, give
4031 // CodeGen the option of emitting that before resorting to a libcall.
4032 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
4033 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
4034 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
4035 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
4036 Ops: {Res.getValue(R: 1), Res});
4037 Results.push_back(Elt: Res);
4038 Results.push_back(Elt: Res.getValue(R: 1));
4039 }
4040 break;
4041 case ISD::FP_TO_FP16:
4042 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
4043 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
4044 SDValue Op = Node->getOperand(Num: 0);
4045 MVT SVT = Op.getSimpleValueType();
4046 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
4047 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
4048 // Under fastmath, we can expand this node into a fround followed by
4049 // a float-half conversion.
4050 SDValue FloatVal =
4051 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
4052 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
4053 Results.push_back(
4054 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
4055 }
4056 }
4057 break;
4058 case ISD::ConstantFP: {
4059 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
4060 // Check to see if this FP immediate is already legal.
4061 // If this is a legal constant, turn it into a TargetConstantFP node.
4062 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
4063 ForCodeSize: DAG.shouldOptForSize()))
4064 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
4065 break;
4066 }
4067 case ISD::Constant: {
4068 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
4069 Results.push_back(Elt: ExpandConstant(CP));
4070 break;
4071 }
4072 case ISD::FSUB: {
4073 EVT VT = Node->getValueType(ResNo: 0);
4074 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
4075 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
4076 const SDNodeFlags Flags = Node->getFlags();
4077 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
4078 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
4079 Results.push_back(Elt: Tmp1);
4080 }
4081 break;
4082 }
4083 case ISD::SUB: {
4084 EVT VT = Node->getValueType(ResNo: 0);
4085 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
4086 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
4087 "Don't know how to expand this subtraction!");
4088 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
4089 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
4090 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
4091 break;
4092 }
4093 case ISD::UREM:
4094 case ISD::SREM:
4095 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
4096 Results.push_back(Elt: Tmp1);
4097 break;
4098 case ISD::UDIV:
4099 case ISD::SDIV: {
4100 bool isSigned = Node->getOpcode() == ISD::SDIV;
4101 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4102 EVT VT = Node->getValueType(ResNo: 0);
4103 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
4104 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4105 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4106 N2: Node->getOperand(Num: 1));
4107 Results.push_back(Elt: Tmp1);
4108 }
4109 break;
4110 }
4111 case ISD::MULHU:
4112 case ISD::MULHS: {
4113 unsigned ExpandOpcode =
4114 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
4115 EVT VT = Node->getValueType(ResNo: 0);
4116 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4117
4118 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4119 N2: Node->getOperand(Num: 1));
4120 Results.push_back(Elt: Tmp1.getValue(R: 1));
4121 break;
4122 }
4123 case ISD::UMUL_LOHI:
4124 case ISD::SMUL_LOHI: {
4125 SDValue LHS = Node->getOperand(Num: 0);
4126 SDValue RHS = Node->getOperand(Num: 1);
4127 EVT VT = LHS.getValueType();
4128 unsigned MULHOpcode =
4129 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
4130
4131 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
4132 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
4133 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
4134 break;
4135 }
4136
4137 SmallVector<SDValue, 4> Halves;
4138 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4139 assert(TLI.isTypeLegal(HalfType));
4140 if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
4141 HiLoVT: HalfType, DAG,
4142 Kind: TargetLowering::MulExpansionKind::Always)) {
4143 for (unsigned i = 0; i < 2; ++i) {
4144 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
4145 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
4146 SDValue Shift =
4147 DAG.getShiftAmountConstant(Val: HalfType.getScalarSizeInBits(), VT, DL: dl);
4148 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4149 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4150 }
4151 break;
4152 }
4153 break;
4154 }
4155 case ISD::MUL: {
4156 EVT VT = Node->getValueType(ResNo: 0);
4157 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4158 // See if multiply or divide can be lowered using two-result operations.
4159 // We just need the low half of the multiply; try both the signed
4160 // and unsigned forms. If the target supports both SMUL_LOHI and
4161 // UMUL_LOHI, form a preference by checking which forms of plain
4162 // MULH it supports.
4163 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
4164 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
4165 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
4166 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
4167 unsigned OpToUse = 0;
4168 if (HasSMUL_LOHI && !HasMULHS) {
4169 OpToUse = ISD::SMUL_LOHI;
4170 } else if (HasUMUL_LOHI && !HasMULHU) {
4171 OpToUse = ISD::UMUL_LOHI;
4172 } else if (HasSMUL_LOHI) {
4173 OpToUse = ISD::SMUL_LOHI;
4174 } else if (HasUMUL_LOHI) {
4175 OpToUse = ISD::UMUL_LOHI;
4176 }
4177 if (OpToUse) {
4178 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4179 N2: Node->getOperand(Num: 1)));
4180 break;
4181 }
4182
4183 SDValue Lo, Hi;
4184 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4185 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
4186 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
4187 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
4188 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
4189 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
4190 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4191 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
4192 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
4193 SDValue Shift =
4194 DAG.getShiftAmountConstant(Val: HalfType.getSizeInBits(), VT, DL: dl);
4195 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4196 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4197 }
4198 break;
4199 }
4200 case ISD::FSHL:
4201 case ISD::FSHR:
4202 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
4203 Results.push_back(Elt: Expanded);
4204 break;
4205 case ISD::ROTL:
4206 case ISD::ROTR:
4207 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
4208 Results.push_back(Elt: Expanded);
4209 break;
4210 case ISD::CLMUL:
4211 case ISD::CLMULR:
4212 case ISD::CLMULH:
4213 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG))
4214 Results.push_back(Elt: Expanded);
4215 break;
4216 case ISD::PEXT:
4217 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
4218 break;
4219 case ISD::PDEP:
4220 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
4221 break;
4222 case ISD::SADDSAT:
4223 case ISD::UADDSAT:
4224 case ISD::SSUBSAT:
4225 case ISD::USUBSAT:
4226 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
4227 break;
4228 case ISD::SCMP:
4229 case ISD::UCMP:
4230 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
4231 break;
4232 case ISD::SSHLSAT:
4233 case ISD::USHLSAT:
4234 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
4235 break;
4236 case ISD::SMULFIX:
4237 case ISD::SMULFIXSAT:
4238 case ISD::UMULFIX:
4239 case ISD::UMULFIXSAT:
4240 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
4241 break;
4242 case ISD::SDIVFIX:
4243 case ISD::SDIVFIXSAT:
4244 case ISD::UDIVFIX:
4245 case ISD::UDIVFIXSAT:
4246 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
4247 LHS: Node->getOperand(Num: 0),
4248 RHS: Node->getOperand(Num: 1),
4249 Scale: Node->getConstantOperandVal(Num: 2),
4250 DAG)) {
4251 Results.push_back(Elt: V);
4252 break;
4253 }
4254 // FIXME: We might want to retry here with a wider type if we fail, if that
4255 // type is legal.
4256 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4257 // <= 128 (which is the case for all of the default Embedded-C types),
4258 // we will only get here with types and scales that we could always expand
4259 // if we were allowed to generate libcalls to division functions of illegal
4260 // type. But we cannot do that.
4261 llvm_unreachable("Cannot expand DIVFIX!");
4262 case ISD::UADDO_CARRY:
4263 case ISD::USUBO_CARRY: {
4264 SDValue LHS = Node->getOperand(Num: 0);
4265 SDValue RHS = Node->getOperand(Num: 1);
4266 SDValue Carry = Node->getOperand(Num: 2);
4267
4268 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4269
4270 // Initial add of the 2 operands.
4271 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4272 EVT VT = LHS.getValueType();
4273 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4274
4275 // Initial check for overflow.
4276 EVT CarryType = Node->getValueType(ResNo: 1);
4277 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4278 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4279 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4280
4281 // Add of the sum and the carry.
4282 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4283 SDValue CarryExt =
4284 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4285 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4286
4287 // Second check for overflow. If we are adding, we can only overflow if the
4288 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4289 // If we are subtracting, we can only overflow if the initial sum is 0 and
4290 // the carry is set, resulting in a new sum of all 1s.
4291 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4292 SDValue Overflow2 =
4293 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4294 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4295 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4296 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4297
4298 SDValue ResultCarry =
4299 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4300
4301 Results.push_back(Elt: Sum2);
4302 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4303 break;
4304 }
4305 case ISD::SADDO:
4306 case ISD::SSUBO: {
4307 SDValue Result, Overflow;
4308 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4309 Results.push_back(Elt: Result);
4310 Results.push_back(Elt: Overflow);
4311 break;
4312 }
4313 case ISD::UADDO:
4314 case ISD::USUBO: {
4315 SDValue Result, Overflow;
4316 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4317 Results.push_back(Elt: Result);
4318 Results.push_back(Elt: Overflow);
4319 break;
4320 }
4321 case ISD::UMULO:
4322 case ISD::SMULO: {
4323 SDValue Result, Overflow;
4324 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4325 Results.push_back(Elt: Result);
4326 Results.push_back(Elt: Overflow);
4327 }
4328 break;
4329 }
4330 case ISD::BUILD_PAIR: {
4331 EVT PairTy = Node->getValueType(ResNo: 0);
4332 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4333 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4334 Tmp2 = DAG.getNode(
4335 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4336 N2: DAG.getShiftAmountConstant(Val: PairTy.getSizeInBits() / 2, VT: PairTy, DL: dl));
4337 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4338 break;
4339 }
4340 case ISD::SELECT:
4341 Tmp1 = Node->getOperand(Num: 0);
4342 Tmp2 = Node->getOperand(Num: 1);
4343 Tmp3 = Node->getOperand(Num: 2);
4344 if (Tmp1.getOpcode() == ISD::SETCC) {
4345 Tmp1 = DAG.getSelectCC(
4346 DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), True: Tmp2, False: Tmp3,
4347 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get(), Flags: Node->getFlags());
4348 } else {
4349 Tmp1 =
4350 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4351 True: Tmp2, False: Tmp3, Cond: ISD::SETNE, Flags: Node->getFlags());
4352 }
4353 Results.push_back(Elt: Tmp1);
4354 break;
4355 case ISD::BR_JT: {
4356 SDValue Chain = Node->getOperand(Num: 0);
4357 SDValue Table = Node->getOperand(Num: 1);
4358 SDValue Index = Node->getOperand(Num: 2);
4359 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4360
4361 const DataLayout &TD = DAG.getDataLayout();
4362 EVT PTy = TLI.getPointerTy(DL: TD);
4363
4364 unsigned EntrySize =
4365 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4366
4367 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4368 // This transformation needs to be done here since otherwise the MIPS
4369 // backend will end up emitting a three instruction multiply sequence
4370 // instead of a single shift and MSP430 will call a runtime function.
4371 if (llvm::isPowerOf2_32(Value: EntrySize))
4372 Index = DAG.getNode(
4373 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4374 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4375 else
4376 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4377 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4378 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4379
4380 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4381 SDValue LD = DAG.getExtLoad(
4382 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4383 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4384 Addr = LD;
4385 if (TLI.isJumpTableRelative()) {
4386 // For PIC, the sequence is:
4387 // BRIND(RelocBase + load(Jumptable + index))
4388 // RelocBase can be JumpTable, GOT or some sort of global base.
4389 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4390 Offset: Addr, DL: dl);
4391 }
4392
4393 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4394 Results.push_back(Elt: Tmp1);
4395 break;
4396 }
4397 case ISD::BRCOND:
4398 // Expand brcond's setcc into its constituent parts and create a BR_CC
4399 // Node.
4400 Tmp1 = Node->getOperand(Num: 0);
4401 Tmp2 = Node->getOperand(Num: 1);
4402 if (Tmp2.getOpcode() == ISD::SETCC &&
4403 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4404 VT: Tmp2.getOperand(i: 0).getValueType())) {
4405 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4406 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4407 N5: Node->getOperand(Num: 2));
4408 } else {
4409 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4410 if (Tmp2.isUndef() ||
4411 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4412 Tmp3 = Tmp2;
4413 else
4414 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4415 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4416 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4417 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4418 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4419 N5: Node->getOperand(Num: 2));
4420 }
4421 Results.push_back(Elt: Tmp1);
4422 break;
4423 case ISD::SETCC:
4424 case ISD::STRICT_FSETCC:
4425 case ISD::STRICT_FSETCCS: {
4426 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4427 Node->getOpcode() == ISD::STRICT_FSETCCS;
4428 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4429 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4430 unsigned Offset = IsStrict ? 1 : 0;
4431 Tmp1 = Node->getOperand(Num: 0 + Offset);
4432 Tmp2 = Node->getOperand(Num: 1 + Offset);
4433 Tmp3 = Node->getOperand(Num: 2 + Offset);
4434 bool Legalized =
4435 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3,
4436 NeedInvert, dl, Chain, IsSignaling);
4437
4438 if (Legalized) {
4439 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4440 // condition code, create a new SETCC node.
4441 if (Tmp3.getNode()) {
4442 if (IsStrict) {
4443 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4444 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4445 Chain = Tmp1.getValue(R: 1);
4446 } else {
4447 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4448 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4449 }
4450 }
4451
4452 // If we expanded the SETCC by inverting the condition code, then wrap
4453 // the existing SETCC in a NOT to restore the intended condition.
4454 if (NeedInvert) {
4455 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4456 }
4457
4458 Results.push_back(Elt: Tmp1);
4459 if (IsStrict)
4460 Results.push_back(Elt: Chain);
4461
4462 break;
4463 }
4464
4465 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4466 // understand if this code is useful for strict nodes.
4467 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4468
4469 // Otherwise, SETCC for the given comparison type must be completely
4470 // illegal; expand it into a SELECT_CC.
4471 EVT VT = Node->getValueType(ResNo: 0);
4472 EVT Tmp1VT = Tmp1.getValueType();
4473 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4474 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4475 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3,
4476 Flags: Node->getFlags());
4477 Results.push_back(Elt: Tmp1);
4478 break;
4479 }
4480 case ISD::SELECT_CC: {
4481 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4482 Tmp1 = Node->getOperand(Num: 0); // LHS
4483 Tmp2 = Node->getOperand(Num: 1); // RHS
4484 Tmp3 = Node->getOperand(Num: 2); // True
4485 Tmp4 = Node->getOperand(Num: 3); // False
4486 EVT VT = Node->getValueType(ResNo: 0);
4487 SDValue Chain;
4488 SDValue CC = Node->getOperand(Num: 4);
4489 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4490
4491 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4492 // If the condition code is legal, then we need to expand this
4493 // node using SETCC and SELECT.
4494 EVT CmpVT = Tmp1.getValueType();
4495 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4496 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4497 "expanded.");
4498 EVT CCVT = getSetCCResultType(VT: CmpVT);
4499 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4500 Results.push_back(
4501 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4502 break;
4503 }
4504
4505 // SELECT_CC is legal, so the condition code must not be.
4506 bool Legalized = false;
4507 // Try to legalize by inverting the condition. This is for targets that
4508 // might support an ordered version of a condition, but not the unordered
4509 // version (or vice versa).
4510 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4511 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4512 // Use the new condition code and swap true and false
4513 Legalized = true;
4514 Tmp1 =
4515 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC, Flags: Node->getFlags());
4516 } else {
4517 // If The inverse is not legal, then try to swap the arguments using
4518 // the inverse condition code.
4519 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4520 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4521 // The swapped inverse condition is legal, so swap true and false,
4522 // lhs and rhs.
4523 Legalized = true;
4524 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC,
4525 Flags: Node->getFlags());
4526 }
4527 }
4528
4529 if (!Legalized) {
4530 Legalized = TLI.LegalizeSetCCCondCode(
4531 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4532 NeedInvert, dl, Chain);
4533
4534 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4535
4536 // If we expanded the SETCC by inverting the condition code, then swap
4537 // the True/False operands to match.
4538 if (NeedInvert)
4539 std::swap(a&: Tmp3, b&: Tmp4);
4540
4541 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4542 // condition code, create a new SELECT_CC node.
4543 if (CC.getNode()) {
4544 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4545 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4546 } else {
4547 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4548 CC = DAG.getCondCode(Cond: ISD::SETNE);
4549 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4550 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4551 }
4552 }
4553 Results.push_back(Elt: Tmp1);
4554 break;
4555 }
4556 case ISD::BR_CC: {
4557 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4558 SDValue Chain;
4559 Tmp1 = Node->getOperand(Num: 0); // Chain
4560 Tmp2 = Node->getOperand(Num: 2); // LHS
4561 Tmp3 = Node->getOperand(Num: 3); // RHS
4562 Tmp4 = Node->getOperand(Num: 1); // CC
4563
4564 bool Legalized =
4565 TLI.LegalizeSetCCCondCode(DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()),
4566 LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4, NeedInvert, dl, Chain);
4567 (void)Legalized;
4568 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4569
4570 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4571 // node.
4572 if (Tmp4.getNode()) {
4573 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4574
4575 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4576 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4577 } else {
4578 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4579 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4580 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4581 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4582 }
4583 Results.push_back(Elt: Tmp1);
4584 break;
4585 }
4586 case ISD::BUILD_VECTOR:
4587 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4588 break;
4589 case ISD::SPLAT_VECTOR:
4590 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4591 break;
4592 case ISD::SRA:
4593 case ISD::SRL:
4594 case ISD::SHL: {
4595 // Scalarize vector SRA/SRL/SHL.
4596 EVT VT = Node->getValueType(ResNo: 0);
4597 assert(VT.isVector() && "Unable to legalize non-vector shift");
4598 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4599 unsigned NumElem = VT.getVectorNumElements();
4600
4601 SmallVector<SDValue, 8> Scalars;
4602 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4603 SDValue Ex =
4604 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4605 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4606 SDValue Sh =
4607 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4608 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4609 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4610 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4611 }
4612
4613 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4614 Results.push_back(Elt: Result);
4615 break;
4616 }
4617 case ISD::VECREDUCE_FADD:
4618 case ISD::VECREDUCE_FMUL:
4619 case ISD::VECREDUCE_ADD:
4620 case ISD::VECREDUCE_MUL:
4621 case ISD::VECREDUCE_AND:
4622 case ISD::VECREDUCE_OR:
4623 case ISD::VECREDUCE_XOR:
4624 case ISD::VECREDUCE_SMAX:
4625 case ISD::VECREDUCE_SMIN:
4626 case ISD::VECREDUCE_UMAX:
4627 case ISD::VECREDUCE_UMIN:
4628 case ISD::VECREDUCE_FMAX:
4629 case ISD::VECREDUCE_FMIN:
4630 case ISD::VECREDUCE_FMAXIMUM:
4631 case ISD::VECREDUCE_FMINIMUM:
4632 case ISD::VECREDUCE_FMAXIMUMNUM:
4633 case ISD::VECREDUCE_FMINIMUMNUM:
4634 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4635 break;
4636 case ISD::VP_CTTZ_ELTS:
4637 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4638 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4639 break;
4640 case ISD::CLEAR_CACHE:
4641 // The default expansion of llvm.clear_cache is simply a no-op for those
4642 // targets where it is not needed.
4643 Results.push_back(Elt: Node->getOperand(Num: 0));
4644 break;
4645 case ISD::LRINT:
4646 case ISD::LLRINT: {
4647 SDValue Arg = Node->getOperand(Num: 0);
4648 EVT ArgVT = Arg.getValueType();
4649 EVT ResVT = Node->getValueType(ResNo: 0);
4650 SDLoc DL(Node);
4651 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL, VT: ArgVT, Operand: Arg);
4652 SDValue ConvertNode = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: ResVT, Operand: RoundNode);
4653 // Non-deterministic results are equivalent to freeze poison.
4654 Results.push_back(Elt: DAG.getFreeze(V: ConvertNode));
4655 break;
4656 }
4657 case ISD::ADDRSPACECAST:
4658 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4659 break;
4660 case ISD::GLOBAL_OFFSET_TABLE:
4661 case ISD::GlobalAddress:
4662 case ISD::GlobalTLSAddress:
4663 case ISD::ExternalSymbol:
4664 case ISD::ConstantPool:
4665 case ISD::JumpTable:
4666 case ISD::INTRINSIC_W_CHAIN:
4667 case ISD::INTRINSIC_WO_CHAIN:
4668 case ISD::INTRINSIC_VOID:
4669 // FIXME: Custom lowering for these operations shouldn't return null!
4670 // Return true so that we don't call ConvertNodeToLibcall which also won't
4671 // do anything.
4672 return true;
4673 }
4674
4675 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4676 // FIXME: We were asked to expand a strict floating-point operation,
4677 // but there is currently no expansion implemented that would preserve
4678 // the "strict" properties. For now, we just fall back to the non-strict
4679 // version if that is legal on the target. The actual mutation of the
4680 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4681 switch (Node->getOpcode()) {
4682 default:
4683 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4684 VT: Node->getValueType(ResNo: 0))
4685 == TargetLowering::Legal)
4686 return true;
4687 break;
4688 case ISD::STRICT_FSUB: {
4689 if (TLI.getStrictFPOperationAction(
4690 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4691 return true;
4692 if (TLI.getStrictFPOperationAction(
4693 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4694 break;
4695
4696 EVT VT = Node->getValueType(ResNo: 0);
4697 const SDNodeFlags Flags = Node->getFlags();
4698 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4699 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4700 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4701 Flags);
4702
4703 Results.push_back(Elt: Fadd);
4704 Results.push_back(Elt: Fadd.getValue(R: 1));
4705 break;
4706 }
4707 case ISD::STRICT_SINT_TO_FP:
4708 case ISD::STRICT_UINT_TO_FP:
4709 case ISD::STRICT_LRINT:
4710 case ISD::STRICT_LLRINT:
4711 case ISD::STRICT_LROUND:
4712 case ISD::STRICT_LLROUND:
4713 // These are registered by the operand type instead of the value
4714 // type. Reflect that here.
4715 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4716 VT: Node->getOperand(Num: 1).getValueType())
4717 == TargetLowering::Legal)
4718 return true;
4719 break;
4720 }
4721 }
4722
4723 // Replace the original node with the legalized result.
4724 if (Results.empty()) {
4725 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4726 return false;
4727 }
4728
4729 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4730 ReplaceNode(Old: Node, New: Results.data());
4731 return true;
4732}
4733
4734/// Return if we can use the FAST_* variant of a math libcall for the node.
4735/// FIXME: This is just guessing, we probably should have unique specific sets
4736/// flags required per libcall.
4737static bool canUseFastMathLibcall(const SDNode *Node) {
4738 // FIXME: Probably should define fast to respect nan/inf and only be
4739 // approximate functions.
4740
4741 SDNodeFlags Flags = Node->getFlags();
4742 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4743 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4744}
4745
4746void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4747 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4748 SmallVector<SDValue, 8> Results;
4749 SDLoc dl(Node);
4750 TargetLowering::MakeLibCallOptions CallOptions;
4751 CallOptions.IsPostTypeLegalization = true;
4752 // FIXME: Check flags on the node to see if we can use a finite call.
4753 unsigned Opc = Node->getOpcode();
4754 switch (Opc) {
4755 case ISD::ATOMIC_FENCE: {
4756 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4757 // FIXME: handle "fence singlethread" more efficiently.
4758 TargetLowering::ArgListTy Args;
4759
4760 TargetLowering::CallLoweringInfo CLI(DAG);
4761 CLI.setDebugLoc(dl)
4762 .setChain(Node->getOperand(Num: 0))
4763 .setLibCallee(
4764 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4765 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4766 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4767 ArgsList: std::move(Args));
4768
4769 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4770
4771 Results.push_back(Elt: CallResult.second);
4772 break;
4773 }
4774 // By default, atomic intrinsics are marked Legal and lowered. Targets
4775 // which don't support them directly, however, may want libcalls, in which
4776 // case they mark them Expand, and we get here.
4777 case ISD::ATOMIC_SWAP:
4778 case ISD::ATOMIC_LOAD_ADD:
4779 case ISD::ATOMIC_LOAD_SUB:
4780 case ISD::ATOMIC_LOAD_AND:
4781 case ISD::ATOMIC_LOAD_CLR:
4782 case ISD::ATOMIC_LOAD_OR:
4783 case ISD::ATOMIC_LOAD_XOR:
4784 case ISD::ATOMIC_LOAD_NAND:
4785 case ISD::ATOMIC_LOAD_MIN:
4786 case ISD::ATOMIC_LOAD_MAX:
4787 case ISD::ATOMIC_LOAD_UMIN:
4788 case ISD::ATOMIC_LOAD_UMAX:
4789 case ISD::ATOMIC_CMP_SWAP: {
4790 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4791 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4792 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4793 EVT RetVT = Node->getValueType(ResNo: 0);
4794 SmallVector<SDValue, 4> Ops;
4795 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported) {
4796 // If outline atomic available, prepare its arguments and expand.
4797 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4798 Ops.push_back(Elt: Node->getOperand(Num: 1));
4799
4800 } else {
4801 LC = RTLIB::getSYNC(Opc, VT);
4802 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4803 "Unexpected atomic op or value type!");
4804 // Arguments for expansion to sync libcall
4805 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4806 }
4807 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4808 Ops, CallOptions,
4809 dl: SDLoc(Node),
4810 Chain: Node->getOperand(Num: 0));
4811 Results.push_back(Elt: Tmp.first);
4812 Results.push_back(Elt: Tmp.second);
4813 break;
4814 }
4815 case ISD::TRAP: {
4816 // If this operation is not supported, lower it to 'abort()' call
4817 TargetLowering::ArgListTy Args;
4818 TargetLowering::CallLoweringInfo CLI(DAG);
4819 CLI.setDebugLoc(dl)
4820 .setChain(Node->getOperand(Num: 0))
4821 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4822 Target: DAG.getExternalSymbol(
4823 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4824 ArgsList: std::move(Args));
4825 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4826
4827 Results.push_back(Elt: CallResult.second);
4828 break;
4829 }
4830 case ISD::CLEAR_CACHE: {
4831 SDValue InputChain = Node->getOperand(Num: 0);
4832 SDValue StartVal = Node->getOperand(Num: 1);
4833 SDValue EndVal = Node->getOperand(Num: 2);
4834 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4835 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
4836 dl: SDLoc(Node), Chain: InputChain);
4837 Results.push_back(Elt: Tmp.second);
4838 break;
4839 }
4840 case ISD::FMINNUM:
4841 case ISD::STRICT_FMINNUM:
4842 ExpandFPLibCall(Node, LC: RTLIB::getFMIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4843 break;
4844 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4845 // libcall legalization for these nodes, but there is no default expasion for
4846 // these nodes either (see PR63267 for example).
4847 case ISD::FMAXNUM:
4848 case ISD::STRICT_FMAXNUM:
4849 ExpandFPLibCall(Node, LC: RTLIB::getFMAX(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4850 break;
4851 case ISD::FMINIMUMNUM:
4852 ExpandFPLibCall(Node, LC: RTLIB::getFMINIMUM_NUM(VT: Node->getSimpleValueType(ResNo: 0)),
4853 Results);
4854 break;
4855 case ISD::FMAXIMUMNUM:
4856 ExpandFPLibCall(Node, LC: RTLIB::getFMAXIMUM_NUM(VT: Node->getSimpleValueType(ResNo: 0)),
4857 Results);
4858 break;
4859 case ISD::FSQRT:
4860 case ISD::STRICT_FSQRT: {
4861 // FIXME: Probably should define fast to respect nan/inf and only be
4862 // approximate functions.
4863 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
4864 Call_F32: {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
4865 Call_F64: {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
4866 Call_F80: {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
4867 Call_F128: {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
4868 Call_PPCF128: {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
4869 Results);
4870 break;
4871 }
4872 case ISD::FCBRT:
4873 ExpandFPLibCall(Node, LC: RTLIB::getCBRT(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4874 break;
4875 case ISD::FSIN:
4876 case ISD::STRICT_FSIN:
4877 ExpandFPLibCall(Node, LC: RTLIB::getSIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4878 break;
4879 case ISD::FCOS:
4880 case ISD::STRICT_FCOS:
4881 ExpandFPLibCall(Node, LC: RTLIB::getCOS(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4882 break;
4883 case ISD::FTAN:
4884 case ISD::STRICT_FTAN:
4885 ExpandFPLibCall(Node, LC: RTLIB::getTAN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4886 break;
4887 case ISD::FASIN:
4888 case ISD::STRICT_FASIN:
4889 ExpandFPLibCall(Node, LC: RTLIB::getASIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4890 break;
4891 case ISD::FACOS:
4892 case ISD::STRICT_FACOS:
4893 ExpandFPLibCall(Node, LC: RTLIB::getACOS(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4894 break;
4895 case ISD::FATAN:
4896 case ISD::STRICT_FATAN:
4897 ExpandFPLibCall(Node, LC: RTLIB::getATAN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4898 break;
4899 case ISD::FATAN2:
4900 case ISD::STRICT_FATAN2:
4901 ExpandFPLibCall(Node, LC: RTLIB::getATAN2(VT: Node->getSimpleValueType(ResNo: 0)),
4902 Results);
4903 break;
4904 case ISD::FSINH:
4905 case ISD::STRICT_FSINH:
4906 ExpandFPLibCall(Node, LC: RTLIB::getSINH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4907 break;
4908 case ISD::FCOSH:
4909 case ISD::STRICT_FCOSH:
4910 ExpandFPLibCall(Node, LC: RTLIB::getCOSH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4911 break;
4912 case ISD::FTANH:
4913 case ISD::STRICT_FTANH:
4914 ExpandFPLibCall(Node, LC: RTLIB::getTANH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4915 break;
4916 case ISD::FSINCOS:
4917 case ISD::FSINCOSPI: {
4918 EVT VT = Node->getValueType(ResNo: 0);
4919
4920 if (Node->getOpcode() == ISD::FSINCOS) {
4921 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(VT);
4922 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
4923 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
4924 Results.push_back(Elt: Expanded);
4925 Results.push_back(Elt: Expanded.getValue(R: 1));
4926 break;
4927 }
4928 }
4929 }
4930
4931 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4932 ? RTLIB::getSINCOS(VT)
4933 : RTLIB::getSINCOSPI(VT);
4934 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);
4935 if (!Expanded) {
4936 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
4937 Node->getOperationName(G: &DAG));
4938 SDValue Poison = DAG.getPOISON(VT);
4939 Results.push_back(Elt: Poison);
4940 Results.push_back(Elt: Poison);
4941 }
4942
4943 break;
4944 }
4945 case ISD::FLOG:
4946 case ISD::STRICT_FLOG:
4947 ExpandFPLibCall(Node, LC: RTLIB::getLOG(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4948 break;
4949 case ISD::FLOG2:
4950 case ISD::STRICT_FLOG2:
4951 ExpandFPLibCall(Node, LC: RTLIB::getLOG2(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4952 break;
4953 case ISD::FLOG10:
4954 case ISD::STRICT_FLOG10:
4955 ExpandFPLibCall(Node, LC: RTLIB::getLOG10(VT: Node->getSimpleValueType(ResNo: 0)),
4956 Results);
4957 break;
4958 case ISD::FEXP:
4959 case ISD::STRICT_FEXP:
4960 ExpandFPLibCall(Node, LC: RTLIB::getEXP(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4961 break;
4962 case ISD::FEXP2:
4963 case ISD::STRICT_FEXP2:
4964 ExpandFPLibCall(Node, LC: RTLIB::getEXP2(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4965 break;
4966 case ISD::FEXP10:
4967 ExpandFPLibCall(Node, LC: RTLIB::getEXP10(VT: Node->getSimpleValueType(ResNo: 0)),
4968 Results);
4969 break;
4970 case ISD::FTRUNC:
4971 case ISD::STRICT_FTRUNC:
4972 ExpandFPLibCall(Node, LC: RTLIB::getTRUNC(VT: Node->getSimpleValueType(ResNo: 0)),
4973 Results);
4974 break;
4975 case ISD::FFLOOR:
4976 case ISD::STRICT_FFLOOR:
4977 ExpandFPLibCall(Node, LC: RTLIB::getFLOOR(VT: Node->getSimpleValueType(ResNo: 0)),
4978 Results);
4979 break;
4980 case ISD::FCEIL:
4981 case ISD::STRICT_FCEIL:
4982 ExpandFPLibCall(Node, LC: RTLIB::getCEIL(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4983 break;
4984 case ISD::FRINT:
4985 case ISD::STRICT_FRINT:
4986 ExpandFPLibCall(Node, LC: RTLIB::getRINT(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4987 break;
4988 case ISD::FNEARBYINT:
4989 case ISD::STRICT_FNEARBYINT:
4990 ExpandFPLibCall(Node, LC: RTLIB::getNEARBYINT(VT: Node->getSimpleValueType(ResNo: 0)),
4991 Results);
4992 break;
4993 case ISD::FROUND:
4994 case ISD::STRICT_FROUND:
4995 ExpandFPLibCall(Node, LC: RTLIB::getROUND(VT: Node->getSimpleValueType(ResNo: 0)),
4996 Results);
4997 break;
4998 case ISD::FROUNDEVEN:
4999 case ISD::STRICT_FROUNDEVEN:
5000 ExpandFPLibCall(Node, LC: RTLIB::getROUNDEVEN(VT: Node->getSimpleValueType(ResNo: 0)),
5001 Results);
5002 break;
5003 case ISD::FLDEXP:
5004 case ISD::STRICT_FLDEXP:
5005 ExpandFPLibCall(Node, LC: RTLIB::getLDEXP(VT: Node->getSimpleValueType(ResNo: 0)),
5006 Results);
5007 break;
5008 case ISD::FMODF:
5009 case ISD::FFREXP: {
5010 EVT VT = Node->getValueType(ResNo: 0);
5011 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
5012 : RTLIB::getFREXP(VT);
5013 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
5014 /*CallRetResNo=*/0);
5015 if (!Expanded) {
5016 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
5017 Node->getOperationName(G: &DAG));
5018 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
5019 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: I)));
5020 }
5021 break;
5022 }
5023 case ISD::FPOWI:
5024 case ISD::STRICT_FPOWI: {
5025 RTLIB::Libcall LC = RTLIB::getPOWI(VT: Node->getSimpleValueType(ResNo: 0));
5026 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
5027 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
5028 // Some targets don't have a powi libcall; use pow instead.
5029 if (Node->isStrictFPOpcode()) {
5030 SDValue Exponent =
5031 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
5032 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5033 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5034 SDValue FPOW =
5035 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
5036 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5037 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
5038 Results.push_back(Elt: FPOW);
5039 Results.push_back(Elt: FPOW.getValue(R: 1));
5040 } else {
5041 SDValue Exponent =
5042 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
5043 Operand: Node->getOperand(Num: 1));
5044 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
5045 VT: Node->getValueType(ResNo: 0),
5046 N1: Node->getOperand(Num: 0), N2: Exponent));
5047 }
5048 break;
5049 }
5050 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
5051 bool ExponentHasSizeOfInt =
5052 DAG.getLibInfo().getIntSize() ==
5053 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
5054 if (!ExponentHasSizeOfInt) {
5055 // If the exponent does not match with sizeof(int) a libcall to
5056 // RTLIB::POWI would use the wrong type for the argument.
5057 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
5058 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
5059 break;
5060 }
5061 ExpandFPLibCall(Node, LC, Results);
5062 break;
5063 }
5064 case ISD::FPOW:
5065 case ISD::STRICT_FPOW:
5066 ExpandFPLibCall(Node, LC: RTLIB::getPOW(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5067 break;
5068 case ISD::LROUND:
5069 case ISD::STRICT_LROUND:
5070 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
5071 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
5072 Call_F128: RTLIB::LROUND_F128,
5073 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
5074 break;
5075 case ISD::LLROUND:
5076 case ISD::STRICT_LLROUND:
5077 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
5078 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
5079 Call_F128: RTLIB::LLROUND_F128,
5080 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
5081 break;
5082 case ISD::LRINT:
5083 case ISD::STRICT_LRINT:
5084 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
5085 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
5086 Call_F128: RTLIB::LRINT_F128,
5087 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
5088 break;
5089 case ISD::LLRINT:
5090 case ISD::STRICT_LLRINT:
5091 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
5092 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
5093 Call_F128: RTLIB::LLRINT_F128,
5094 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
5095 break;
5096 case ISD::FDIV:
5097 case ISD::STRICT_FDIV: {
5098 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5099 Call_F32: {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5100 Call_F64: {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5101 Call_F80: {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5102 Call_F128: {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5103 Call_PPCF128: {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5104 break;
5105 }
5106 case ISD::FREM:
5107 case ISD::STRICT_FREM:
5108 ExpandFPLibCall(Node, LC: RTLIB::getREM(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5109 break;
5110 case ISD::FMA:
5111 case ISD::STRICT_FMA:
5112 ExpandFPLibCall(Node, LC: RTLIB::getFMA(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5113 break;
5114 case ISD::FADD:
5115 case ISD::STRICT_FADD: {
5116 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5117 Call_F32: {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5118 Call_F64: {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5119 Call_F80: {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5120 Call_F128: {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5121 Call_PPCF128: {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5122 break;
5123 }
5124 case ISD::FMUL:
5125 case ISD::STRICT_FMUL: {
5126 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5127 Call_F32: {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5128 Call_F64: {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5129 Call_F80: {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5130 Call_F128: {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5131 Call_PPCF128: {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5132 break;
5133 }
5134 case ISD::FP16_TO_FP:
5135 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5136 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
5137 }
5138 break;
5139 case ISD::STRICT_BF16_TO_FP:
5140 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5141 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5142 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
5143 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5144 Results.push_back(Elt: Tmp.first);
5145 Results.push_back(Elt: Tmp.second);
5146 }
5147 break;
5148 case ISD::STRICT_FP16_TO_FP: {
5149 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5150 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5151 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
5152 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5153 Results.push_back(Elt: Tmp.first);
5154 Results.push_back(Elt: Tmp.second);
5155 }
5156 break;
5157 }
5158 case ISD::FP_TO_FP16: {
5159 RTLIB::Libcall LC =
5160 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
5161 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5162 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5163 break;
5164 }
5165 case ISD::FP_TO_BF16: {
5166 RTLIB::Libcall LC =
5167 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
5168 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5169 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5170 break;
5171 }
5172 case ISD::STRICT_SINT_TO_FP:
5173 case ISD::STRICT_UINT_TO_FP:
5174 case ISD::SINT_TO_FP:
5175 case ISD::UINT_TO_FP: {
5176 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
5177 bool IsStrict = Node->isStrictFPOpcode();
5178 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
5179 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
5180 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
5181 EVT RVT = Node->getValueType(ResNo: 0);
5182 EVT NVT = EVT();
5183 SDLoc dl(Node);
5184
5185 // Even if the input is legal, no libcall may exactly match, eg. we don't
5186 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
5187 // eg: i13 -> fp. Then, look for an appropriate libcall.
5188 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5189 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5190 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5191 ++t) {
5192 NVT = (MVT::SimpleValueType)t;
5193 // The source needs to big enough to hold the operand.
5194 if (NVT.bitsGE(VT: SVT))
5195 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
5196 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
5197 }
5198 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5199
5200 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5201 // Sign/zero extend the argument if the libcall takes a larger type.
5202 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
5203 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
5204 CallOptions.setIsSigned(Signed);
5205 std::pair<SDValue, SDValue> Tmp =
5206 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
5207 Results.push_back(Elt: Tmp.first);
5208 if (IsStrict)
5209 Results.push_back(Elt: Tmp.second);
5210 break;
5211 }
5212 case ISD::FP_TO_SINT:
5213 case ISD::FP_TO_UINT:
5214 case ISD::STRICT_FP_TO_SINT:
5215 case ISD::STRICT_FP_TO_UINT: {
5216 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
5217 bool IsStrict = Node->isStrictFPOpcode();
5218 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
5219 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
5220
5221 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5222 EVT SVT = Op.getValueType();
5223 EVT RVT = Node->getValueType(ResNo: 0);
5224 EVT NVT = EVT();
5225 SDLoc dl(Node);
5226
5227 // Even if the result is legal, no libcall may exactly match, eg. we don't
5228 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
5229 // eg: fp -> i32. Then, look for an appropriate libcall.
5230 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5231 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5232 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5233 ++IntVT) {
5234 NVT = (MVT::SimpleValueType)IntVT;
5235 // The type needs to big enough to hold the result.
5236 if (NVT.bitsGE(VT: RVT))
5237 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
5238 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
5239 }
5240 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5241
5242 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5243 std::pair<SDValue, SDValue> Tmp =
5244 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
5245
5246 // Truncate the result if the libcall returns a larger type.
5247 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
5248 if (IsStrict)
5249 Results.push_back(Elt: Tmp.second);
5250 break;
5251 }
5252
5253 case ISD::FP_ROUND:
5254 case ISD::STRICT_FP_ROUND: {
5255 // X = FP_ROUND(Y, TRUNC)
5256 // TRUNC is a flag, which is always an integer that is zero or one.
5257 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5258 // is known to not change the value of Y.
5259 // We can only expand it into libcall if the TRUNC is 0.
5260 bool IsStrict = Node->isStrictFPOpcode();
5261 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5262 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5263 EVT VT = Node->getValueType(ResNo: 0);
5264 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5265 "Unable to expand as libcall if it is not normal rounding");
5266
5267 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5268 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5269
5270 std::pair<SDValue, SDValue> Tmp =
5271 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5272 Results.push_back(Elt: Tmp.first);
5273 if (IsStrict)
5274 Results.push_back(Elt: Tmp.second);
5275 break;
5276 }
5277 case ISD::FP_EXTEND: {
5278 Results.push_back(
5279 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5280 RetVT: Node->getValueType(ResNo: 0)),
5281 Node, isSigned: false).first);
5282 break;
5283 }
5284 case ISD::STRICT_FP_EXTEND:
5285 case ISD::STRICT_FP_TO_FP16:
5286 case ISD::STRICT_FP_TO_BF16: {
5287 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5288 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5289 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5290 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5291 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5292 else
5293 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5294 RetVT: Node->getValueType(ResNo: 0));
5295
5296 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5297
5298 std::pair<SDValue, SDValue> Tmp =
5299 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5300 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5301 Results.push_back(Elt: Tmp.first);
5302 Results.push_back(Elt: Tmp.second);
5303 break;
5304 }
5305 case ISD::FSUB:
5306 case ISD::STRICT_FSUB: {
5307 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5308 Call_F32: {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5309 Call_F64: {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5310 Call_F80: {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5311 Call_F128: {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5312 Call_PPCF128: {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5313 break;
5314 }
5315 case ISD::SREM:
5316 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5317 Call_I8: RTLIB::SREM_I8,
5318 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5319 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5320 break;
5321 case ISD::UREM:
5322 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5323 Call_I8: RTLIB::UREM_I8,
5324 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5325 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5326 break;
5327 case ISD::SDIV:
5328 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5329 Call_I8: RTLIB::SDIV_I8,
5330 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5331 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5332 break;
5333 case ISD::UDIV:
5334 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5335 Call_I8: RTLIB::UDIV_I8,
5336 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5337 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5338 break;
5339 case ISD::SDIVREM:
5340 case ISD::UDIVREM:
5341 // Expand into divrem libcall
5342 ExpandDivRemLibCall(Node, Results);
5343 break;
5344 case ISD::MUL:
5345 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5346 Call_I8: RTLIB::MUL_I8,
5347 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5348 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5349 break;
5350 case ISD::CTLZ_ZERO_POISON:
5351 Results.push_back(Elt: ExpandBitCountingLibCall(
5352 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5353 break;
5354 case ISD::CTPOP:
5355 Results.push_back(Elt: ExpandBitCountingLibCall(
5356 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5357 break;
5358 case ISD::RESET_FPENV: {
5359 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5360 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5361 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5362 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5363 SDValue Chain = Node->getOperand(Num: 0);
5364 Results.push_back(
5365 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
5366 break;
5367 }
5368 case ISD::GET_FPENV_MEM: {
5369 SDValue Chain = Node->getOperand(Num: 0);
5370 SDValue EnvPtr = Node->getOperand(Num: 1);
5371 Results.push_back(
5372 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5373 break;
5374 }
5375 case ISD::SET_FPENV_MEM: {
5376 SDValue Chain = Node->getOperand(Num: 0);
5377 SDValue EnvPtr = Node->getOperand(Num: 1);
5378 Results.push_back(
5379 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5380 break;
5381 }
5382 case ISD::GET_FPMODE: {
5383 // Call fegetmode, which saves control modes into a stack slot. Then load
5384 // the value to return from the stack.
5385 EVT ModeVT = Node->getValueType(ResNo: 0);
5386 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5387 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5388 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5389 InChain: Node->getOperand(Num: 0), DLoc: dl);
5390 SDValue LdInst = DAG.getLoad(
5391 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5392 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5393 Results.push_back(Elt: LdInst);
5394 Results.push_back(Elt: LdInst.getValue(R: 1));
5395 break;
5396 }
5397 case ISD::SET_FPMODE: {
5398 // Move control modes to stack slot and then call fesetmode with the pointer
5399 // to the slot as argument.
5400 SDValue Mode = Node->getOperand(Num: 1);
5401 EVT ModeVT = Mode.getValueType();
5402 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5403 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5404 SDValue StInst = DAG.getStore(
5405 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5406 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5407 Results.push_back(
5408 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
5409 break;
5410 }
5411 case ISD::RESET_FPMODE: {
5412 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5413 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5414 // target must provide custom lowering.
5415 const DataLayout &DL = DAG.getDataLayout();
5416 EVT PtrTy = TLI.getPointerTy(DL);
5417 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5418 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5419 InChain: Node->getOperand(Num: 0), DLoc: dl));
5420 break;
5421 }
5422 }
5423
5424 // Replace the original node with the legalized result.
5425 if (!Results.empty()) {
5426 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5427 ReplaceNode(Old: Node, New: Results.data());
5428 } else
5429 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5430}
5431
5432// Determine the vector type to use in place of an original scalar element when
5433// promoting equally sized vectors.
5434static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5435 MVT EltVT, MVT NewEltVT) {
5436 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5437 MVT MidVT = OldEltsPerNewElt == 1
5438 ? NewEltVT
5439 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5440 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5441 return MidVT;
5442}
5443
5444void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5445 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5446 SmallVector<SDValue, 8> Results;
5447 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5448 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5449 Node->getOpcode() == ISD::SINT_TO_FP || Node->getOpcode() == ISD::SETCC ||
5450 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5451 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5452 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5453 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5454 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5455 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM ||
5456 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUMNUM ||
5457 Node->getOpcode() == ISD::VECREDUCE_FMINIMUMNUM) {
5458 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5459 }
5460 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5461 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5462 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5463 Node->getOpcode() == ISD::STRICT_FSETCC ||
5464 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5465 Node->getOpcode() == ISD::STRICT_LRINT ||
5466 Node->getOpcode() == ISD::STRICT_LLRINT ||
5467 Node->getOpcode() == ISD::STRICT_LROUND ||
5468 Node->getOpcode() == ISD::STRICT_LLROUND ||
5469 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5470 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5471 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5472 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5473 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5474 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5475 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5476 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5477 if (Node->getOpcode() == ISD::BR_CC ||
5478 Node->getOpcode() == ISD::SELECT_CC)
5479 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5480 // Preserve fast math flags
5481 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5482 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5483 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5484 SDLoc dl(Node);
5485 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5486 switch (Node->getOpcode()) {
5487 case ISD::CTTZ:
5488 case ISD::CTTZ_ZERO_POISON:
5489 case ISD::CTLZ:
5490 case ISD::CTPOP: {
5491 // Zero extend the argument unless its cttz, then use any_extend.
5492 if (Node->getOpcode() == ISD::CTTZ ||
5493 Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
5494 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5495 else
5496 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5497
5498 unsigned NewOpc = Node->getOpcode();
5499 if (NewOpc == ISD::CTTZ) {
5500 // The count is the same in the promoted type except if the original
5501 // value was zero. This can be handled by setting the bit just off
5502 // the top of the original type.
5503 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5504 BitNo: OVT.getSizeInBits());
5505 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5506 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5507 NewOpc = ISD::CTTZ_ZERO_POISON;
5508 }
5509 // Perform the larger operation. For CTPOP and CTTZ_ZERO_POISON, this is
5510 // already the correct result.
5511 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5512 if (NewOpc == ISD::CTLZ) {
5513 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5514 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5515 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5516 OVT.getSizeInBits(), DL: dl, VT: NVT));
5517 }
5518 Results.push_back(
5519 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5520 break;
5521 }
5522 case ISD::CTLZ_ZERO_POISON: {
5523 // We know that the argument is unlikely to be zero, hence we can take a
5524 // different approach as compared to ISD::CTLZ
5525
5526 // Any Extend the argument
5527 auto AnyExtendedNode =
5528 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5529
5530 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5531 auto ShiftConstant = DAG.getShiftAmountConstant(
5532 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5533 auto LeftShiftResult =
5534 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5535
5536 // Perform the larger operation
5537 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5538 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5539 break;
5540 }
5541 case ISD::PEXT: {
5542 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5543 Tmp2 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5544 Tmp1 = DAG.getNode(Opcode: ISD::PEXT, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5545 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5546 break;
5547 }
5548 case ISD::PDEP: {
5549 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5550 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5551 Tmp1 = DAG.getNode(Opcode: ISD::PDEP, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5552 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5553 break;
5554 }
5555 case ISD::BITREVERSE:
5556 case ISD::BSWAP: {
5557 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5558 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5559 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5560 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5561 N2: DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl));
5562
5563 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5564 break;
5565 }
5566 case ISD::FP_TO_UINT:
5567 case ISD::STRICT_FP_TO_UINT:
5568 case ISD::FP_TO_SINT:
5569 case ISD::STRICT_FP_TO_SINT:
5570 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5571 break;
5572 case ISD::FP_TO_UINT_SAT:
5573 case ISD::FP_TO_SINT_SAT:
5574 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5575 break;
5576 case ISD::UINT_TO_FP:
5577 case ISD::STRICT_UINT_TO_FP:
5578 case ISD::SINT_TO_FP:
5579 case ISD::STRICT_SINT_TO_FP:
5580 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5581 break;
5582 case ISD::VAARG: {
5583 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5584 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5585
5586 unsigned TruncOp;
5587 if (OVT.isVector()) {
5588 TruncOp = ISD::BITCAST;
5589 } else {
5590 assert(OVT.isInteger()
5591 && "VAARG promotion is supported only for vectors or integer types");
5592 TruncOp = ISD::TRUNCATE;
5593 }
5594
5595 // Perform the larger operation, then convert back
5596 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5597 Align: Node->getConstantOperandVal(Num: 3));
5598 Chain = Tmp1.getValue(R: 1);
5599
5600 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5601
5602 // Modified the chain result - switch anything that used the old chain to
5603 // use the new one.
5604 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5605 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5606 if (UpdatedNodes) {
5607 UpdatedNodes->insert(X: Tmp2.getNode());
5608 UpdatedNodes->insert(X: Chain.getNode());
5609 }
5610 ReplacedNode(N: Node);
5611 break;
5612 }
5613 case ISD::MUL:
5614 case ISD::SDIV:
5615 case ISD::SREM:
5616 case ISD::UDIV:
5617 case ISD::UREM:
5618 case ISD::SMIN:
5619 case ISD::SMAX:
5620 case ISD::UMIN:
5621 case ISD::UMAX:
5622 case ISD::AND:
5623 case ISD::OR:
5624 case ISD::XOR: {
5625 unsigned ExtOp, TruncOp;
5626 if (OVT.isVector()) {
5627 ExtOp = ISD::BITCAST;
5628 TruncOp = ISD::BITCAST;
5629 } else {
5630 assert(OVT.isInteger() && "Cannot promote logic operation");
5631
5632 switch (Node->getOpcode()) {
5633 default:
5634 ExtOp = ISD::ANY_EXTEND;
5635 break;
5636 case ISD::SDIV:
5637 case ISD::SREM:
5638 case ISD::SMIN:
5639 case ISD::SMAX:
5640 ExtOp = ISD::SIGN_EXTEND;
5641 break;
5642 case ISD::UDIV:
5643 case ISD::UREM:
5644 ExtOp = ISD::ZERO_EXTEND;
5645 break;
5646 case ISD::UMIN:
5647 case ISD::UMAX:
5648 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5649 ExtOp = ISD::SIGN_EXTEND;
5650 else
5651 ExtOp = ISD::ZERO_EXTEND;
5652 break;
5653 }
5654 TruncOp = ISD::TRUNCATE;
5655 }
5656 // Promote each of the values to the new type.
5657 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5658 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5659 // Perform the larger operation, then convert back
5660 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5661 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5662 break;
5663 }
5664 case ISD::UMUL_LOHI:
5665 case ISD::SMUL_LOHI: {
5666 // Promote to a multiply in a wider integer type.
5667 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5668 : ISD::SIGN_EXTEND;
5669 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5670 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5671 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5672
5673 unsigned OriginalSize = OVT.getScalarSizeInBits();
5674 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5675 N2: DAG.getShiftAmountConstant(Val: OriginalSize, VT: NVT, DL: dl));
5676 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5677 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5678 break;
5679 }
5680 case ISD::SELECT: {
5681 unsigned ExtOp, TruncOp;
5682 if (Node->getValueType(ResNo: 0).isVector() ||
5683 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5684 ExtOp = ISD::BITCAST;
5685 TruncOp = ISD::BITCAST;
5686 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5687 ExtOp = ISD::ANY_EXTEND;
5688 TruncOp = ISD::TRUNCATE;
5689 } else {
5690 ExtOp = ISD::FP_EXTEND;
5691 TruncOp = ISD::FP_ROUND;
5692 }
5693 Tmp1 = Node->getOperand(Num: 0);
5694 // Promote each of the values to the new type.
5695 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5696 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5697 // Perform the larger operation, then round down.
5698 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5699 if (TruncOp != ISD::FP_ROUND)
5700 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5701 else
5702 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5703 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5704 Results.push_back(Elt: Tmp1);
5705 break;
5706 }
5707 case ISD::VECTOR_SHUFFLE: {
5708 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5709
5710 // Cast the two input vectors.
5711 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5712 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5713
5714 // Convert the shuffle mask to the right # elements.
5715 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5716 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5717 Results.push_back(Elt: Tmp1);
5718 break;
5719 }
5720 case ISD::VECTOR_SPLICE_LEFT:
5721 case ISD::VECTOR_SPLICE_RIGHT: {
5722 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5723 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5724 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5725 N3: Node->getOperand(Num: 2));
5726 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5727 break;
5728 }
5729 case ISD::SELECT_CC: {
5730 SDValue Cond = Node->getOperand(Num: 4);
5731 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5732 // Type of the comparison operands.
5733 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5734 assert(CVT == OVT && "not handled");
5735
5736 unsigned ExtOp = ISD::FP_EXTEND;
5737 if (NVT.isInteger()) {
5738 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5739 }
5740
5741 // Promote the comparison operands, if needed.
5742 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5743 Tmp1 = Node->getOperand(Num: 0);
5744 Tmp2 = Node->getOperand(Num: 1);
5745 } else {
5746 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5747 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5748 }
5749 // Cast the true/false operands.
5750 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5751 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5752
5753 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5754 Flags: Node->getFlags());
5755
5756 // Cast the result back to the original type.
5757 if (ExtOp != ISD::FP_EXTEND)
5758 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5759 else
5760 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5761 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5762
5763 Results.push_back(Elt: Tmp1);
5764 break;
5765 }
5766 case ISD::SETCC:
5767 case ISD::STRICT_FSETCC:
5768 case ISD::STRICT_FSETCCS: {
5769 unsigned ExtOp = ISD::FP_EXTEND;
5770 if (NVT.isInteger()) {
5771 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5772 if (isSignedIntSetCC(Code: CCCode) ||
5773 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5774 ExtOp = ISD::SIGN_EXTEND;
5775 else
5776 ExtOp = ISD::ZERO_EXTEND;
5777 }
5778 if (Node->isStrictFPOpcode()) {
5779 SDValue InChain = Node->getOperand(Num: 0);
5780 std::tie(args&: Tmp1, args: std::ignore) =
5781 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5782 std::tie(args&: Tmp2, args: std::ignore) =
5783 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5784 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5785 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5786 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
5787 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5788 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5789 Flags: Node->getFlags()));
5790 Results.push_back(Elt: Results.back().getValue(R: 1));
5791 break;
5792 }
5793 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5794 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5795 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5796 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5797 break;
5798 }
5799 case ISD::BR_CC: {
5800 unsigned ExtOp = ISD::FP_EXTEND;
5801 if (NVT.isInteger()) {
5802 ISD::CondCode CCCode =
5803 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5804 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5805 }
5806 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5807 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5808 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5809 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5810 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5811 break;
5812 }
5813 case ISD::FADD:
5814 case ISD::FSUB:
5815 case ISD::FMUL:
5816 case ISD::FDIV:
5817 case ISD::FREM:
5818 case ISD::FMINNUM:
5819 case ISD::FMAXNUM:
5820 case ISD::FMINIMUM:
5821 case ISD::FMAXIMUM:
5822 case ISD::FMINIMUMNUM:
5823 case ISD::FMAXIMUMNUM:
5824 case ISD::FPOW:
5825 case ISD::FATAN2:
5826 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5827 if (!OVT.isVector() && NVT.isVector() &&
5828 NVT.getVectorElementType() == OVT) {
5829 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5830 Tmp2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5831 Tmp3 =
5832 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, Flags: Node->getFlags());
5833 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Tmp3,
5834 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5835 break;
5836 }
5837 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5838 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5839 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5840 Results.push_back(
5841 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5842 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5843 break;
5844
5845 case ISD::STRICT_FMINIMUM:
5846 case ISD::STRICT_FMAXIMUM: {
5847 SDValue InChain = Node->getOperand(Num: 0);
5848 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
5849 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5850 N2: Node->getOperand(Num: 1));
5851 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5852 N2: Node->getOperand(Num: 2));
5853 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
5854 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
5855 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
5856 N1: InChain, N2: Tmp3,
5857 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5858 Results.push_back(Elt: Tmp4);
5859 Results.push_back(Elt: Tmp4.getValue(R: 1));
5860 break;
5861 }
5862
5863 case ISD::STRICT_FADD:
5864 case ISD::STRICT_FSUB:
5865 case ISD::STRICT_FMUL:
5866 case ISD::STRICT_FDIV:
5867 case ISD::STRICT_FMINNUM:
5868 case ISD::STRICT_FMAXNUM:
5869 case ISD::STRICT_FREM:
5870 case ISD::STRICT_FPOW:
5871 case ISD::STRICT_FATAN2:
5872 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5873 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5874 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5875 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5876 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5877 N2: Tmp2.getValue(R: 1));
5878 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5879 Ops: {Tmp3, Tmp1, Tmp2});
5880 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5881 Ops: {Tmp1.getValue(R: 1), Tmp1,
5882 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5883 Results.push_back(Elt: Tmp1);
5884 Results.push_back(Elt: Tmp1.getValue(R: 1));
5885 break;
5886 case ISD::FMA:
5887 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5888 if (!OVT.isVector() && NVT.isVector() &&
5889 NVT.getVectorElementType() == OVT) {
5890 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5891 Tmp2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5892 Tmp3 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5893 SDValue Result = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3,
5894 Flags: Node->getFlags());
5895 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Result,
5896 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5897 break;
5898 }
5899 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5900 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5901 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5902 Results.push_back(
5903 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5904 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5905 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5906 break;
5907 case ISD::STRICT_FMA:
5908 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5909 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5910 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5911 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5912 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5913 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
5914 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5915 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
5916 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5917 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
5918 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5919 Ops: {Tmp4.getValue(R: 1), Tmp4,
5920 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5921 Results.push_back(Elt: Tmp4);
5922 Results.push_back(Elt: Tmp4.getValue(R: 1));
5923 break;
5924 case ISD::FCOPYSIGN:
5925 case ISD::FLDEXP:
5926 case ISD::FPOWI: {
5927 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5928 Tmp2 = Node->getOperand(Num: 1);
5929 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5930
5931 // fcopysign doesn't change anything but the sign bit, so
5932 // (fp_round (fcopysign (fpext a), b))
5933 // is as precise as
5934 // (fp_round (fpext a))
5935 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5936 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5937 Results.push_back(
5938 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5939 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5940 break;
5941 }
5942 case ISD::STRICT_FLDEXP: {
5943 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5944 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5945 Tmp2 = Node->getOperand(Num: 2);
5946 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
5947 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
5948 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5949 Ops: {Tmp3.getValue(R: 1), Tmp3,
5950 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5951 Results.push_back(Elt: Tmp4);
5952 Results.push_back(Elt: Tmp4.getValue(R: 1));
5953 break;
5954 }
5955 case ISD::STRICT_FPOWI:
5956 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5957 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5958 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5959 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
5960 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5961 Ops: {Tmp2.getValue(R: 1), Tmp2,
5962 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5963 Results.push_back(Elt: Tmp3);
5964 Results.push_back(Elt: Tmp3.getValue(R: 1));
5965 break;
5966 case ISD::FFREXP: {
5967 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5968 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5969
5970 Results.push_back(
5971 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5972 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5973
5974 Results.push_back(Elt: Tmp2.getValue(R: 1));
5975 break;
5976 }
5977 case ISD::FMODF:
5978 case ISD::FSINCOS:
5979 case ISD::FSINCOSPI: {
5980 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5981 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
5982 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
5983 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5984 Results.push_back(
5985 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
5986 break;
5987 }
5988 case ISD::FFLOOR:
5989 case ISD::FCEIL:
5990 case ISD::FRINT:
5991 case ISD::FNEARBYINT:
5992 case ISD::FROUND:
5993 case ISD::FROUNDEVEN:
5994 case ISD::FTRUNC:
5995 case ISD::FNEG:
5996 case ISD::FSQRT:
5997 case ISD::FSIN:
5998 case ISD::FCOS:
5999 case ISD::FTAN:
6000 case ISD::FASIN:
6001 case ISD::FACOS:
6002 case ISD::FATAN:
6003 case ISD::FSINH:
6004 case ISD::FCOSH:
6005 case ISD::FTANH:
6006 case ISD::FLOG:
6007 case ISD::FLOG2:
6008 case ISD::FLOG10:
6009 case ISD::FABS:
6010 case ISD::FEXP:
6011 case ISD::FEXP2:
6012 case ISD::FEXP10:
6013 case ISD::FCANONICALIZE:
6014 // Promote scalar operations to vector using SCALAR_TO_VECTOR
6015 if (!OVT.isVector() && NVT.isVector() &&
6016 NVT.getVectorElementType() == OVT) {
6017 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6018 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1, Flags: Node->getFlags());
6019 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Tmp2,
6020 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
6021 break;
6022 }
6023 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6024 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6025 Results.push_back(
6026 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6027 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6028 break;
6029 case ISD::STRICT_FFLOOR:
6030 case ISD::STRICT_FCEIL:
6031 case ISD::STRICT_FRINT:
6032 case ISD::STRICT_FNEARBYINT:
6033 case ISD::STRICT_FROUND:
6034 case ISD::STRICT_FROUNDEVEN:
6035 case ISD::STRICT_FTRUNC:
6036 case ISD::STRICT_FSQRT:
6037 case ISD::STRICT_FSIN:
6038 case ISD::STRICT_FCOS:
6039 case ISD::STRICT_FTAN:
6040 case ISD::STRICT_FASIN:
6041 case ISD::STRICT_FACOS:
6042 case ISD::STRICT_FATAN:
6043 case ISD::STRICT_FSINH:
6044 case ISD::STRICT_FCOSH:
6045 case ISD::STRICT_FTANH:
6046 case ISD::STRICT_FLOG:
6047 case ISD::STRICT_FLOG2:
6048 case ISD::STRICT_FLOG10:
6049 case ISD::STRICT_FEXP:
6050 case ISD::STRICT_FEXP2:
6051 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6052 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6053 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6054 Ops: {Tmp1.getValue(R: 1), Tmp1});
6055 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6056 Ops: {Tmp2.getValue(R: 1), Tmp2,
6057 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6058 Results.push_back(Elt: Tmp3);
6059 Results.push_back(Elt: Tmp3.getValue(R: 1));
6060 break;
6061 case ISD::LLROUND:
6062 case ISD::LROUND:
6063 case ISD::LRINT:
6064 case ISD::LLRINT:
6065 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6066 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
6067 Results.push_back(Elt: Tmp2);
6068 break;
6069 case ISD::STRICT_LLROUND:
6070 case ISD::STRICT_LROUND:
6071 case ISD::STRICT_LRINT:
6072 case ISD::STRICT_LLRINT:
6073 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6074 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6075 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6076 Ops: {Tmp1.getValue(R: 1), Tmp1});
6077 Results.push_back(Elt: Tmp2);
6078 Results.push_back(Elt: Tmp2.getValue(R: 1));
6079 break;
6080 case ISD::BUILD_VECTOR: {
6081 MVT EltVT = OVT.getVectorElementType();
6082 MVT NewEltVT = NVT.getVectorElementType();
6083
6084 // Handle bitcasts to a different vector type with the same total bit size
6085 //
6086 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
6087 // =>
6088 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
6089
6090 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6091 "Invalid promote type for build_vector");
6092 assert(NewEltVT.bitsLE(EltVT) && "not handled");
6093
6094 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6095
6096 SmallVector<SDValue, 8> NewOps;
6097 for (const SDValue &Op : Node->op_values())
6098 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
6099
6100 SDLoc SL(Node);
6101 SDValue Concat =
6102 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
6103 DL: SL, VT: NVT, Ops: NewOps);
6104 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6105 Results.push_back(Elt: CvtVec);
6106 break;
6107 }
6108 case ISD::EXTRACT_VECTOR_ELT: {
6109 MVT EltVT = OVT.getVectorElementType();
6110 MVT NewEltVT = NVT.getVectorElementType();
6111
6112 // Handle bitcasts to a different vector type with the same total bit size.
6113 //
6114 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
6115 // =>
6116 // v4i32:castx = bitcast x:v2i64
6117 //
6118 // i64 = bitcast
6119 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
6120 // (i32 (extract_vector_elt castx, (2 * y + 1)))
6121 //
6122
6123 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6124 "Invalid promote type for extract_vector_elt");
6125 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6126
6127 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6128 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6129
6130 SDValue Idx = Node->getOperand(Num: 1);
6131 EVT IdxVT = Idx.getValueType();
6132 SDLoc SL(Node);
6133 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
6134 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6135
6136 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6137
6138 SmallVector<SDValue, 8> NewOps;
6139 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6140 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6141 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6142
6143 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6144 N1: CastVec, N2: TmpIdx);
6145 NewOps.push_back(Elt);
6146 }
6147
6148 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
6149 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
6150 break;
6151 }
6152 case ISD::INSERT_VECTOR_ELT: {
6153 MVT EltVT = OVT.getVectorElementType();
6154 MVT NewEltVT = NVT.getVectorElementType();
6155
6156 // Handle bitcasts to a different vector type with the same total bit size
6157 //
6158 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
6159 // =>
6160 // v4i32:castx = bitcast x:v2i64
6161 // v2i32:casty = bitcast y:i64
6162 //
6163 // v2i64 = bitcast
6164 // (v4i32 insert_vector_elt
6165 // (v4i32 insert_vector_elt v4i32:castx,
6166 // (extract_vector_elt casty, 0), 2 * z),
6167 // (extract_vector_elt casty, 1), (2 * z + 1))
6168
6169 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6170 "Invalid promote type for insert_vector_elt");
6171 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6172
6173 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6174 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6175
6176 SDValue Val = Node->getOperand(Num: 1);
6177 SDValue Idx = Node->getOperand(Num: 2);
6178 EVT IdxVT = Idx.getValueType();
6179 SDLoc SL(Node);
6180
6181 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
6182 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6183
6184 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6185 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6186
6187 SDValue NewVec = CastVec;
6188 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6189 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6190 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6191
6192 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6193 N1: CastVal, N2: IdxOffset);
6194
6195 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
6196 N1: NewVec, N2: Elt, N3: InEltIdx);
6197 }
6198
6199 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
6200 break;
6201 }
6202 case ISD::SCALAR_TO_VECTOR: {
6203 MVT EltVT = OVT.getVectorElementType();
6204 MVT NewEltVT = NVT.getVectorElementType();
6205
6206 // Handle bitcasts to different vector type with the same total bit size.
6207 //
6208 // e.g. v2i64 = scalar_to_vector x:i64
6209 // =>
6210 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
6211 //
6212
6213 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6214 SDValue Val = Node->getOperand(Num: 0);
6215 SDLoc SL(Node);
6216
6217 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6218 SDValue Undef = DAG.getUNDEF(VT: MidVT);
6219
6220 SmallVector<SDValue, 8> NewElts;
6221 NewElts.push_back(Elt: CastVal);
6222 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
6223 NewElts.push_back(Elt: Undef);
6224
6225 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
6226 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6227 Results.push_back(Elt: CvtVec);
6228 break;
6229 }
6230 case ISD::ATOMIC_SWAP:
6231 case ISD::ATOMIC_STORE: {
6232 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6233 SDLoc SL(Node);
6234 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
6235 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6236 "unexpected promotion type");
6237 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6238 "unexpected atomic_swap with illegal type");
6239
6240 SDValue Op0 = AM->getBasePtr();
6241 SDValue Op1 = CastVal;
6242
6243 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
6244 // but really it should merge with ISD::STORE.
6245 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6246 std::swap(a&: Op0, b&: Op1);
6247
6248 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
6249 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
6250
6251 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6252 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6253 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6254 } else
6255 Results.push_back(Elt: NewAtomic);
6256 break;
6257 }
6258 case ISD::ATOMIC_LOAD: {
6259 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6260 SDLoc SL(Node);
6261 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6262 "unexpected promotion type");
6263 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6264 "unexpected atomic_load with illegal type");
6265
6266 SDValue NewAtomic =
6267 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
6268 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
6269 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6270 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6271 break;
6272 }
6273 case ISD::SPLAT_VECTOR: {
6274 SDValue Scalar = Node->getOperand(Num: 0);
6275 MVT ScalarType = Scalar.getSimpleValueType();
6276 MVT NewScalarType = NVT.getVectorElementType();
6277 if (ScalarType.isInteger()) {
6278 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6279 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6280 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
6281 break;
6282 }
6283 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6284 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6285 Results.push_back(
6286 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6287 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6288 break;
6289 }
6290 case ISD::VECREDUCE_FMAX:
6291 case ISD::VECREDUCE_FMIN:
6292 case ISD::VECREDUCE_FMAXIMUM:
6293 case ISD::VECREDUCE_FMINIMUM:
6294 case ISD::VECREDUCE_FMAXIMUMNUM:
6295 case ISD::VECREDUCE_FMINIMUMNUM:
6296 case ISD::VP_REDUCE_FMAX:
6297 case ISD::VP_REDUCE_FMIN:
6298 case ISD::VP_REDUCE_FMAXIMUM:
6299 case ISD::VP_REDUCE_FMINIMUM:
6300 Results.push_back(Elt: PromoteReduction(Node));
6301 break;
6302 }
6303
6304 // Replace the original node with the legalized result.
6305 if (!Results.empty()) {
6306 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6307 ReplaceNode(Old: Node, New: Results.data());
6308 } else
6309 LLVM_DEBUG(dbgs() << "Could not promote node\n");
6310}
6311
6312/// This is the entry point for the file.
6313void SelectionDAG::Legalize() {
6314 AssignTopologicalOrder();
6315
6316 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6317 // Use a delete listener to remove nodes which were deleted during
6318 // legalization from LegalizeNodes. This is needed to handle the situation
6319 // where a new node is allocated by the object pool to the same address of a
6320 // previously deleted node.
6321 DAGNodeDeletedListener DeleteListener(
6322 *this,
6323 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
6324
6325 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6326
6327 // Visit all the nodes. We start in topological order, so that we see
6328 // nodes with their original operands intact. Legalization can produce
6329 // new nodes which may themselves need to be legalized. Iterate until all
6330 // nodes have been legalized.
6331 while (true) {
6332 bool AnyLegalized = false;
6333 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6334 --NI;
6335
6336 SDNode *N = &*NI;
6337 if (N->use_empty() && N != getRoot().getNode()) {
6338 ++NI;
6339 DeleteNode(N);
6340 continue;
6341 }
6342
6343 if (LegalizedNodes.insert(Ptr: N).second) {
6344 AnyLegalized = true;
6345 Legalizer.LegalizeOp(Node: N);
6346
6347 if (N->use_empty() && N != getRoot().getNode()) {
6348 ++NI;
6349 DeleteNode(N);
6350 }
6351 }
6352 }
6353 if (!AnyLegalized)
6354 break;
6355
6356 }
6357
6358 // Remove dead nodes now.
6359 RemoveDeadNodes();
6360}
6361
6362bool SelectionDAG::LegalizeOp(SDNode *N,
6363 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6364 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6365 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6366
6367 // Directly insert the node in question, and legalize it. This will recurse
6368 // as needed through operands.
6369 LegalizedNodes.insert(Ptr: N);
6370 Legalizer.LegalizeOp(Node: N);
6371
6372 return LegalizedNodes.count(Ptr: N);
6373}
6374