1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SelectionDAGNodes.h"
32#include "llvm/CodeGen/TargetFrameLowering.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/CodeGenTypes/MachineValueType.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Compiler.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/MathExtras.h"
49#include "llvm/Support/raw_ostream.h"
50#include "llvm/Target/TargetMachine.h"
51#include "llvm/Target/TargetOptions.h"
52#include <cassert>
53#include <cstdint>
54#include <tuple>
55#include <utility>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "legalizedag"
60
61namespace {
62
63/// Keeps track of state when getting the sign of a floating-point value as an
64/// integer.
65struct FloatSignAsInt {
66 EVT FloatVT;
67 SDValue Chain;
68 SDValue FloatPtr;
69 SDValue IntPtr;
70 MachinePointerInfo IntPointerInfo;
71 MachinePointerInfo FloatPointerInfo;
72 SDValue IntValue;
73 APInt SignMask;
74 uint8_t SignBit;
75};
76
77//===----------------------------------------------------------------------===//
78/// This takes an arbitrary SelectionDAG as input and
79/// hacks on it until the target machine can handle it. This involves
80/// eliminating value sizes the machine cannot handle (promoting small sizes to
81/// large sizes or splitting up large values into small values) as well as
82/// eliminating operations the machine cannot handle.
83///
84/// This code also does a small amount of optimization and recognition of idioms
85/// as part of its processing. For example, if a target does not support a
86/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
87/// will attempt merge setcc and brc instructions into brcc's.
88class SelectionDAGLegalize {
89 const TargetMachine &TM;
90 const TargetLowering &TLI;
91 SelectionDAG &DAG;
92
93 /// The set of nodes which have already been legalized. We hold a
94 /// reference to it in order to update as necessary on node deletion.
95 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
96
97 /// A set of all the nodes updated during legalization.
98 SmallSetVector<SDNode *, 16> *UpdatedNodes;
99
100 EVT getSetCCResultType(EVT VT) const {
101 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
102 }
103
104 // Libcall insertion helpers.
105
106public:
107 SelectionDAGLegalize(SelectionDAG &DAG,
108 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
109 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
110 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
112
113 /// Legalizes the given operation.
114 void LegalizeOp(SDNode *Node);
115
116private:
117 SDValue OptimizeFloatStore(StoreSDNode *ST);
118
119 void LegalizeLoadOps(SDNode *Node);
120 void LegalizeStoreOps(SDNode *Node);
121
122 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
123
124 /// Return a vector shuffle operation which
125 /// performs the same shuffe in terms of order or result bytes, but on a type
126 /// whose vector element type is narrower than the original shuffle type.
127 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
128 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
129 SDValue N1, SDValue N2,
130 ArrayRef<int> Mask) const;
131
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 TargetLowering::ArgListTy &&Args,
134 bool IsSigned, EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
136
137 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 SmallVectorImpl<SDValue> &Results);
139
140 void
141 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
142 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
143 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
144 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
145 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
146 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
147 SmallVectorImpl<SDValue> &Results);
148
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
150 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
151 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
152 void ExpandArgFPLibCall(SDNode *Node,
153 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
154 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
155 RTLIB::Libcall Call_PPCF128,
156 SmallVectorImpl<SDValue> &Results);
157 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
158 RTLIB::Libcall CallI64,
159 RTLIB::Libcall CallI128);
160 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161
162 SDValue ExpandSincosStretLibCall(SDNode *Node) const;
163
164 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
165 const SDLoc &dl);
166 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
167 const SDLoc &dl, SDValue ChainIn);
168 SDValue ExpandBUILD_VECTOR(SDNode *Node);
169 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
170 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
171 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
172 SmallVectorImpl<SDValue> &Results);
173 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
174 SDValue Value) const;
175 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
176 SDValue NewIntValue) const;
177 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
178 SDValue ExpandFABS(SDNode *Node) const;
179 SDValue ExpandFNEG(SDNode *Node) const;
180 SDValue expandLdexp(SDNode *Node) const;
181 SDValue expandFrexp(SDNode *Node) const;
182 SDValue expandModf(SDNode *Node) const;
183
184 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
185 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
186 SmallVectorImpl<SDValue> &Results);
187 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
188 SmallVectorImpl<SDValue> &Results);
189 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
190
191 /// Implements vector reduce operation promotion.
192 ///
193 /// All vector operands are promoted to a vector type with larger element
194 /// type, and the start value is promoted to a larger scalar type. Then the
195 /// result is truncated back to the original scalar type.
196 SDValue PromoteReduction(SDNode *Node);
197
198 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
199
200 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
201 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
202 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
203 SDValue ExpandConcatVectors(SDNode *Node);
204
205 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
206 SDValue ExpandConstant(ConstantSDNode *CP);
207
208 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
209 bool ExpandNode(SDNode *Node);
210 void ConvertNodeToLibcall(SDNode *Node);
211 void PromoteNode(SDNode *Node);
212
213public:
214 // Node replacement helpers
215
216 void ReplacedNode(SDNode *N) {
217 LegalizedNodes.erase(Ptr: N);
218 if (UpdatedNodes)
219 UpdatedNodes->insert(X: N);
220 }
221
222 void ReplaceNode(SDNode *Old, SDNode *New) {
223 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
224 dbgs() << " with: "; New->dump(&DAG));
225
226 assert(Old->getNumValues() == New->getNumValues() &&
227 "Replacing one node with another that produces a different number "
228 "of values!");
229 DAG.ReplaceAllUsesWith(From: Old, To: New);
230 if (UpdatedNodes)
231 UpdatedNodes->insert(X: New);
232 ReplacedNode(N: Old);
233 }
234
235 void ReplaceNode(SDValue Old, SDValue New) {
236 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
237 dbgs() << " with: "; New->dump(&DAG));
238
239 DAG.ReplaceAllUsesWith(From: Old, To: New);
240 if (UpdatedNodes)
241 UpdatedNodes->insert(X: New.getNode());
242 ReplacedNode(N: Old.getNode());
243 }
244
245 void ReplaceNode(SDNode *Old, const SDValue *New) {
246 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
247
248 DAG.ReplaceAllUsesWith(From: Old, To: New);
249 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
250 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
251 New[i]->dump(&DAG));
252 if (UpdatedNodes)
253 UpdatedNodes->insert(X: New[i].getNode());
254 }
255 ReplacedNode(N: Old);
256 }
257
258 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
259 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
260 dbgs() << " with: "; New->dump(&DAG));
261
262 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
263 if (UpdatedNodes)
264 UpdatedNodes->insert(X: New.getNode());
265 ReplacedNode(N: Old.getNode());
266 }
267};
268
269} // end anonymous namespace
270
271// Helper function that generates an MMO that considers the alignment of the
272// stack, and the size of the stack object
273static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
274 MachineFunction &MF,
275 bool isObjectScalable) {
276 auto &MFI = MF.getFrameInfo();
277 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
278 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
279 LocationSize ObjectSize = isObjectScalable
280 ? LocationSize::beforeOrAfterPointer()
281 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
282 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
283 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
284}
285
286/// Return a vector shuffle operation which
287/// performs the same shuffle in terms of order or result bytes, but on a type
288/// whose vector element type is narrower than the original shuffle type.
289/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
290SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
291 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
292 ArrayRef<int> Mask) const {
293 unsigned NumMaskElts = VT.getVectorNumElements();
294 unsigned NumDestElts = NVT.getVectorNumElements();
295 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
296
297 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
298
299 if (NumEltsGrowth == 1)
300 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
301
302 SmallVector<int, 8> NewMask;
303 for (unsigned i = 0; i != NumMaskElts; ++i) {
304 int Idx = Mask[i];
305 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
306 if (Idx < 0)
307 NewMask.push_back(Elt: -1);
308 else
309 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
310 }
311 }
312 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
313 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
314 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
315}
316
317/// Expands the ConstantFP node to an integer constant or
318/// a load from the constant pool.
319SDValue
320SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
321 bool Extend = false;
322 SDLoc dl(CFP);
323
324 // If a FP immediate is precise when represented as a float and if the
325 // target can do an extending load from float to double, we put it into
326 // the constant pool as a float, even if it's is statically typed as a
327 // double. This shrinks FP constants and canonicalizes them for targets where
328 // an FP extending load is the same cost as a normal load (such as on the x87
329 // fp stack or PPC FP unit).
330 EVT VT = CFP->getValueType(ResNo: 0);
331 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
332 if (!UseCP) {
333 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
334 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
335 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
336 }
337
338 APFloat APF = CFP->getValueAPF();
339 EVT OrigVT = VT;
340 EVT SVT = VT;
341
342 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
343 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
344 if (!APF.isSignaling()) {
345 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
346 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
347 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
348 // Only do this if the target has a native EXTLOAD instruction from
349 // smaller type.
350 TLI.isLoadLegal(
351 ValVT: OrigVT, MemVT: SVT,
352 Alignment: Align(DAG.getDataLayout().getPrefTypeAlign(
353 Ty: SVT.getTypeForEVT(Context&: *DAG.getContext()))),
354 AddrSpace: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction())
355 .getAddrSpace(),
356 ExtType: ISD::EXTLOAD, Atomic: false) &&
357 TLI.ShouldShrinkFPConstant(OrigVT)) {
358 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
359 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
360 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
361 VT = SVT;
362 Extend = true;
363 }
364 }
365 }
366
367 SDValue CPIdx =
368 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
369 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
370 if (Extend) {
371 SDValue Result = DAG.getExtLoad(
372 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
373 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
374 Alignment);
375 return Result;
376 }
377 SDValue Result = DAG.getLoad(
378 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
379 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
380 return Result;
381}
382
383/// Expands the Constant node to a load from the constant pool.
384SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
385 SDLoc dl(CP);
386 EVT VT = CP->getValueType(ResNo: 0);
387 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
388 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
389 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
390 SDValue Result = DAG.getLoad(
391 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
392 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
393 return Result;
394}
395
396SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
397 SDValue Vec = Op.getOperand(i: 0);
398 SDValue Val = Op.getOperand(i: 1);
399 SDValue Idx = Op.getOperand(i: 2);
400 SDLoc dl(Op);
401
402 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
403 // SCALAR_TO_VECTOR requires that the type of the value being inserted
404 // match the element type of the vector being created, except for
405 // integers in which case the inserted value can be over width.
406 EVT EltVT = Vec.getValueType().getVectorElementType();
407 if (Val.getValueType() == EltVT ||
408 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
409 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
410 VT: Vec.getValueType(), Operand: Val);
411
412 unsigned NumElts = Vec.getValueType().getVectorNumElements();
413 // We generate a shuffle of InVec and ScVec, so the shuffle mask
414 // should be 0,1,2,3,4,5... with the appropriate element replaced with
415 // elt 0 of the RHS.
416 SmallVector<int, 8> ShufOps;
417 for (unsigned i = 0; i != NumElts; ++i)
418 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
419
420 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
421 }
422 }
423 return ExpandInsertToVectorThroughStack(Op);
424}
425
426SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
427 if (!ISD::isNormalStore(N: ST))
428 return SDValue();
429
430 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
431 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
432 // FIXME: move this to the DAG Combiner! Note that we can't regress due
433 // to phase ordering between legalized code and the dag combiner. This
434 // probably means that we need to integrate dag combiner and legalizer
435 // together.
436 // We generally can't do this one for long doubles.
437 SDValue Chain = ST->getChain();
438 SDValue Ptr = ST->getBasePtr();
439 SDValue Value = ST->getValue();
440 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
441 AAMDNodes AAInfo = ST->getAAInfo();
442 SDLoc dl(ST);
443
444 // Don't optimise TargetConstantFP
445 if (Value.getOpcode() == ISD::TargetConstantFP)
446 return SDValue();
447
448 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
449 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
450 TLI.isTypeLegal(VT: MVT::i32)) {
451 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
452 bitcastToAPInt().zextOrTrunc(width: 32),
453 DL: SDLoc(CFP), VT: MVT::i32);
454 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
455 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
456 }
457
458 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
459 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
460 // If this target supports 64-bit registers, do a single 64-bit store.
461 if (TLI.isTypeLegal(VT: MVT::i64)) {
462 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
463 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
464 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
465 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
466 }
467
468 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
469 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
470 // stores. If the target supports neither 32- nor 64-bits, this
471 // xform is certainly not worth it.
472 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
473 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
474 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
475 if (DAG.getDataLayout().isBigEndian())
476 std::swap(a&: Lo, b&: Hi);
477
478 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
479 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
480 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
481 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
482 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
483 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
484
485 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
486 }
487 }
488 }
489 return SDValue();
490}
491
492void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
493 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
494 SDValue Chain = ST->getChain();
495 SDValue Ptr = ST->getBasePtr();
496 SDLoc dl(Node);
497
498 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
499 MMOMetadata Metadata = ST->getMMOMetadataForSubAccess();
500
501 if (!ST->isTruncatingStore()) {
502 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
503 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
504 ReplaceNode(Old: ST, New: OptStore);
505 return;
506 }
507
508 SDValue Value = ST->getValue();
509 MVT VT = Value.getSimpleValueType();
510 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
511 default: llvm_unreachable("This action is not supported yet!");
512 case TargetLowering::Legal: {
513 // If this is an unaligned store and the target doesn't support it,
514 // expand it.
515 EVT MemVT = ST->getMemoryVT();
516 const DataLayout &DL = DAG.getDataLayout();
517 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
518 MMO: *ST->getMemOperand())) {
519 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
520 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
521 ReplaceNode(Old: SDValue(ST, 0), New: Result);
522 } else
523 LLVM_DEBUG(dbgs() << "Legal store\n");
524 break;
525 }
526 case TargetLowering::Custom: {
527 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
528 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
529 if (Res && Res != SDValue(Node, 0))
530 ReplaceNode(Old: SDValue(Node, 0), New: Res);
531 return;
532 }
533 case TargetLowering::Promote: {
534 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
535 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
536 "Can only promote stores to same size type");
537 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
538 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
539 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
540 ReplaceNode(Old: SDValue(Node, 0), New: Result);
541 break;
542 }
543 }
544 return;
545 }
546
547 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
548 SDValue Value = ST->getValue();
549 EVT StVT = ST->getMemoryVT();
550 TypeSize StWidth = StVT.getSizeInBits();
551 TypeSize StSize = StVT.getStoreSizeInBits();
552 auto &DL = DAG.getDataLayout();
553
554 if (StWidth != StSize) {
555 // Promote to a byte-sized store with upper bits zero if not
556 // storing an integral number of bytes. For example, promote
557 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
558 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
559 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
560 SDValue Result =
561 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
562 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
563 ReplaceNode(Old: SDValue(Node, 0), New: Result);
564 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
565 // If not storing a power-of-2 number of bits, expand as two stores.
566 assert(!StVT.isVector() && "Unsupported truncstore!");
567 unsigned StWidthBits = StWidth.getFixedValue();
568 unsigned LogStWidth = Log2_32(Value: StWidthBits);
569 assert(LogStWidth < 32);
570 unsigned RoundWidth = 1 << LogStWidth;
571 assert(RoundWidth < StWidthBits);
572 unsigned ExtraWidth = StWidthBits - RoundWidth;
573 assert(ExtraWidth < RoundWidth);
574 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
575 "Store size not an integral number of bytes!");
576 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
577 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
578 SDValue Lo, Hi;
579 unsigned IncrementSize;
580
581 if (DL.isLittleEndian()) {
582 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
583 // Store the bottom RoundWidth bits.
584 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
585 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
586
587 // Store the remaining ExtraWidth bits.
588 IncrementSize = RoundWidth / 8;
589 Ptr =
590 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
591 Hi = DAG.getNode(
592 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
593 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Value.getValueType(), DL: dl));
594 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
595 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
596 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
597 } else {
598 // Big endian - avoid unaligned stores.
599 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
600 // Store the top RoundWidth bits.
601 Hi = DAG.getNode(
602 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
603 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Value.getValueType(), DL: dl));
604 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
605 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
606
607 // Store the remaining ExtraWidth bits.
608 IncrementSize = RoundWidth / 8;
609 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
610 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
611 VT: Ptr.getValueType()));
612 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
613 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
614 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
615 }
616
617 // The order of the stores doesn't matter.
618 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
619 ReplaceNode(Old: SDValue(Node, 0), New: Result);
620 } else {
621 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT,
622 Alignment: ST->getAlign(), AddrSpace: ST->getAddressSpace())) {
623 default:
624 llvm_unreachable("This action is not supported yet!");
625 case TargetLowering::Legal: {
626 EVT MemVT = ST->getMemoryVT();
627 // If this is an unaligned store and the target doesn't support it,
628 // expand it.
629 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
630 MMO: *ST->getMemOperand())) {
631 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
632 ReplaceNode(Old: SDValue(ST, 0), New: Result);
633 }
634 break;
635 }
636 case TargetLowering::Custom: {
637 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
638 if (Res && Res != SDValue(Node, 0))
639 ReplaceNode(Old: SDValue(Node, 0), New: Res);
640 return;
641 }
642 case TargetLowering::Expand:
643 assert(!StVT.isVector() &&
644 "Vector Stores are handled in LegalizeVectorOps");
645
646 SDValue Result;
647
648 // TRUNCSTORE:i16 i32 -> STORE i16
649 if (TLI.isTypeLegal(VT: StVT)) {
650 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
651 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
652 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
653 } else {
654 // The in-memory type isn't legal. Truncate to the type it would promote
655 // to, and then do a truncstore.
656 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
657 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
658 Operand: Value);
659 Result =
660 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: StVT,
661 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
662 }
663
664 ReplaceNode(Old: SDValue(Node, 0), New: Result);
665 break;
666 }
667 }
668}
669
670void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
671 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
672 SDValue Chain = LD->getChain(); // The chain.
673 SDValue Ptr = LD->getBasePtr(); // The base pointer.
674 SDValue Value; // The value returned by the load op.
675 SDLoc dl(Node);
676
677 ISD::LoadExtType ExtType = LD->getExtensionType();
678 if (ExtType == ISD::NON_EXTLOAD) {
679 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
680 MVT VT = Node->getSimpleValueType(ResNo: 0);
681 SDValue RVal = SDValue(Node, 0);
682 SDValue RChain = SDValue(Node, 1);
683
684 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
685 default: llvm_unreachable("This action is not supported yet!");
686 case TargetLowering::Legal: {
687 EVT MemVT = LD->getMemoryVT();
688 const DataLayout &DL = DAG.getDataLayout();
689 // If this is an unaligned load and the target doesn't support it,
690 // expand it.
691 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
692 MMO: *LD->getMemOperand())) {
693 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
694 }
695 break;
696 }
697 case TargetLowering::Custom:
698 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
699 RVal = Res;
700 RChain = Res.getValue(R: 1);
701 }
702 break;
703
704 case TargetLowering::Promote: {
705 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
706 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
707 "Can only promote loads to same size type");
708
709 // If the range metadata type does not match the legalized memory
710 // operation type, remove the range metadata.
711 if (const MDNode *MD = LD->getRanges()) {
712 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
713 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
714 !NVT.isInteger())
715 LD->getMemOperand()->clearRanges();
716 }
717 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
718 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
719 RChain = Res.getValue(R: 1);
720 break;
721 }
722 }
723 if (RChain.getNode() != Node) {
724 assert(RVal.getNode() != Node && "Load must be completely replaced");
725 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
726 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
727 if (UpdatedNodes) {
728 UpdatedNodes->insert(X: RVal.getNode());
729 UpdatedNodes->insert(X: RChain.getNode());
730 }
731 ReplacedNode(N: Node);
732 }
733 return;
734 }
735
736 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
737 EVT SrcVT = LD->getMemoryVT();
738 TypeSize SrcWidth = SrcVT.getSizeInBits();
739 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
740 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
741
742 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
743 // Some targets pretend to have an i1 loading operation, and actually
744 // load an i8. This trick is correct for ZEXTLOAD because the top 7
745 // bits are guaranteed to be zero; it helps the optimizers understand
746 // that these bits are zero. It is also useful for EXTLOAD, since it
747 // tells the optimizers that those bits are undefined. It would be
748 // nice to have an effective generic way of getting these benefits...
749 // Until such a way is found, don't insist on promoting i1 here.
750 (SrcVT != MVT::i1 ||
751 TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1, Alignment: LD->getAlign(),
752 AddrSpace: LD->getAddressSpace(), ExtType,
753 Atomic: false) == TargetLowering::Promote)) {
754 // Promote to a byte-sized load if not loading an integral number of
755 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
756 unsigned NewWidth = SrcVT.getStoreSizeInBits();
757 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
758 SDValue Ch;
759
760 // The extra bits are guaranteed to be zero, since we stored them that
761 // way. A zext load from NVT thus automatically gives zext from SrcVT.
762
763 ISD::LoadExtType NewExtType =
764 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
765
766 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
767 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
768 Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
769
770 Ch = Result.getValue(R: 1); // The chain.
771
772 if (ExtType == ISD::SEXTLOAD)
773 // Having the top bits zero doesn't help when sign extending.
774 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
775 VT: Result.getValueType(),
776 N1: Result, N2: DAG.getValueType(SrcVT));
777 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
778 // All the top bits are guaranteed to be zero - inform the optimizers.
779 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
780 VT: Result.getValueType(), N1: Result,
781 N2: DAG.getValueType(SrcVT));
782
783 Value = Result;
784 Chain = Ch;
785 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
786 // If not loading a power-of-2 number of bits, expand as two loads.
787 assert(!SrcVT.isVector() && "Unsupported extload!");
788 unsigned SrcWidthBits = SrcWidth.getFixedValue();
789 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
790 assert(LogSrcWidth < 32);
791 unsigned RoundWidth = 1 << LogSrcWidth;
792 assert(RoundWidth < SrcWidthBits);
793 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
794 assert(ExtraWidth < RoundWidth);
795 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
796 "Load size not an integral number of bytes!");
797 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
798 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
799 SDValue Lo, Hi, Ch;
800 unsigned IncrementSize;
801 auto &DL = DAG.getDataLayout();
802
803 if (DL.isLittleEndian()) {
804 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
805 // Load the bottom RoundWidth bits.
806 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
807 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
808 MMOFlags, Metadata);
809
810 // Load the remaining ExtraWidth bits.
811 IncrementSize = RoundWidth / 8;
812 Ptr =
813 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
814 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
815 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
816 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
817
818 // Build a factor node to remember that this load is independent of
819 // the other one.
820 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
821 N2: Hi.getValue(R: 1));
822
823 // Move the top bits to the right place.
824 Hi = DAG.getNode(
825 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
826 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Hi.getValueType(), DL: dl));
827
828 // Join the hi and lo parts.
829 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
830 } else {
831 // Big endian - avoid unaligned loads.
832 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
833 // Load the top RoundWidth bits.
834 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
835 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
836 MMOFlags, Metadata);
837
838 // Load the remaining ExtraWidth bits.
839 IncrementSize = RoundWidth / 8;
840 Ptr =
841 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
842 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
843 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
844 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
845
846 // Build a factor node to remember that this load is independent of
847 // the other one.
848 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
849 N2: Hi.getValue(R: 1));
850
851 // Move the top bits to the right place.
852 Hi = DAG.getNode(
853 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
854 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Hi.getValueType(), DL: dl));
855
856 // Join the hi and lo parts.
857 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
858 }
859
860 Chain = Ch;
861 } else {
862 bool isCustom = false;
863 switch (TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: SrcVT.getSimpleVT(),
864 Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(), ExtType,
865 Atomic: false)) {
866 default:
867 llvm_unreachable("This action is not supported yet!");
868 case TargetLowering::Custom:
869 isCustom = true;
870 [[fallthrough]];
871 case TargetLowering::Legal:
872 Value = SDValue(Node, 0);
873 Chain = SDValue(Node, 1);
874
875 if (isCustom) {
876 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
877 Value = Res;
878 Chain = Res.getValue(R: 1);
879 }
880 } else {
881 // If this is an unaligned load and the target doesn't support it,
882 // expand it.
883 EVT MemVT = LD->getMemoryVT();
884 const DataLayout &DL = DAG.getDataLayout();
885 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
886 MMO: *LD->getMemOperand())) {
887 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
888 }
889 }
890 break;
891
892 case TargetLowering::Expand: {
893 EVT DestVT = Node->getValueType(ResNo: 0);
894 if (!TLI.isLoadLegal(ValVT: DestVT, MemVT: SrcVT, Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(),
895 ExtType: ISD::EXTLOAD, Atomic: false)) {
896 // If the source type is not legal, see if there is a legal extload to
897 // an intermediate type that we can then extend further.
898 EVT LoadVT =
899 TLI.getRegisterType(Context&: *DAG.getContext(), VT: SrcVT.getSimpleVT());
900 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
901 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
902 TLI.isLoadLegal(ValVT: LoadVT, MemVT: SrcVT, Alignment: LD->getAlign(),
903 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false))) {
904 // If we are loading a legal type, this is a non-extload followed by a
905 // full extend.
906 ISD::LoadExtType MidExtType =
907 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
908
909 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
910 MemVT: SrcVT, MMO: LD->getMemOperand());
911 unsigned ExtendOp =
912 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
913 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
914 Chain = Load.getValue(R: 1);
915 break;
916 }
917
918 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
919 // normal undefined upper bits behavior to allow using an in-reg extend
920 // with the illegal FP type, so load as an integer and do the
921 // from-integer conversion.
922 EVT SVT = SrcVT.getScalarType();
923 if (SVT == MVT::f16 || SVT == MVT::bf16) {
924 EVT ISrcVT = SrcVT.changeTypeToInteger();
925 EVT IDestVT = DestVT.changeTypeToInteger();
926 EVT ILoadVT =
927 TLI.getRegisterType(Context&: *DAG.getContext(), VT: IDestVT.getSimpleVT());
928
929 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
930 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
931 Value =
932 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
933 DL: dl, VT: DestVT, Operand: Result);
934 Chain = Result.getValue(R: 1);
935 break;
936 }
937 }
938
939 assert(!SrcVT.isVector() &&
940 "Vector Loads are handled in LegalizeVectorOps");
941
942 // FIXME: This does not work for vectors on most targets. Sign-
943 // and zero-extend operations are currently folded into extending
944 // loads, whether they are legal or not, and then we end up here
945 // without any support for legalizing them.
946 assert(ExtType != ISD::EXTLOAD &&
947 "EXTLOAD should always be supported!");
948 // Turn the unsupported load into an EXTLOAD followed by an
949 // explicit zero/sign extend inreg.
950 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
951 VT: Node->getValueType(ResNo: 0),
952 Chain, Ptr, MemVT: SrcVT,
953 MMO: LD->getMemOperand());
954 SDValue ValRes;
955 if (ExtType == ISD::SEXTLOAD)
956 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
957 VT: Result.getValueType(),
958 N1: Result, N2: DAG.getValueType(SrcVT));
959 else
960 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
961 Value = ValRes;
962 Chain = Result.getValue(R: 1);
963 break;
964 }
965 }
966 }
967
968 // Since loads produce two values, make sure to remember that we legalized
969 // both of them.
970 if (Chain.getNode() != Node) {
971 assert(Value.getNode() != Node && "Load must be completely replaced");
972 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
973 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
974 if (UpdatedNodes) {
975 UpdatedNodes->insert(X: Value.getNode());
976 UpdatedNodes->insert(X: Chain.getNode());
977 }
978 ReplacedNode(N: Node);
979 }
980}
981
982/// Return a legal replacement for the given operation, with all legal operands.
983void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
984 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
985
986 // Allow illegal target nodes and illegal registers.
987 if (Node->getOpcode() == ISD::TargetConstant ||
988 Node->getOpcode() == ISD::Register)
989 return;
990
991#ifndef NDEBUG
992 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
993 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
994 TargetLowering::TypeLegal &&
995 "Unexpected illegal type!");
996
997 for (const SDValue &Op : Node->op_values())
998 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
999 TargetLowering::TypeLegal ||
1000 Op.getOpcode() == ISD::TargetConstant ||
1001 Op.getOpcode() == ISD::Register) &&
1002 "Unexpected illegal type!");
1003#endif
1004
1005 // Figure out the correct action; the way to query this varies by opcode
1006 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1007 bool SimpleFinishLegalizing = true;
1008 switch (Node->getOpcode()) {
1009 case ISD::INTRINSIC_W_CHAIN:
1010 case ISD::INTRINSIC_WO_CHAIN:
1011 case ISD::INTRINSIC_VOID:
1012 case ISD::STACKSAVE:
1013 case ISD::STACKADDRESS:
1014 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1015 break;
1016 case ISD::GET_DYNAMIC_AREA_OFFSET:
1017 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1018 VT: Node->getValueType(ResNo: 0));
1019 break;
1020 case ISD::VAARG:
1021 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1022 VT: Node->getValueType(ResNo: 0));
1023 if (Action != TargetLowering::Promote)
1024 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1025 break;
1026 case ISD::SET_FPENV:
1027 case ISD::SET_FPMODE:
1028 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1029 VT: Node->getOperand(Num: 1).getValueType());
1030 break;
1031 case ISD::FP_TO_FP16:
1032 case ISD::FP_TO_BF16:
1033 case ISD::SINT_TO_FP:
1034 case ISD::UINT_TO_FP:
1035 case ISD::EXTRACT_VECTOR_ELT:
1036 case ISD::LROUND:
1037 case ISD::LLROUND:
1038 case ISD::LRINT:
1039 case ISD::LLRINT:
1040 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1041 VT: Node->getOperand(Num: 0).getValueType());
1042 break;
1043 case ISD::STRICT_FP_TO_FP16:
1044 case ISD::STRICT_FP_TO_BF16:
1045 case ISD::STRICT_SINT_TO_FP:
1046 case ISD::STRICT_UINT_TO_FP:
1047 case ISD::STRICT_LRINT:
1048 case ISD::STRICT_LLRINT:
1049 case ISD::STRICT_LROUND:
1050 case ISD::STRICT_LLROUND:
1051 // These pseudo-ops are the same as the other STRICT_ ops except
1052 // they are registered with setOperationAction() using the input type
1053 // instead of the output type.
1054 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1055 VT: Node->getOperand(Num: 1).getValueType());
1056 break;
1057 case ISD::SIGN_EXTEND_INREG: {
1058 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1059 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1060 break;
1061 }
1062 case ISD::ATOMIC_STORE:
1063 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1064 VT: Node->getOperand(Num: 1).getValueType());
1065 break;
1066 case ISD::SELECT_CC:
1067 case ISD::STRICT_FSETCC:
1068 case ISD::STRICT_FSETCCS:
1069 case ISD::SETCC:
1070 case ISD::SETCCCARRY:
1071 case ISD::BR_CC: {
1072 unsigned Opc = Node->getOpcode();
1073 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1074 : Opc == ISD::STRICT_FSETCC ? 3
1075 : Opc == ISD::STRICT_FSETCCS ? 3
1076 : Opc == ISD::SETCCCARRY ? 3
1077 : Opc == ISD::SETCC ? 2
1078 : 1;
1079 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1080 : Opc == ISD::STRICT_FSETCC ? 1
1081 : Opc == ISD::STRICT_FSETCCS ? 1
1082 : 0;
1083 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1084 ISD::CondCode CCCode =
1085 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1086 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1087 if (Action == TargetLowering::Legal) {
1088 if (Node->getOpcode() == ISD::SELECT_CC)
1089 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1090 VT: Node->getValueType(ResNo: 0));
1091 else
1092 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1093 }
1094 break;
1095 }
1096 case ISD::LOAD:
1097 case ISD::STORE:
1098 // FIXME: Model these properly. LOAD and STORE are complicated, and
1099 // STORE expects the unlegalized operand in some cases.
1100 SimpleFinishLegalizing = false;
1101 break;
1102 case ISD::CALLSEQ_START:
1103 case ISD::CALLSEQ_END:
1104 // FIXME: This shouldn't be necessary. These nodes have special properties
1105 // dealing with the recursive nature of legalization. Removing this
1106 // special case should be done as part of making LegalizeDAG non-recursive.
1107 SimpleFinishLegalizing = false;
1108 break;
1109 case ISD::EXTRACT_ELEMENT:
1110 case ISD::GET_ROUNDING:
1111 case ISD::MERGE_VALUES:
1112 case ISD::EH_RETURN:
1113 case ISD::FRAME_TO_ARGS_OFFSET:
1114 case ISD::EH_DWARF_CFA:
1115 case ISD::EH_SJLJ_SETJMP:
1116 case ISD::EH_SJLJ_LONGJMP:
1117 case ISD::EH_SJLJ_SETUP_DISPATCH:
1118 // These operations lie about being legal: when they claim to be legal,
1119 // they should actually be expanded.
1120 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1121 if (Action == TargetLowering::Legal)
1122 Action = TargetLowering::Expand;
1123 break;
1124 case ISD::INIT_TRAMPOLINE:
1125 case ISD::ADJUST_TRAMPOLINE:
1126 case ISD::FRAMEADDR:
1127 case ISD::RETURNADDR:
1128 case ISD::ADDROFRETURNADDR:
1129 case ISD::SPONENTRY:
1130 // These operations lie about being legal: when they claim to be legal,
1131 // they should actually be custom-lowered.
1132 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1133 if (Action == TargetLowering::Legal)
1134 Action = TargetLowering::Custom;
1135 break;
1136 case ISD::CLEAR_CACHE:
1137 // This operation is typically going to be LibCall unless the target wants
1138 // something differrent.
1139 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1140 break;
1141 case ISD::READCYCLECOUNTER:
1142 case ISD::READSTEADYCOUNTER:
1143 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1144 // legalization might have expanded that to several smaller types.
1145 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1146 break;
1147 case ISD::READ_REGISTER:
1148 case ISD::WRITE_REGISTER:
1149 // Named register is legal in the DAG, but blocked by register name
1150 // selection if not implemented by target (to chose the correct register)
1151 // They'll be converted to Copy(To/From)Reg.
1152 Action = TargetLowering::Legal;
1153 break;
1154 case ISD::UBSANTRAP:
1155 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1156 if (Action == TargetLowering::Expand) {
1157 // replace ISD::UBSANTRAP with ISD::TRAP
1158 SDValue NewVal;
1159 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1160 N: Node->getOperand(Num: 0));
1161 ReplaceNode(Old: Node, New: NewVal.getNode());
1162 LegalizeOp(Node: NewVal.getNode());
1163 return;
1164 }
1165 break;
1166 case ISD::DEBUGTRAP:
1167 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1168 if (Action == TargetLowering::Expand) {
1169 // replace ISD::DEBUGTRAP with ISD::TRAP
1170 SDValue NewVal;
1171 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1172 N: Node->getOperand(Num: 0));
1173 ReplaceNode(Old: Node, New: NewVal.getNode());
1174 LegalizeOp(Node: NewVal.getNode());
1175 return;
1176 }
1177 break;
1178 case ISD::SADDSAT:
1179 case ISD::UADDSAT:
1180 case ISD::SSUBSAT:
1181 case ISD::USUBSAT:
1182 case ISD::SSHLSAT:
1183 case ISD::USHLSAT:
1184 case ISD::SCMP:
1185 case ISD::UCMP:
1186 case ISD::FP_TO_SINT_SAT:
1187 case ISD::FP_TO_UINT_SAT:
1188 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1189 break;
1190 case ISD::SMULFIX:
1191 case ISD::SMULFIXSAT:
1192 case ISD::UMULFIX:
1193 case ISD::UMULFIXSAT:
1194 case ISD::SDIVFIX:
1195 case ISD::SDIVFIXSAT:
1196 case ISD::UDIVFIX:
1197 case ISD::UDIVFIXSAT: {
1198 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1199 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1200 VT: Node->getValueType(ResNo: 0), Scale);
1201 break;
1202 }
1203 case ISD::MSCATTER:
1204 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1205 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1206 break;
1207 case ISD::MSTORE:
1208 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1209 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1210 break;
1211 case ISD::VP_SCATTER:
1212 Action = TLI.getOperationAction(
1213 Op: Node->getOpcode(),
1214 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1215 break;
1216 case ISD::VP_STORE:
1217 Action = TLI.getOperationAction(
1218 Op: Node->getOpcode(),
1219 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1220 break;
1221 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1222 Action = TLI.getOperationAction(
1223 Op: Node->getOpcode(),
1224 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1225 break;
1226 case ISD::VECREDUCE_FADD:
1227 case ISD::VECREDUCE_FMUL:
1228 case ISD::VECREDUCE_ADD:
1229 case ISD::VECREDUCE_MUL:
1230 case ISD::VECREDUCE_AND:
1231 case ISD::VECREDUCE_OR:
1232 case ISD::VECREDUCE_XOR:
1233 case ISD::VECREDUCE_SMAX:
1234 case ISD::VECREDUCE_SMIN:
1235 case ISD::VECREDUCE_UMAX:
1236 case ISD::VECREDUCE_UMIN:
1237 case ISD::VECREDUCE_FMAX:
1238 case ISD::VECREDUCE_FMIN:
1239 case ISD::VECREDUCE_FMAXIMUM:
1240 case ISD::VECREDUCE_FMINIMUM:
1241 case ISD::VECREDUCE_FMAXIMUMNUM:
1242 case ISD::VECREDUCE_FMINIMUMNUM:
1243 case ISD::IS_FPCLASS:
1244 Action = TLI.getOperationAction(
1245 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1246 break;
1247 case ISD::VECREDUCE_SEQ_FADD:
1248 case ISD::VECREDUCE_SEQ_FMUL:
1249 case ISD::VP_REDUCE_FADD:
1250 case ISD::VP_REDUCE_FMUL:
1251 case ISD::VP_REDUCE_ADD:
1252 case ISD::VP_REDUCE_MUL:
1253 case ISD::VP_REDUCE_AND:
1254 case ISD::VP_REDUCE_OR:
1255 case ISD::VP_REDUCE_XOR:
1256 case ISD::VP_REDUCE_SMAX:
1257 case ISD::VP_REDUCE_SMIN:
1258 case ISD::VP_REDUCE_UMAX:
1259 case ISD::VP_REDUCE_UMIN:
1260 case ISD::VP_REDUCE_FMAX:
1261 case ISD::VP_REDUCE_FMIN:
1262 case ISD::VP_REDUCE_FMAXIMUM:
1263 case ISD::VP_REDUCE_FMINIMUM:
1264 case ISD::VP_REDUCE_SEQ_FADD:
1265 case ISD::VP_REDUCE_SEQ_FMUL:
1266 Action = TLI.getOperationAction(
1267 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1268 break;
1269 case ISD::CTTZ_ELTS:
1270 case ISD::CTTZ_ELTS_ZERO_POISON:
1271 case ISD::VP_CTTZ_ELTS:
1272 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
1273 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1274 VT: Node->getOperand(Num: 0).getValueType());
1275 break;
1276 case ISD::VECTOR_INTERLEAVE:
1277 case ISD::VECTOR_DEINTERLEAVE:
1278 Action = TLI.getVectorInterleaveAction(
1279 Opc: Node->getOpcode(), Factor: Node->getNumOperands(), VT: Node->getValueType(ResNo: 0));
1280 break;
1281 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1282 Action = TLI.getOperationAction(
1283 Op: Node->getOpcode(),
1284 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1285 break;
1286 default:
1287 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1288 Action = TLI.getCustomOperationAction(Op&: *Node);
1289 } else {
1290 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1291 }
1292 break;
1293 }
1294
1295 if (SimpleFinishLegalizing) {
1296 SDNode *NewNode = Node;
1297 switch (Node->getOpcode()) {
1298 default: break;
1299 case ISD::SHL:
1300 case ISD::SRL:
1301 case ISD::SRA:
1302 case ISD::ROTL:
1303 case ISD::ROTR:
1304 case ISD::SSHLSAT:
1305 case ISD::USHLSAT: {
1306 // Legalizing shifts/rotates requires adjusting the shift amount
1307 // to the appropriate width.
1308 SDValue Op0 = Node->getOperand(Num: 0);
1309 SDValue Op1 = Node->getOperand(Num: 1);
1310 if (!Op1.getValueType().isVector()) {
1311 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1312 // The getShiftAmountOperand() may create a new operand node or
1313 // return the existing one. If new operand is created we need
1314 // to update the parent node.
1315 // Do not try to legalize SAO here! It will be automatically legalized
1316 // in the next round.
1317 if (SAO != Op1)
1318 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1319 }
1320 break;
1321 }
1322 case ISD::FSHL:
1323 case ISD::FSHR:
1324 case ISD::SRL_PARTS:
1325 case ISD::SRA_PARTS:
1326 case ISD::SHL_PARTS: {
1327 // Legalizing shifts/rotates requires adjusting the shift amount
1328 // to the appropriate width.
1329 SDValue Op0 = Node->getOperand(Num: 0);
1330 SDValue Op1 = Node->getOperand(Num: 1);
1331 SDValue Op2 = Node->getOperand(Num: 2);
1332 if (!Op2.getValueType().isVector()) {
1333 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1334 // The getShiftAmountOperand() may create a new operand node or
1335 // return the existing one. If new operand is created we need
1336 // to update the parent node.
1337 if (SAO != Op2)
1338 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1339 }
1340 break;
1341 }
1342 }
1343
1344 if (NewNode != Node) {
1345 ReplaceNode(Old: Node, New: NewNode);
1346 Node = NewNode;
1347 }
1348 switch (Action) {
1349 case TargetLowering::Legal:
1350 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1351 return;
1352 case TargetLowering::Custom:
1353 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1354 // FIXME: The handling for custom lowering with multiple results is
1355 // a complete mess.
1356 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1357 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1358 return;
1359
1360 if (Node->getNumValues() == 1) {
1361 // Verify the new types match the original. Glue is waived because
1362 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1363 assert((Res.getValueType() == Node->getValueType(0) ||
1364 Node->getValueType(0) == MVT::Glue) &&
1365 "Type mismatch for custom legalized operation");
1366 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1367 // We can just directly replace this node with the lowered value.
1368 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1369 return;
1370 }
1371
1372 SmallVector<SDValue, 8> ResultVals;
1373 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1374 // Verify the new types match the original. Glue is waived because
1375 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1376 assert((Res->getValueType(i) == Node->getValueType(i) ||
1377 Node->getValueType(i) == MVT::Glue) &&
1378 "Type mismatch for custom legalized operation");
1379 ResultVals.push_back(Elt: Res.getValue(R: i));
1380 }
1381 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1382 ReplaceNode(Old: Node, New: ResultVals.data());
1383 return;
1384 }
1385 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1386 [[fallthrough]];
1387 case TargetLowering::Expand:
1388 if (ExpandNode(Node))
1389 return;
1390 [[fallthrough]];
1391 case TargetLowering::LibCall:
1392 ConvertNodeToLibcall(Node);
1393 return;
1394 case TargetLowering::Promote:
1395 PromoteNode(Node);
1396 return;
1397 }
1398 }
1399
1400 switch (Node->getOpcode()) {
1401 default:
1402#ifndef NDEBUG
1403 dbgs() << "NODE: ";
1404 Node->dump( &DAG);
1405 dbgs() << "\n";
1406#endif
1407 llvm_unreachable("Do not know how to legalize this operator!");
1408
1409 case ISD::CALLSEQ_START:
1410 case ISD::CALLSEQ_END:
1411 break;
1412 case ISD::LOAD:
1413 return LegalizeLoadOps(Node);
1414 case ISD::STORE:
1415 return LegalizeStoreOps(Node);
1416 }
1417}
1418
1419SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1420 SDValue Vec = Op.getOperand(i: 0);
1421 SDValue Idx = Op.getOperand(i: 1);
1422 SDLoc dl(Op);
1423
1424 // Before we generate a new store to a temporary stack slot, see if there is
1425 // already one that we can use. There often is because when we scalarize
1426 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1427 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1428 // the vector. If all are expanded here, we don't want one store per vector
1429 // element.
1430
1431 // Caches for hasPredecessorHelper
1432 SmallPtrSet<const SDNode *, 32> Visited;
1433 SmallVector<const SDNode *, 16> Worklist;
1434 Visited.insert(Ptr: Op.getNode());
1435 Worklist.push_back(Elt: Idx.getNode());
1436 SDValue StackPtr, Ch;
1437 for (SDNode *User : Vec.getNode()->users()) {
1438 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1439 if (ST->isIndexed() || ST->isTruncatingStore() ||
1440 ST->getValue() != Vec)
1441 continue;
1442
1443 // Make sure that nothing else could have stored into the destination of
1444 // this store.
1445 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1446 continue;
1447
1448 // If the index is dependent on the store we will introduce a cycle when
1449 // creating the load (the load uses the index, and by replacing the chain
1450 // we will make the index dependent on the load). Also, the store might be
1451 // dependent on the extractelement and introduce a cycle when creating
1452 // the load.
1453 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1454 ST->hasPredecessor(N: Op.getNode()))
1455 continue;
1456
1457 StackPtr = ST->getBasePtr();
1458 Ch = SDValue(ST, 0);
1459 break;
1460 }
1461 }
1462
1463 EVT VecVT = Vec.getValueType();
1464
1465 if (!Ch.getNode()) {
1466 // Store the value to a temporary stack slot, then LOAD the returned part.
1467 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1468 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1469 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1470 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1471 }
1472
1473 SDValue NewLoad;
1474 Align ElementAlignment =
1475 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1476 b: DAG.getDataLayout().getPrefTypeAlign(
1477 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1478
1479 if (Op.getValueType().isVector()) {
1480 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1481 SubVecVT: Op.getValueType(), Index: Idx);
1482 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1483 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1484 } else {
1485 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1486 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1487 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1488 Alignment: ElementAlignment);
1489 }
1490
1491 // Replace the chain going out of the store, by the one out of the load.
1492 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1493
1494 // We introduced a cycle though, so update the loads operands, making sure
1495 // to use the original store's chain as an incoming chain.
1496 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1497 NewLoadOperands[0] = Ch;
1498 NewLoad =
1499 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1500 return NewLoad;
1501}
1502
1503SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1504 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1505
1506 SDValue Vec = Op.getOperand(i: 0);
1507 SDValue Part = Op.getOperand(i: 1);
1508 SDValue Idx = Op.getOperand(i: 2);
1509 SDLoc dl(Op);
1510
1511 // Store the value to a temporary stack slot, then LOAD the returned part.
1512 EVT VecVT = Vec.getValueType();
1513 EVT PartVT = Part.getValueType();
1514 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1515 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1516 MachinePointerInfo PtrInfo =
1517 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1518
1519 // First store the whole vector.
1520 Align BaseVecAlignment =
1521 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1522 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1523 Alignment: BaseVecAlignment);
1524
1525 // Freeze the index so we don't poison the clamping code we're about to emit.
1526 Idx = DAG.getFreeze(V: Idx);
1527
1528 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1529 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1530
1531 // Then store the inserted part.
1532 if (PartVT.isVector()) {
1533 SDValue SubStackPtr =
1534 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1535
1536 // Store the subvector.
1537 Ch = DAG.getStore(
1538 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1539 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1540 Alignment: PartAlignment);
1541 } else {
1542 SDValue SubStackPtr =
1543 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1544
1545 // Store the scalar value.
1546 Ch = DAG.getTruncStore(
1547 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1548 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1549 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1550 }
1551
1552 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1553 "ElementAlignment does not match!");
1554
1555 // Finally, load the updated vector.
1556 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1557 Alignment: BaseVecAlignment);
1558}
1559
1560SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1561 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1562 SDLoc DL(Node);
1563 SmallVector<SDValue, 16> Ops;
1564 unsigned NumOperands = Node->getNumOperands();
1565 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1566 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1567 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1568 EVT ElementValueType = TLI.getTypeToTransformTo(
1569 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1570 for (unsigned I = 0; I < NumOperands; ++I) {
1571 SDValue SubOp = Node->getOperand(Num: I);
1572 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1573 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1574 N1: SubOp,
1575 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1576 }
1577 }
1578 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1579}
1580
1581SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1582 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1583 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1584 "Unexpected opcode!");
1585
1586 // We can't handle this case efficiently. Allocate a sufficiently
1587 // aligned object on the stack, store each operand into it, then load
1588 // the result as a vector.
1589 // Create the stack frame object.
1590 EVT VT = Node->getValueType(ResNo: 0);
1591 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1592 : Node->getOperand(Num: 0).getValueType();
1593 SDLoc dl(Node);
1594 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1595 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1596 MachinePointerInfo PtrInfo =
1597 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1598
1599 // Emit a store of each element to the stack slot.
1600 SmallVector<SDValue, 8> Stores;
1601 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1602 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1603
1604 // If the destination vector element type of a BUILD_VECTOR is narrower than
1605 // the source element type, only store the bits necessary.
1606 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1607 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1608
1609 // Store (in the right endianness) the elements to memory.
1610 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1611 // Ignore undef elements.
1612 if (Node->getOperand(Num: i).isUndef()) continue;
1613
1614 unsigned Offset = TypeByteSize*i;
1615
1616 SDValue Idx =
1617 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1618
1619 if (Truncate)
1620 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1621 Val: Node->getOperand(Num: i), Ptr: Idx,
1622 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1623 else
1624 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1625 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1626 }
1627
1628 SDValue StoreChain;
1629 if (!Stores.empty()) // Not all undef elements?
1630 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1631 else
1632 StoreChain = DAG.getEntryNode();
1633
1634 // Result is a load from the stack slot.
1635 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1636}
1637
1638/// Bitcast a floating-point value to an integer value. Only bitcast the part
1639/// containing the sign bit if the target has no integer value capable of
1640/// holding all bits of the floating-point value.
1641void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1642 const SDLoc &DL,
1643 SDValue Value) const {
1644 EVT FloatVT = Value.getValueType();
1645 unsigned NumBits = FloatVT.getScalarSizeInBits();
1646 State.FloatVT = FloatVT;
1647 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1648 // Convert to an integer of the same size.
1649 if (TLI.isTypeLegal(VT: IVT)) {
1650 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1651 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1652 State.SignBit = NumBits - 1;
1653 return;
1654 }
1655
1656 auto &DataLayout = DAG.getDataLayout();
1657 // Store the float to memory, then load the sign part out as an integer.
1658 MVT LoadTy = TLI.getRegisterType(Context&: *DAG.getContext(), VT: MVT::i8);
1659 // First create a temporary that is aligned for both the load and store.
1660 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1661 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1662 // Then store the float to it.
1663 State.FloatPtr = StackPtr;
1664 MachineFunction &MF = DAG.getMachineFunction();
1665 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1666 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1667 PtrInfo: State.FloatPointerInfo);
1668
1669 SDValue IntPtr;
1670 if (DataLayout.isBigEndian()) {
1671 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1672 // Load out a legal integer with the same sign bit as the float.
1673 IntPtr = StackPtr;
1674 State.IntPointerInfo = State.FloatPointerInfo;
1675 } else {
1676 // Advance the pointer so that the loaded byte will contain the sign bit.
1677 unsigned ByteOffset = (NumBits / 8) - 1;
1678 IntPtr =
1679 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1680 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1681 Offset: ByteOffset);
1682 }
1683
1684 State.IntPtr = IntPtr;
1685 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1686 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1687 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1688 State.SignBit = 7;
1689}
1690
1691/// Replace the integer value produced by getSignAsIntValue() with a new value
1692/// and cast the result back to a floating-point type.
1693SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1694 const SDLoc &DL,
1695 SDValue NewIntValue) const {
1696 if (!State.Chain)
1697 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1698
1699 // Override the part containing the sign bit in the value stored on the stack.
1700 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1701 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1702 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1703 PtrInfo: State.FloatPointerInfo);
1704}
1705
1706SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1707 SDLoc DL(Node);
1708 SDValue Mag = Node->getOperand(Num: 0);
1709 SDValue Sign = Node->getOperand(Num: 1);
1710
1711 if (Sign.getValueType().isVector())
1712 return DAG.UnrollVectorOp(N: Node);
1713
1714 // Get sign bit into an integer value.
1715 FloatSignAsInt SignAsInt;
1716 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1717
1718 EVT IntVT = SignAsInt.IntValue.getValueType();
1719 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1720 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1721 N2: SignMask);
1722
1723 // If FABS is legal transform
1724 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1725 EVT FloatVT = Mag.getValueType();
1726 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1727 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1728 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1729 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1730 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1731 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1732 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1733 }
1734
1735 // Transform Mag value to integer, and clear the sign bit.
1736 FloatSignAsInt MagAsInt;
1737 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1738 EVT MagVT = MagAsInt.IntValue.getValueType();
1739 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1740 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1741 N2: ClearSignMask);
1742
1743 // Get the signbit at the right position for MagAsInt.
1744 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1745 EVT ShiftVT = IntVT;
1746 if (SignBit.getScalarValueSizeInBits() <
1747 ClearedSign.getScalarValueSizeInBits()) {
1748 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1749 ShiftVT = MagVT;
1750 }
1751 if (ShiftAmount > 0) {
1752 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1753 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1754 } else if (ShiftAmount < 0) {
1755 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1756 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1757 }
1758 if (SignBit.getScalarValueSizeInBits() >
1759 ClearedSign.getScalarValueSizeInBits()) {
1760 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1761 }
1762
1763 // Store the part with the modified sign and convert back to float.
1764 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1765 Flags: SDNodeFlags::Disjoint);
1766
1767 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1768}
1769
1770SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1771 // Get the sign bit as an integer.
1772 SDLoc DL(Node);
1773 if (Node->getValueType(ResNo: 0).isVector())
1774 return DAG.UnrollVectorOp(N: Node);
1775
1776 FloatSignAsInt SignAsInt;
1777 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1778 EVT IntVT = SignAsInt.IntValue.getValueType();
1779
1780 // Flip the sign.
1781 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1782 SDValue SignFlip =
1783 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1784
1785 // Convert back to float.
1786 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1787}
1788
1789SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1790 SDLoc DL(Node);
1791 SDValue Value = Node->getOperand(Num: 0);
1792
1793 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1794 EVT FloatVT = Value.getValueType();
1795 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1796 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1797 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1798 }
1799
1800 if (FloatVT.isVector())
1801 return DAG.UnrollVectorOp(N: Node);
1802
1803 // Transform value to integer, clear the sign bit and transform back.
1804 FloatSignAsInt ValueAsInt;
1805 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1806 EVT IntVT = ValueAsInt.IntValue.getValueType();
1807 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1808 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1809 N2: ClearSignMask);
1810 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1811}
1812
1813void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1814 SmallVectorImpl<SDValue> &Results) {
1815 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1816 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1817 " not tell us which reg is the stack pointer!");
1818 SDLoc dl(Node);
1819 EVT VT = Node->getValueType(ResNo: 0);
1820 SDValue Tmp1 = SDValue(Node, 0);
1821 SDValue Tmp2 = SDValue(Node, 1);
1822 SDValue Tmp3 = Node->getOperand(Num: 2);
1823 SDValue Chain = Tmp1.getOperand(i: 0);
1824
1825 // Chain the dynamic stack allocation so that it doesn't modify the stack
1826 // pointer when other instructions are using the stack.
1827 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1828
1829 SDValue Size = Tmp2.getOperand(i: 1);
1830 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1831 Chain = SP.getValue(R: 1);
1832 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1833 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1834 unsigned Opc =
1835 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1836 ISD::ADD : ISD::SUB;
1837
1838 Align StackAlign = TFL->getStackAlign();
1839 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1840 if (Alignment > StackAlign)
1841 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1842 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1843 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1844
1845 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1846
1847 Results.push_back(Elt: Tmp1);
1848 Results.push_back(Elt: Tmp2);
1849}
1850
1851SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1852 EVT DestVT, const SDLoc &dl) {
1853 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1854}
1855
1856SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1857 EVT DestVT, const SDLoc &dl,
1858 SDValue Chain) {
1859 EVT SrcVT = SrcOp.getValueType();
1860 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1861 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1862 // Don't convert with stack if the load/store is expensive.
1863 if ((SrcVT.bitsGT(VT: SlotVT) && !TLI.isTruncStoreLegalOrCustom(
1864 ValVT: SrcVT, MemVT: SlotVT, Alignment: DestAlign,
1865 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace())) ||
1866 (SlotVT.bitsLT(VT: DestVT) &&
1867 !TLI.isLoadLegalOrCustom(ValVT: DestVT, MemVT: SlotVT, Alignment: DestAlign,
1868 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace(),
1869 ExtType: ISD::EXTLOAD, Atomic: false)))
1870 return SDValue();
1871
1872 return DAG.emitStackConvert(SrcOp, SlotVT, DestVT, DL: dl, Chain);
1873}
1874
1875SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1876 SDLoc dl(Node);
1877 // Create a vector sized/aligned stack slot, store the value to element #0,
1878 // then load the whole vector back out.
1879 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1880
1881 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1882 int SPFI = StackPtrFI->getIndex();
1883
1884 SDValue Ch = DAG.getTruncStore(
1885 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1886 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1887 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1888 return DAG.getLoad(
1889 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1890 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1891}
1892
1893static bool
1894ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1895 const TargetLowering &TLI, SDValue &Res) {
1896 unsigned NumElems = Node->getNumOperands();
1897 SDLoc dl(Node);
1898 EVT VT = Node->getValueType(ResNo: 0);
1899
1900 // Try to group the scalars into pairs, shuffle the pairs together, then
1901 // shuffle the pairs of pairs together, etc. until the vector has
1902 // been built. This will work only if all of the necessary shuffle masks
1903 // are legal.
1904
1905 // We do this in two phases; first to check the legality of the shuffles,
1906 // and next, assuming that all shuffles are legal, to create the new nodes.
1907 for (int Phase = 0; Phase < 2; ++Phase) {
1908 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1909 NewIntermedVals;
1910 for (unsigned i = 0; i < NumElems; ++i) {
1911 SDValue V = Node->getOperand(Num: i);
1912 if (V.isUndef())
1913 continue;
1914
1915 SDValue Vec;
1916 if (Phase)
1917 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1918 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1919 }
1920
1921 while (IntermedVals.size() > 2) {
1922 NewIntermedVals.clear();
1923 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1924 // This vector and the next vector are shuffled together (simply to
1925 // append the one to the other).
1926 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1927
1928 SmallVector<int, 16> FinalIndices;
1929 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1930 IntermedVals[i+1].second.size());
1931
1932 int k = 0;
1933 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1934 ++j, ++k) {
1935 ShuffleVec[k] = j;
1936 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1937 }
1938 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1939 ++j, ++k) {
1940 ShuffleVec[k] = NumElems + j;
1941 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1942 }
1943
1944 SDValue Shuffle;
1945 if (Phase)
1946 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1947 N2: IntermedVals[i+1].first,
1948 Mask: ShuffleVec);
1949 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1950 return false;
1951 NewIntermedVals.push_back(
1952 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1953 }
1954
1955 // If we had an odd number of defined values, then append the last
1956 // element to the array of new vectors.
1957 if ((IntermedVals.size() & 1) != 0)
1958 NewIntermedVals.push_back(Elt: IntermedVals.back());
1959
1960 IntermedVals.swap(RHS&: NewIntermedVals);
1961 }
1962
1963 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1964 "Invalid number of intermediate vectors");
1965 SDValue Vec1 = IntermedVals[0].first;
1966 SDValue Vec2;
1967 if (IntermedVals.size() > 1)
1968 Vec2 = IntermedVals[1].first;
1969 else if (Phase)
1970 Vec2 = DAG.getPOISON(VT);
1971
1972 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1973 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1974 ShuffleVec[IntermedVals[0].second[i]] = i;
1975 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1976 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1977
1978 if (Phase)
1979 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1980 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1981 return false;
1982 }
1983
1984 return true;
1985}
1986
1987/// Expand a BUILD_VECTOR node on targets that don't
1988/// support the operation, but do support the resultant vector type.
1989SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1990 unsigned NumElems = Node->getNumOperands();
1991 SDValue Value1, Value2;
1992 SDLoc dl(Node);
1993 EVT VT = Node->getValueType(ResNo: 0);
1994 EVT OpVT = Node->getOperand(Num: 0).getValueType();
1995 EVT EltVT = VT.getVectorElementType();
1996
1997 // If the only non-undef value is the low element, turn this into a
1998 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1999 bool isOnlyLowElement = true;
2000 bool MoreThanTwoValues = false;
2001 bool isConstant = true;
2002 for (unsigned i = 0; i < NumElems; ++i) {
2003 SDValue V = Node->getOperand(Num: i);
2004 if (V.isUndef())
2005 continue;
2006 if (i > 0)
2007 isOnlyLowElement = false;
2008 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2009 isConstant = false;
2010
2011 if (!Value1.getNode()) {
2012 Value1 = V;
2013 } else if (!Value2.getNode()) {
2014 if (V != Value1)
2015 Value2 = V;
2016 } else if (V != Value1 && V != Value2) {
2017 MoreThanTwoValues = true;
2018 }
2019 }
2020
2021 if (!Value1.getNode())
2022 return DAG.getUNDEF(VT);
2023
2024 if (isOnlyLowElement)
2025 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2026
2027 // If all elements are constants, create a load from the constant pool.
2028 if (isConstant) {
2029 SmallVector<Constant*, 16> CV;
2030 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2031 if (ConstantFPSDNode *V =
2032 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2033 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2034 } else if (ConstantSDNode *V =
2035 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2036 if (OpVT==EltVT)
2037 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2038 else {
2039 // If OpVT and EltVT don't match, EltVT is not legal and the
2040 // element values have been promoted/truncated earlier. Undo this;
2041 // we don't want a v16i8 to become a v16i32 for example.
2042 const ConstantInt *CI = V->getConstantIntValue();
2043 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2044 V: CI->getZExtValue(), /*IsSigned=*/false,
2045 /*ImplicitTrunc=*/true));
2046 }
2047 } else {
2048 assert(Node->getOperand(i).isUndef());
2049 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2050 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2051 }
2052 }
2053 Constant *CP = ConstantVector::get(V: CV);
2054 SDValue CPIdx =
2055 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2056 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2057 return DAG.getLoad(
2058 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2059 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2060 Alignment);
2061 }
2062
2063 SmallSet<SDValue, 16> DefinedValues;
2064 for (unsigned i = 0; i < NumElems; ++i) {
2065 if (Node->getOperand(Num: i).isUndef())
2066 continue;
2067 DefinedValues.insert(V: Node->getOperand(Num: i));
2068 }
2069
2070 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2071 if (!MoreThanTwoValues) {
2072 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2073 for (unsigned i = 0; i < NumElems; ++i) {
2074 SDValue V = Node->getOperand(Num: i);
2075 if (V.isUndef())
2076 continue;
2077 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2078 }
2079 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2080 // Get the splatted value into the low element of a vector register.
2081 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2082 SDValue Vec2;
2083 if (Value2.getNode())
2084 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2085 else
2086 Vec2 = DAG.getPOISON(VT);
2087
2088 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2089 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2090 }
2091 } else {
2092 SDValue Res;
2093 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2094 return Res;
2095 }
2096 }
2097
2098 // Otherwise, we can't handle this case efficiently.
2099 return ExpandVectorBuildThroughStack(Node);
2100}
2101
2102SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2103 SDLoc DL(Node);
2104 EVT VT = Node->getValueType(ResNo: 0);
2105 SDValue SplatVal = Node->getOperand(Num: 0);
2106
2107 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2108}
2109
2110// Expand a node into a call to a libcall, returning the value as the first
2111// result and the chain as the second. If the result value does not fit into a
2112// register, return the lo part and set the hi part to the by-reg argument in
2113// the first. If it does fit into a single register, return the result and
2114// leave the Hi part unset.
2115std::pair<SDValue, SDValue>
2116SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2117 TargetLowering::ArgListTy &&Args,
2118 bool IsSigned, EVT RetVT) {
2119 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2120 SDValue Callee;
2121 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2122 if (LCImpl != RTLIB::Unsupported)
2123 Callee = DAG.getExternalSymbol(LCImpl, VT: CodePtrTy);
2124 else {
2125 Callee = DAG.getPOISON(VT: CodePtrTy);
2126 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2127 Node->getOperationName(G: &DAG));
2128 }
2129
2130 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2131
2132 // By default, the input chain to this libcall is the entry node of the
2133 // function. If the libcall is going to be emitted as a tail call then
2134 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2135 // node which is being folded has a non-entry input chain.
2136 SDValue InChain = DAG.getEntryNode();
2137
2138 // isTailCall may be true since the callee does not reference caller stack
2139 // frame. Check if it's in the right position and that the return types match.
2140 SDValue TCChain = InChain;
2141 const Function &F = DAG.getMachineFunction().getFunction();
2142 bool isTailCall =
2143 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2144 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()) &&
2145 // Lowering doesn't support tail calling inside a function with
2146 // a swifterror argument yet.
2147 !DAG.hasSwiftErrorArg();
2148 if (isTailCall)
2149 InChain = TCChain;
2150
2151 TargetLowering::CallLoweringInfo CLI(DAG);
2152 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2153 CLI.setDebugLoc(SDLoc(Node))
2154 .setChain(InChain)
2155 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
2156 Target: Callee, ArgsList: std::move(Args))
2157 .setTailCall(isTailCall)
2158 .setSExtResult(signExtend)
2159 .setZExtResult(!signExtend)
2160 .setIsPostTypeLegalization(true);
2161
2162 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2163
2164 if (!CallInfo.second.getNode()) {
2165 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2166 // It's a tailcall, return the chain (which is the DAG root).
2167 return {DAG.getRoot(), DAG.getRoot()};
2168 }
2169
2170 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2171 return CallInfo;
2172}
2173
2174std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2175 bool isSigned) {
2176 TargetLowering::ArgListTy Args;
2177 for (const SDValue &Op : Node->op_values()) {
2178 EVT ArgVT = Op.getValueType();
2179 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2180 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2181 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2182 Entry.IsZExt = !Entry.IsSExt;
2183 Args.push_back(x: Entry);
2184 }
2185
2186 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2187 RetVT: Node->getValueType(ResNo: 0));
2188}
2189
2190void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2191 RTLIB::Libcall LC,
2192 SmallVectorImpl<SDValue> &Results) {
2193 if (LC == RTLIB::UNKNOWN_LIBCALL)
2194 llvm_unreachable("Can't create an unknown libcall!");
2195
2196 if (Node->isStrictFPOpcode()) {
2197 EVT RetVT = Node->getValueType(ResNo: 0);
2198 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2199 if (LCImpl == RTLIB::Unsupported) {
2200 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2201 Node->getOperationName(G: &DAG));
2202 Results.push_back(Elt: DAG.getPOISON(VT: RetVT));
2203 Results.push_back(Elt: Node->getOperand(Num: 0));
2204 return;
2205 }
2206 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2207 TargetLowering::MakeLibCallOptions CallOptions;
2208 CallOptions.IsPostTypeLegalization = true;
2209 // FIXME: This doesn't support tail calls.
2210 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2211 DAG, LibcallImpl: LCImpl, RetVT, Ops, CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
2212 Results.push_back(Elt: Tmp.first);
2213 Results.push_back(Elt: Tmp.second);
2214 } else {
2215 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2216 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2217 Results.push_back(Elt: Tmp);
2218 }
2219}
2220
2221/// Expand the node to a libcall based on the result type.
2222void SelectionDAGLegalize::ExpandFastFPLibCall(
2223 SDNode *Node, bool IsFast,
2224 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2225 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2226 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2227 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2228 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2229 SmallVectorImpl<SDValue> &Results) {
2230
2231 EVT VT = Node->getSimpleValueType(ResNo: 0);
2232
2233 RTLIB::Libcall LC;
2234
2235 // FIXME: Probably should define fast to respect nan/inf and only be
2236 // approximate functions.
2237
2238 if (IsFast) {
2239 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.first, Call_F64: Call_F64.first, Call_F80: Call_F80.first,
2240 Call_F128: Call_F128.first, Call_PPCF128: Call_PPCF128.first);
2241 }
2242
2243 if (!IsFast || DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
2244 // Fall back if we don't have a fast implementation.
2245 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.second, Call_F64: Call_F64.second,
2246 Call_F80: Call_F80.second, Call_F128: Call_F128.second,
2247 Call_PPCF128: Call_PPCF128.second);
2248 }
2249
2250 ExpandFPLibCall(Node, LC, Results);
2251}
2252
2253SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2254 RTLIB::Libcall Call_I8,
2255 RTLIB::Libcall Call_I16,
2256 RTLIB::Libcall Call_I32,
2257 RTLIB::Libcall Call_I64,
2258 RTLIB::Libcall Call_I128) {
2259 RTLIB::Libcall LC;
2260 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2261 default: llvm_unreachable("Unexpected request for libcall!");
2262 case MVT::i8: LC = Call_I8; break;
2263 case MVT::i16: LC = Call_I16; break;
2264 case MVT::i32: LC = Call_I32; break;
2265 case MVT::i64: LC = Call_I64; break;
2266 case MVT::i128: LC = Call_I128; break;
2267 }
2268 return ExpandLibCall(LC, Node, isSigned).first;
2269}
2270
2271/// Expand the node to a libcall based on first argument type (for instance
2272/// lround and its variant).
2273void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2274 RTLIB::Libcall Call_F32,
2275 RTLIB::Libcall Call_F64,
2276 RTLIB::Libcall Call_F80,
2277 RTLIB::Libcall Call_F128,
2278 RTLIB::Libcall Call_PPCF128,
2279 SmallVectorImpl<SDValue> &Results) {
2280 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2281 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2282 Call_F32, Call_F64, Call_F80,
2283 Call_F128, Call_PPCF128);
2284 ExpandFPLibCall(Node, LC, Results);
2285}
2286
2287SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2288 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2289 RTLIB::Libcall CallI128) {
2290 RTLIB::Libcall LC;
2291 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2292 default:
2293 llvm_unreachable("Unexpected request for libcall!");
2294 case MVT::i32:
2295 LC = CallI32;
2296 break;
2297 case MVT::i64:
2298 LC = CallI64;
2299 break;
2300 case MVT::i128:
2301 LC = CallI128;
2302 break;
2303 }
2304
2305 // Bit-counting libcalls have one unsigned argument and return `int`.
2306 // Note that `int` may be illegal on this target; ExpandLibCall will
2307 // take care of promoting it to a legal type.
2308 SDValue Op = Node->getOperand(Num: 0);
2309 EVT IntVT =
2310 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2311
2312 EVT ArgVT = Op.getValueType();
2313 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2314 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2315 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2316 Arg.IsZExt = !Arg.IsSExt;
2317
2318 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2319 /*IsSigned=*/true, RetVT: IntVT)
2320 .first;
2321
2322 // If ExpandLibCall created a tail call, the result was already
2323 // of the correct type. Otherwise, we need to sign extend it.
2324 if (Res.getValueType() != MVT::Other)
2325 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2326 return Res;
2327}
2328
2329/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2330void
2331SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2332 SmallVectorImpl<SDValue> &Results) {
2333 unsigned Opcode = Node->getOpcode();
2334 bool isSigned = Opcode == ISD::SDIVREM;
2335
2336 RTLIB::Libcall LC;
2337 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2338 default: llvm_unreachable("Unexpected request for libcall!");
2339 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2340 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2341 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2342 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2343 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2344 }
2345
2346 // The input chain to this libcall is the entry node of the function.
2347 // Legalizing the call will automatically add the previous call to the
2348 // dependence.
2349 SDValue InChain = DAG.getEntryNode();
2350
2351 EVT RetVT = Node->getValueType(ResNo: 0);
2352 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2353
2354 TargetLowering::ArgListTy Args;
2355 for (const SDValue &Op : Node->op_values()) {
2356 EVT ArgVT = Op.getValueType();
2357 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2358 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2359 Entry.IsSExt = isSigned;
2360 Entry.IsZExt = !isSigned;
2361 Args.push_back(x: Entry);
2362 }
2363
2364 // Also pass the return address of the remainder.
2365 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2366 TargetLowering::ArgListEntry Entry(
2367 FIPtr, PointerType::getUnqual(C&: RetTy->getContext()));
2368 Entry.IsSExt = isSigned;
2369 Entry.IsZExt = !isSigned;
2370 Args.push_back(x: Entry);
2371
2372 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2373 if (LibcallImpl == RTLIB::Unsupported) {
2374 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2375 Node->getOperationName(G: &DAG));
2376 SDValue Poison = DAG.getPOISON(VT: RetVT);
2377 Results.push_back(Elt: Poison);
2378 Results.push_back(Elt: Poison);
2379 return;
2380 }
2381
2382 SDValue Callee =
2383 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2384
2385 SDLoc dl(Node);
2386 TargetLowering::CallLoweringInfo CLI(DAG);
2387 CLI.setDebugLoc(dl)
2388 .setChain(InChain)
2389 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
2390 ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2391 .setSExtResult(isSigned)
2392 .setZExtResult(!isSigned);
2393
2394 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2395
2396 // Remainder is loaded back from the stack frame.
2397 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
2398 MachinePointerInfo PtrInfo =
2399 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2400
2401 SDValue Rem = DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo);
2402 Results.push_back(Elt: CallInfo.first);
2403 Results.push_back(Elt: Rem);
2404}
2405
2406/// Return true if sincos or __sincos_stret libcall is available.
2407static bool isSinCosLibcallAvailable(SDNode *Node,
2408 const LibcallLoweringInfo &Libcalls) {
2409 MVT::SimpleValueType VT = Node->getSimpleValueType(ResNo: 0).SimpleTy;
2410 return Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS(VT)) != RTLIB::Unsupported ||
2411 Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS_STRET(VT)) !=
2412 RTLIB::Unsupported;
2413}
2414
2415/// Only issue sincos libcall if both sin and cos are needed.
2416static bool useSinCos(SDNode *Node) {
2417 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2418 ? ISD::FCOS : ISD::FSIN;
2419
2420 SDValue Op0 = Node->getOperand(Num: 0);
2421 for (const SDNode *User : Op0.getNode()->users()) {
2422 if (User == Node)
2423 continue;
2424 // The other user might have been turned into sincos already.
2425 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2426 return true;
2427 }
2428 return false;
2429}
2430
2431SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2432 // For iOS, we want to call an alternative entry point: __sincos_stret,
2433 // which returns the values in two S / D registers.
2434 SDLoc dl(Node);
2435 SDValue Arg = Node->getOperand(Num: 0);
2436 EVT ArgVT = Arg.getValueType();
2437 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(VT: ArgVT);
2438 RTLIB::LibcallImpl SincosStret = DAG.getLibcalls().getLibcallImpl(Call: LC);
2439 if (SincosStret == RTLIB::Unsupported)
2440 return SDValue();
2441
2442 /// There are 3 different ABI cases to handle:
2443 /// - Direct return of separate fields in registers
2444 /// - Single return as vector elements
2445 /// - sret struct
2446
2447 const RTLIB::RuntimeLibcallsInfo &CallsInfo = TLI.getRuntimeLibcallsInfo();
2448
2449 const DataLayout &DL = DAG.getDataLayout();
2450
2451 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2452 Ctx&: *DAG.getContext(), TT: TM.getTargetTriple(), DL, LibcallImpl: SincosStret);
2453
2454 Type *SincosStretRetTy = FuncTy->getReturnType();
2455 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(Call: SincosStret);
2456
2457 SDValue Callee =
2458 DAG.getExternalSymbol(LCImpl: SincosStret, VT: TLI.getProgramPointerTy(DL));
2459
2460 TargetLowering::ArgListTy Args;
2461 SDValue SRet;
2462
2463 int FrameIdx;
2464 if (FuncTy->getParamType(i: 0)->isPointerTy()) {
2465 // Uses sret
2466 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2467
2468 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(ArgNo: 0);
2469 Type *StructTy = PtrAttrs.getStructRetType();
2470 const uint64_t ByteSize = DL.getTypeAllocSize(Ty: StructTy);
2471 const Align StackAlign = DL.getPrefTypeAlign(Ty: StructTy);
2472
2473 FrameIdx = MFI.CreateStackObject(Size: ByteSize, Alignment: StackAlign, isSpillSlot: false);
2474 SRet = DAG.getFrameIndex(FI: FrameIdx, VT: TLI.getFrameIndexTy(DL));
2475
2476 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(i: 0));
2477 Entry.IsSRet = true;
2478 Entry.IndirectType = StructTy;
2479 Entry.Alignment = StackAlign;
2480
2481 Args.push_back(x: Entry);
2482 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 1));
2483 } else {
2484 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 0));
2485 }
2486
2487 TargetLowering::CallLoweringInfo CLI(DAG);
2488 CLI.setDebugLoc(dl)
2489 .setChain(DAG.getEntryNode())
2490 .setLibCallee(CC: CallConv, ResultType: SincosStretRetTy, Target: Callee, ArgsList: std::move(Args))
2491 .setIsPostTypeLegalization();
2492
2493 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2494
2495 if (SRet) {
2496 MachinePointerInfo PtrInfo =
2497 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2498 SDValue LoadSin = DAG.getLoad(VT: ArgVT, dl, Chain: CallResult.second, Ptr: SRet, PtrInfo);
2499
2500 TypeSize StoreSize = ArgVT.getStoreSize();
2501
2502 // Address of cos field.
2503 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr: SRet, Offset: StoreSize);
2504 SDValue LoadCos = DAG.getLoad(VT: ArgVT, dl, Chain: LoadSin.getValue(R: 1), Ptr: Add,
2505 PtrInfo: PtrInfo.getWithOffset(O: StoreSize));
2506
2507 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2508 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: LoadSin.getValue(R: 0),
2509 N2: LoadCos.getValue(R: 0));
2510 }
2511
2512 if (!CallResult.first.getValueType().isVector())
2513 return CallResult.first;
2514
2515 SDValue SinVal =
2516 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2517 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2518 SDValue CosVal =
2519 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2520 N2: DAG.getVectorIdxConstant(Val: 1, DL: dl));
2521 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2522 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: SinVal, N2: CosVal);
2523}
2524
2525SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2526 SDLoc dl(Node);
2527 EVT VT = Node->getValueType(ResNo: 0);
2528 SDValue X = Node->getOperand(Num: 0);
2529 SDValue N = Node->getOperand(Num: 1);
2530 EVT ExpVT = N.getValueType();
2531 EVT AsIntVT = VT.changeTypeToInteger();
2532 if (AsIntVT == EVT()) // TODO: How to handle f80?
2533 return SDValue();
2534
2535 // The expansion works through the integer-equivalent type; if that is not
2536 // legal, bail out and let the caller use a libcall (or diagnose a missing
2537 // one).
2538 if (!TLI.isTypeLegal(VT: AsIntVT))
2539 return SDValue();
2540
2541 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2542 return SDValue();
2543
2544 SDNodeFlags NSW;
2545 NSW.setNoSignedWrap(true);
2546 SDNodeFlags NUW_NSW;
2547 NUW_NSW.setNoUnsignedWrap(true);
2548 NUW_NSW.setNoSignedWrap(true);
2549
2550 EVT SetCCVT =
2551 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2552 const fltSemantics &FltSem = VT.getFltSemantics();
2553
2554 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2555 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2556 const int Precision = APFloat::semanticsPrecision(FltSem);
2557
2558 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2559 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2560
2561 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2562
2563 const APFloat One(FltSem, "1.0");
2564 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2565
2566 // Offset by precision to avoid denormal range.
2567 APFloat ScaleDownK =
2568 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2569
2570 // TODO: Should really introduce control flow and use a block for the >
2571 // MaxExp, < MinExp cases
2572
2573 // First, handle exponents Exp > MaxExp and scale down.
2574 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2575
2576 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2577 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2578 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2579 SDValue DecN1 =
2580 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2581
2582 SDValue ScaleUpTwice =
2583 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2584
2585 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2586 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2587 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2588
2589 SDValue SelectN_Big =
2590 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2591 SDValue SelectX_Big =
2592 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2593
2594 // Now handle exponents Exp < MinExp
2595 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2596
2597 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2598 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2599
2600 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2601
2602 SDValue ClampMinVal =
2603 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2604 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2605 SDValue IncN1 =
2606 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2607
2608 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2609 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2610 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2611
2612 SDValue ScaleDownTwice = DAG.getSetCC(
2613 DL: dl, VT: SetCCVT, LHS: N,
2614 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2615
2616 SDValue SelectN_Small =
2617 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2618 SDValue SelectX_Small =
2619 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2620
2621 // Now combine the two out of range exponent handling cases with the base
2622 // case.
2623 SDValue NewX = DAG.getNode(
2624 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2625 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2626
2627 SDValue NewN = DAG.getNode(
2628 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2629 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2630
2631 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2632
2633 SDValue ExponentShiftAmt =
2634 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2635 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2636
2637 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2638 N2: ExponentShiftAmt, Flags: NUW_NSW);
2639 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2640 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2641}
2642
2643SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2644 SDLoc dl(Node);
2645 SDValue Val = Node->getOperand(Num: 0);
2646 EVT VT = Val.getValueType();
2647 EVT ExpVT = Node->getValueType(ResNo: 1);
2648 EVT AsIntVT = VT.changeTypeToInteger();
2649 if (AsIntVT == EVT()) // TODO: How to handle f80?
2650 return SDValue();
2651
2652 // The expansion works through the integer-equivalent type; if that is not
2653 // legal, bail out and let the caller use a libcall (or diagnose a missing
2654 // one).
2655 if (!TLI.isTypeLegal(VT: AsIntVT))
2656 return SDValue();
2657
2658 const fltSemantics &FltSem = VT.getFltSemantics();
2659 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2660 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2661 const unsigned BitSize = VT.getScalarSizeInBits();
2662
2663 // TODO: Could introduce control flow and skip over the denormal handling.
2664
2665 // scale_up = fmul value, scalbn(1.0, precision + 1)
2666 // extracted_exp = (bitcast value to uint) >> precision - 1
2667 // biased_exp = extracted_exp + min_exp
2668 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2669 //
2670 // is_denormal = val < smallest_normalized
2671 // computed_fract = is_denormal ? scale_up : extracted_fract
2672 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2673 //
2674 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2675 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2676
2677 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2678 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2679 VT: AsIntVT);
2680
2681 SDValue SmallestNormalizedInt = DAG.getConstant(
2682 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2683 VT: AsIntVT);
2684
2685 // Masks out the exponent bits.
2686 SDValue ExpMask =
2687 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2688
2689 // Mask out the exponent part of the value.
2690 //
2691 // e.g, for f32 FractSignMaskVal = 0x807fffff
2692 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2693 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2694
2695 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2696 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2697
2698 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2699
2700 const APFloat One(FltSem, "1.0");
2701 // Scale a possible denormal input.
2702 // e.g., for f64, 0x1p+54
2703 APFloat ScaleUpKVal =
2704 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2705
2706 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2707 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2708
2709 EVT SetCCVT =
2710 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2711
2712 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2713
2714 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2715
2716 SDValue AddNegSmallestNormal =
2717 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2718 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2719 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2720
2721 SDValue IsDenormal =
2722 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2723
2724 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2725 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2726
2727 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2728 SDValue ScaledSelect =
2729 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2730
2731 SDValue ExpMaskScaled =
2732 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2733
2734 SDValue ScaledValue =
2735 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2736
2737 // Extract the exponent bits.
2738 SDValue ExponentShiftAmt =
2739 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2740 SDValue ShiftedExp =
2741 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2742 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2743
2744 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2745 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2746 SDValue DenormalExpBias =
2747 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2748
2749 SDValue MaskedFractAsInt =
2750 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2751 const APFloat Half(FltSem, "0.5");
2752 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2753 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2754 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2755
2756 SDValue ComputedExp =
2757 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2758
2759 SDValue Result0 =
2760 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2761
2762 SDValue Result1 =
2763 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2764
2765 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2766}
2767
2768SDValue SelectionDAGLegalize::expandModf(SDNode *Node) const {
2769 SDLoc dl(Node);
2770 SDValue Val = Node->getOperand(Num: 0);
2771 EVT VT = Val.getValueType();
2772 SDNodeFlags Flags = Node->getFlags();
2773
2774 SDValue IntPart = DAG.getNode(Opcode: ISD::FTRUNC, DL: dl, VT, Operand: Val, Flags);
2775 SDValue FracPart = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT, N1: Val, N2: IntPart, Flags);
2776
2777 SDValue FracToUse;
2778 if (Flags.hasNoInfs()) {
2779 FracToUse = FracPart;
2780 } else {
2781 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val, Flags);
2782 SDValue Inf =
2783 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
2784 EVT SetCCVT =
2785 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2786 SDValue IsInf = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: Inf, Cond: ISD::SETOEQ);
2787 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: dl, VT);
2788 FracToUse = DAG.getSelect(DL: dl, VT, Cond: IsInf, LHS: Zero, RHS: FracPart);
2789 }
2790
2791 SDValue ResultFrac =
2792 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT, N1: FracToUse, N2: Val, Flags);
2793 return DAG.getMergeValues(Ops: {ResultFrac, IntPart}, dl);
2794}
2795
2796/// This function is responsible for legalizing a
2797/// INT_TO_FP operation of the specified operand when the target requests that
2798/// we expand it. At this point, we know that the result and operand types are
2799/// legal for the target.
2800SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2801 SDValue &Chain) {
2802 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2803 Node->getOpcode() == ISD::SINT_TO_FP);
2804 EVT DestVT = Node->getValueType(ResNo: 0);
2805 SDLoc dl(Node);
2806 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2807 SDValue Op0 = Node->getOperand(Num: OpNo);
2808 EVT SrcVT = Op0.getValueType();
2809
2810 // TODO: Should any fast-math-flags be set for the created nodes?
2811 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2812 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2813 (DestVT.bitsLE(VT: MVT::f64) ||
2814 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2815 : ISD::FP_EXTEND,
2816 VT: DestVT))) {
2817 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2818 "expansion\n");
2819
2820 // Get the stack frame index of a 8 byte buffer.
2821 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2822
2823 SDValue Lo = Op0;
2824 // if signed map to unsigned space
2825 if (isSigned) {
2826 // Invert sign bit (signed to unsigned mapping).
2827 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2828 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2829 }
2830 // Initial hi portion of constructed double.
2831 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2832
2833 // If this a big endian target, swap the lo and high data.
2834 if (DAG.getDataLayout().isBigEndian())
2835 std::swap(a&: Lo, b&: Hi);
2836
2837 SDValue MemChain = DAG.getEntryNode();
2838
2839 // Store the lo of the constructed double.
2840 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2841 PtrInfo: MachinePointerInfo());
2842 // Store the hi of the constructed double.
2843 SDValue HiPtr =
2844 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2845 SDValue Store2 =
2846 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2847 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2848
2849 // load the constructed double
2850 SDValue Load =
2851 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2852 // FP constant to bias correct the final result
2853 SDValue Bias = DAG.getConstantFP(
2854 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2855 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2856 DL: dl, VT: MVT::f64);
2857 // Subtract the bias and get the final result.
2858 SDValue Sub;
2859 SDValue Result;
2860 if (Node->isStrictFPOpcode()) {
2861 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2862 Ops: {Node->getOperand(Num: 0), Load, Bias});
2863 Chain = Sub.getValue(R: 1);
2864 if (DestVT != Sub.getValueType()) {
2865 std::pair<SDValue, SDValue> ResultPair;
2866 ResultPair =
2867 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2868 Result = ResultPair.first;
2869 Chain = ResultPair.second;
2870 }
2871 else
2872 Result = Sub;
2873 } else {
2874 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2875 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2876 }
2877 return Result;
2878 }
2879
2880 if (isSigned)
2881 return SDValue();
2882
2883 // TODO: Generalize this for use with other types.
2884 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2885 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2886 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2887 // For unsigned conversions, convert them to signed conversions using the
2888 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2889 // should be valid for i32->f32 as well.
2890
2891 // More generally this transform should be valid if there are 3 more bits
2892 // in the integer type than the significand. Rounding uses the first bit
2893 // after the width of the significand and the OR of all bits after that. So
2894 // we need to be able to OR the shifted out bit into one of the bits that
2895 // participate in the OR.
2896
2897 // TODO: This really should be implemented using a branch rather than a
2898 // select. We happen to get lucky and machinesink does the right
2899 // thing most of the time. This would be a good candidate for a
2900 // pseudo-op, or, even better, for whole-function isel.
2901 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2902
2903 SDValue SignBitTest = DAG.getSetCC(
2904 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2905
2906 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL: dl);
2907 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2908 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2909 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2910 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2911
2912 SDValue Slow, Fast;
2913 if (Node->isStrictFPOpcode()) {
2914 // In strict mode, we must avoid spurious exceptions, and therefore
2915 // must make sure to only emit a single STRICT_SINT_TO_FP.
2916 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2917 // The STRICT_SINT_TO_FP inherits the exception mode from the
2918 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2919 // never raise any exception.
2920 SDNodeFlags Flags;
2921 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2922 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: {DestVT, MVT::Other},
2923 Ops: {Node->getOperand(Num: 0), InCvt}, Flags);
2924 Flags.setNoFPExcept(true);
2925 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: {DestVT, MVT::Other},
2926 Ops: {Fast.getValue(R: 1), Fast, Fast}, Flags);
2927 Chain = Slow.getValue(R: 1);
2928 } else {
2929 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2930 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2931 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2932 }
2933
2934 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2935 }
2936
2937 // Don't expand it if there isn't cheap fadd.
2938 if (!TLI.isOperationLegalOrCustom(
2939 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2940 return SDValue();
2941
2942 // The following optimization is valid only if every value in SrcVT (when
2943 // treated as signed) is representable in DestVT. Check that the mantissa
2944 // size of DestVT is >= than the number of bits in SrcVT -1.
2945 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2946 SrcVT.getSizeInBits() - 1 &&
2947 "Cannot perform lossless SINT_TO_FP!");
2948
2949 SDValue Tmp1;
2950 if (Node->isStrictFPOpcode()) {
2951 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2952 Ops: { Node->getOperand(Num: 0), Op0 });
2953 } else
2954 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2955
2956 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2957 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2958 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2959 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2960 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2961 Cond: SignSet, LHS: Four, RHS: Zero);
2962
2963 // If the sign bit of the integer is set, the large number will be treated
2964 // as a negative number. To counteract this, the dynamic code adds an
2965 // offset depending on the data type.
2966 uint64_t FF;
2967 switch (SrcVT.getSimpleVT().SimpleTy) {
2968 default:
2969 return SDValue();
2970 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2971 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2972 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2973 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2974 }
2975 if (DAG.getDataLayout().isLittleEndian())
2976 FF <<= 32;
2977 Constant *FudgeFactor = ConstantInt::get(
2978 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2979
2980 SDValue CPIdx =
2981 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2982 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2983 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2984 Alignment = commonAlignment(A: Alignment, Offset: 4);
2985 SDValue FudgeInReg;
2986 if (DestVT == MVT::f32)
2987 FudgeInReg = DAG.getLoad(
2988 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2989 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2990 Alignment);
2991 else {
2992 SDValue Load = DAG.getExtLoad(
2993 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2994 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
2995 Alignment);
2996 HandleSDNode Handle(Load);
2997 LegalizeOp(Node: Load.getNode());
2998 FudgeInReg = Handle.getValue();
2999 }
3000
3001 if (Node->isStrictFPOpcode()) {
3002 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
3003 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
3004 Chain = Result.getValue(R: 1);
3005 return Result;
3006 }
3007
3008 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
3009}
3010
3011/// This function is responsible for legalizing a
3012/// *INT_TO_FP operation of the specified operand when the target requests that
3013/// we promote it. At this point, we know that the result and operand types are
3014/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
3015/// operation that takes a larger input.
3016void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
3017 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
3018 bool IsStrict = N->isStrictFPOpcode();
3019 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
3020 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
3021 EVT DestVT = N->getValueType(ResNo: 0);
3022 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3023 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
3024 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
3025
3026 // First step, figure out the appropriate *INT_TO_FP operation to use.
3027 EVT NewInTy = LegalOp.getValueType();
3028
3029 unsigned OpToUse = 0;
3030
3031 // Scan for the appropriate larger type to use.
3032 while (true) {
3033 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
3034 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3035
3036 // If the target supports SINT_TO_FP of this type, use it.
3037 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
3038 OpToUse = SIntOp;
3039 break;
3040 }
3041 if (IsSigned)
3042 continue;
3043
3044 // If the target supports UINT_TO_FP of this type, use it.
3045 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
3046 OpToUse = UIntOp;
3047 break;
3048 }
3049
3050 // Otherwise, try a larger type.
3051 }
3052
3053 // Okay, we found the operation and type to use. Zero extend our input to the
3054 // desired type then run the operation on it.
3055 if (IsStrict) {
3056 SDValue Res =
3057 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
3058 Ops: {N->getOperand(Num: 0),
3059 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3060 DL: dl, VT: NewInTy, Operand: LegalOp)});
3061 Results.push_back(Elt: Res);
3062 Results.push_back(Elt: Res.getValue(R: 1));
3063 return;
3064 }
3065
3066 Results.push_back(
3067 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
3068 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3069 DL: dl, VT: NewInTy, Operand: LegalOp)));
3070}
3071
3072/// This function is responsible for legalizing a
3073/// FP_TO_*INT operation of the specified operand when the target requests that
3074/// we promote it. At this point, we know that the result and operand types are
3075/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
3076/// operation that returns a larger result.
3077void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3078 SmallVectorImpl<SDValue> &Results) {
3079 bool IsStrict = N->isStrictFPOpcode();
3080 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3081 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3082 EVT DestVT = N->getValueType(ResNo: 0);
3083 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3084 // First step, figure out the appropriate FP_TO*INT operation to use.
3085 EVT NewOutTy = DestVT;
3086
3087 unsigned OpToUse = 0;
3088
3089 // Scan for the appropriate larger type to use.
3090 while (true) {
3091 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
3092 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3093
3094 // A larger signed type can hold all unsigned values of the requested type,
3095 // so using FP_TO_SINT is valid
3096 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
3097 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3098 break;
3099
3100 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
3101 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
3102 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3103 break;
3104
3105 // Otherwise, try a larger type.
3106 }
3107
3108 // Okay, we found the operation and type to use.
3109 SDValue Operation;
3110 if (IsStrict) {
3111 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
3112 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
3113 } else
3114 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
3115
3116 // Truncate the result of the extended FP_TO_*INT operation to the desired
3117 // size.
3118 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
3119 Results.push_back(Elt: Trunc);
3120 if (IsStrict)
3121 Results.push_back(Elt: Operation.getValue(R: 1));
3122}
3123
3124/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
3125/// the result and operand types are legal and there must be a legal
3126/// FP_TO_*INT_SAT operation for a larger result type.
3127SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3128 const SDLoc &dl) {
3129 unsigned Opcode = Node->getOpcode();
3130
3131 // Scan for the appropriate larger type to use.
3132 EVT NewOutTy = Node->getValueType(ResNo: 0);
3133 while (true) {
3134 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
3135 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3136
3137 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
3138 break;
3139 }
3140
3141 // Saturation width is determined by second operand, so we don't have to
3142 // perform any fixup and can directly truncate the result.
3143 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
3144 N2: Node->getOperand(Num: 1));
3145 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
3146}
3147
3148/// Open code the operations for PARITY of the specified operation.
3149SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3150 EVT VT = Op.getValueType();
3151 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3152 unsigned Sz = VT.getScalarSizeInBits();
3153
3154 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3155 SDValue Result;
3156 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3157 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3158 } else {
3159 Result = Op;
3160 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3161 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3162 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3163 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3164 }
3165 }
3166
3167 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3168}
3169
3170SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3171 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3172 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3173 : Node->getOperand(Num: 0).getSimpleValueType();
3174 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3175 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3176 MVT NewScalarVT = NewVecVT.getVectorElementType();
3177
3178 SDLoc DL(Node);
3179 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3180
3181 // FIXME: Support integer.
3182 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3183 "Only FP promotion is supported");
3184
3185 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3186 if (Node->getOperand(Num: j).getValueType().isVector() &&
3187 !(IsVPOpcode &&
3188 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3189 // promote the vector operand.
3190 // FIXME: Support integer.
3191 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3192 "Only FP promotion is supported");
3193 Operands[j] =
3194 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3195 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3196 // promote the initial value.
3197 Operands[j] =
3198 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3199 } else {
3200 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3201 }
3202
3203 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3204 Flags: Node->getFlags());
3205
3206 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3207 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3208 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3209}
3210
3211bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3212 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3213 SmallVector<SDValue, 8> Results;
3214 SDLoc dl(Node);
3215 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3216 bool NeedInvert;
3217 switch (Node->getOpcode()) {
3218 case ISD::ABS:
3219 case ISD::ABS_MIN_POISON:
3220 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3221 Results.push_back(Elt: Tmp1);
3222 break;
3223 case ISD::ABDS:
3224 case ISD::ABDU:
3225 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3226 Results.push_back(Elt: Tmp1);
3227 break;
3228 case ISD::AVGCEILS:
3229 case ISD::AVGCEILU:
3230 case ISD::AVGFLOORS:
3231 case ISD::AVGFLOORU:
3232 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3233 Results.push_back(Elt: Tmp1);
3234 break;
3235 case ISD::CTPOP:
3236 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3237 Results.push_back(Elt: Tmp1);
3238 break;
3239 case ISD::CTLZ:
3240 case ISD::CTLZ_ZERO_POISON:
3241 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3242 Results.push_back(Elt: Tmp1);
3243 break;
3244 case ISD::CTLS:
3245 if ((Tmp1 = TLI.expandCTLS(N: Node, DAG)))
3246 Results.push_back(Elt: Tmp1);
3247 break;
3248 case ISD::CTTZ:
3249 case ISD::CTTZ_ZERO_POISON:
3250 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3251 Results.push_back(Elt: Tmp1);
3252 break;
3253 case ISD::BITREVERSE:
3254 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3255 Results.push_back(Elt: Tmp1);
3256 break;
3257 case ISD::BSWAP:
3258 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3259 Results.push_back(Elt: Tmp1);
3260 break;
3261 case ISD::PARITY:
3262 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3263 break;
3264 case ISD::FRAMEADDR:
3265 case ISD::RETURNADDR:
3266 case ISD::FRAME_TO_ARGS_OFFSET:
3267 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3268 break;
3269 case ISD::EH_DWARF_CFA: {
3270 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3271 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3272 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3273 VT: CfaArg.getValueType(),
3274 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3275 VT: CfaArg.getValueType()),
3276 N2: CfaArg);
3277 SDValue FA = DAG.getNode(
3278 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3279 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3280 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3281 N1: FA, N2: Offset));
3282 break;
3283 }
3284 case ISD::GET_ROUNDING:
3285 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3286 Results.push_back(Elt: Node->getOperand(Num: 0));
3287 break;
3288 case ISD::EH_RETURN:
3289 case ISD::PREFETCH:
3290 case ISD::VAEND:
3291 case ISD::EH_SJLJ_LONGJMP:
3292 // If the target didn't expand these, there's nothing to do, so just
3293 // preserve the chain and be done.
3294 Results.push_back(Elt: Node->getOperand(Num: 0));
3295 break;
3296 case ISD::READCYCLECOUNTER:
3297 case ISD::READSTEADYCOUNTER:
3298 // If the target didn't expand this, just return 'zero' and preserve the
3299 // chain.
3300 Results.append(NumInputs: Node->getNumValues() - 1,
3301 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3302 Results.push_back(Elt: Node->getOperand(Num: 0));
3303 break;
3304 case ISD::EH_SJLJ_SETJMP:
3305 // If the target didn't expand this, just return 'zero' and preserve the
3306 // chain.
3307 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3308 Results.push_back(Elt: Node->getOperand(Num: 0));
3309 break;
3310 case ISD::ATOMIC_LOAD: {
3311 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3312 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3313 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3314 SDValue Swap = DAG.getAtomicCmpSwap(
3315 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3316 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3317 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3318 Results.push_back(Elt: Swap.getValue(R: 0));
3319 Results.push_back(Elt: Swap.getValue(R: 1));
3320 break;
3321 }
3322 case ISD::ATOMIC_STORE: {
3323 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3324 SDValue Swap = DAG.getAtomic(
3325 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3326 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3327 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3328 Results.push_back(Elt: Swap.getValue(R: 1));
3329 break;
3330 }
3331 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3332 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3333 // splits out the success value as a comparison. Expanding the resulting
3334 // ATOMIC_CMP_SWAP will produce a libcall.
3335 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3336 SDValue Res = DAG.getAtomicCmpSwap(
3337 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3338 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3339 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3340
3341 SDValue ExtRes = Res;
3342 SDValue LHS = Res;
3343 SDValue RHS = Node->getOperand(Num: 1);
3344
3345 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3346 EVT OuterType = Node->getValueType(ResNo: 0);
3347 switch (TLI.getExtendForAtomicOps()) {
3348 case ISD::SIGN_EXTEND:
3349 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3350 N2: DAG.getValueType(AtomicType));
3351 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3352 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3353 ExtRes = LHS;
3354 break;
3355 case ISD::ZERO_EXTEND:
3356 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3357 N2: DAG.getValueType(AtomicType));
3358 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3359 ExtRes = LHS;
3360 break;
3361 case ISD::ANY_EXTEND:
3362 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3363 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3364 break;
3365 default:
3366 llvm_unreachable("Invalid atomic op extension");
3367 }
3368
3369 SDValue Success =
3370 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3371
3372 Results.push_back(Elt: ExtRes.getValue(R: 0));
3373 Results.push_back(Elt: Success);
3374 Results.push_back(Elt: Res.getValue(R: 1));
3375 break;
3376 }
3377 case ISD::ATOMIC_LOAD_SUB: {
3378 SDLoc DL(Node);
3379 EVT VT = Node->getValueType(ResNo: 0);
3380 SDValue RHS = Node->getOperand(Num: 2);
3381 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3382 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3383 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3384 RHS = RHS->getOperand(Num: 0);
3385 SDValue NewRHS =
3386 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3387 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3388 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3389 Val: NewRHS, MMO: AN->getMemOperand());
3390 Results.push_back(Elt: Res);
3391 Results.push_back(Elt: Res.getValue(R: 1));
3392 break;
3393 }
3394 case ISD::ATOMIC_LOAD_FSUB: {
3395 SDLoc DL(Node);
3396 EVT VT = Node->getValueType(ResNo: 0);
3397 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3398 SDValue NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Node->getOperand(Num: 2));
3399 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_FADD, dl: DL, MemVT: AN->getMemoryVT(),
3400 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3401 Val: NewRHS, MMO: AN->getMemOperand());
3402 Results.push_back(Elt: Res);
3403 Results.push_back(Elt: Res.getValue(R: 1));
3404 break;
3405 }
3406 case ISD::DYNAMIC_STACKALLOC:
3407 ExpandDYNAMIC_STACKALLOC(Node, Results);
3408 break;
3409 case ISD::MERGE_VALUES:
3410 for (unsigned i = 0; i < Node->getNumValues(); i++)
3411 Results.push_back(Elt: Node->getOperand(Num: i));
3412 break;
3413 case ISD::POISON:
3414 case ISD::UNDEF: {
3415 EVT VT = Node->getValueType(ResNo: 0);
3416 if (VT.isInteger())
3417 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3418 else {
3419 assert(VT.isFloatingPoint() && "Unknown value type!");
3420 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3421 }
3422 break;
3423 }
3424 case ISD::STRICT_FP_ROUND:
3425 // When strict mode is enforced we can't do expansion because it
3426 // does not honor the "strict" properties. Only libcall is allowed.
3427 if (TLI.isStrictFPEnabled())
3428 break;
3429 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3430 // since this operation is more efficient than stack operation.
3431 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3432 VT: Node->getValueType(ResNo: 0))
3433 == TargetLowering::Legal)
3434 break;
3435 // We fall back to use stack operation when the FP_ROUND operation
3436 // isn't available.
3437 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3438 DestVT: Node->getValueType(ResNo: 0), dl,
3439 Chain: Node->getOperand(Num: 0)))) {
3440 ReplaceNode(Old: Node, New: Tmp1.getNode());
3441 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3442 return true;
3443 }
3444 break;
3445 case ISD::FP_ROUND: {
3446 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3447 Results.push_back(Elt: Tmp1);
3448 break;
3449 }
3450
3451 [[fallthrough]];
3452 }
3453 case ISD::BITCAST:
3454 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3455 DestVT: Node->getValueType(ResNo: 0), dl)))
3456 Results.push_back(Elt: Tmp1);
3457 break;
3458 case ISD::STRICT_FP_EXTEND:
3459 // When strict mode is enforced we can't do expansion because it
3460 // does not honor the "strict" properties. Only libcall is allowed.
3461 if (TLI.isStrictFPEnabled())
3462 break;
3463 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3464 // since this operation is more efficient than stack operation.
3465 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3466 VT: Node->getValueType(ResNo: 0))
3467 == TargetLowering::Legal)
3468 break;
3469 // We fall back to use stack operation when the FP_EXTEND operation
3470 // isn't available.
3471 if ((Tmp1 = EmitStackConvert(
3472 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3473 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3474 ReplaceNode(Old: Node, New: Tmp1.getNode());
3475 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3476 return true;
3477 }
3478 break;
3479 case ISD::FP_EXTEND: {
3480 SDValue Op = Node->getOperand(Num: 0);
3481 EVT SrcVT = Op.getValueType();
3482 EVT DstVT = Node->getValueType(ResNo: 0);
3483 if (SrcVT.getScalarType() == MVT::bf16) {
3484 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3485 break;
3486 }
3487
3488 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3489 Results.push_back(Elt: Tmp1);
3490 break;
3491 }
3492 case ISD::BF16_TO_FP: {
3493 // Always expand bf16 to f32 casts, they lower to ext + shift.
3494 //
3495 // Note that the operand of this code can be bf16 or an integer type in case
3496 // bf16 is not supported on the target and was softened.
3497 SDValue Op = Node->getOperand(Num: 0);
3498 if (Op.getValueType() == MVT::bf16) {
3499 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3500 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3501 } else {
3502 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3503 }
3504 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3505 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3506 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3507 // Add fp_extend in case the output is bigger than f32.
3508 if (Node->getValueType(ResNo: 0) != MVT::f32)
3509 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3510 Results.push_back(Elt: Op);
3511 break;
3512 }
3513 case ISD::FP_TO_BF16: {
3514 SDValue Op = Node->getOperand(Num: 0);
3515 if (Op.getValueType() != MVT::f32)
3516 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3517 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3518 // Certain SNaNs will turn into infinities if we do a simple shift right.
3519 if (!DAG.isKnownNeverSNaN(Op)) {
3520 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3521 }
3522 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
3523 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3524 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3525 // The result of this node can be bf16 or an integer type in case bf16 is
3526 // not supported on the target and was softened to i16 for storage.
3527 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3528 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3529 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3530 } else {
3531 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3532 }
3533 Results.push_back(Elt: Op);
3534 break;
3535 }
3536 case ISD::CONVERT_FROM_ARBITRARY_FP: {
3537 // Expand conversion from arbitrary FP format stored in an integer to a
3538 // native IEEE float type using integer bit manipulation.
3539 //
3540 // TODO: currently only conversions from FP4, FP6 and FP8 formats from OCP
3541 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3542 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3543 // Float8E8M0FNU.
3544 EVT DstVT = Node->getValueType(ResNo: 0);
3545 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
3546 Results.push_back(Elt: Expanded);
3547 else
3548 Results.push_back(Elt: DAG.getPOISON(VT: DstVT));
3549 break;
3550 }
3551 case ISD::CONVERT_TO_ARBITRARY_FP: {
3552 // Expand conversion from a native IEEE float type to an arbitrary FP
3553 // format, returning the result as an integer using bit manipulation.
3554 //
3555 // TODO: currently only conversions to FP4, FP6 and FP8 formats from OCP
3556 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3557 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3558 // Float8E8M0FNU.
3559 EVT ResVT = Node->getValueType(ResNo: 0);
3560 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
3561 Results.push_back(Elt: Expanded);
3562 else
3563 Results.push_back(Elt: DAG.getPOISON(VT: ResVT));
3564 break;
3565 }
3566 case ISD::FCANONICALIZE: {
3567 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
3568 Results.push_back(Elt: Mul);
3569 break;
3570 }
3571 case ISD::SIGN_EXTEND_INREG: {
3572 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3573 EVT VT = Node->getValueType(ResNo: 0);
3574
3575 // An in-register sign-extend of a boolean is a negation:
3576 // 'true' (1) sign-extended is -1.
3577 // 'false' (0) sign-extended is 0.
3578 // However, we must mask the high bits of the source operand because the
3579 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3580
3581 // TODO: Do this for vectors too?
3582 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3583 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3584 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3585 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3586 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3587 Results.push_back(Elt: Neg);
3588 break;
3589 }
3590
3591 // NOTE: we could fall back on load/store here too for targets without
3592 // SRA. However, it is doubtful that any exist.
3593 unsigned BitsDiff = VT.getScalarSizeInBits() -
3594 ExtraVT.getScalarSizeInBits();
3595 SDValue ShiftCst = DAG.getShiftAmountConstant(Val: BitsDiff, VT, DL: dl);
3596 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: ShiftCst);
3597 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Tmp1, N2: ShiftCst);
3598 Results.push_back(Elt: Tmp1);
3599 break;
3600 }
3601 case ISD::UINT_TO_FP:
3602 case ISD::STRICT_UINT_TO_FP:
3603 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3604 Results.push_back(Elt: Tmp1);
3605 if (Node->isStrictFPOpcode())
3606 Results.push_back(Elt: Tmp2);
3607 break;
3608 }
3609 [[fallthrough]];
3610 case ISD::SINT_TO_FP:
3611 case ISD::STRICT_SINT_TO_FP:
3612 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3613 Results.push_back(Elt: Tmp1);
3614 if (Node->isStrictFPOpcode())
3615 Results.push_back(Elt: Tmp2);
3616 }
3617 break;
3618 case ISD::FP_TO_SINT:
3619 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3620 Results.push_back(Elt: Tmp1);
3621 break;
3622 case ISD::STRICT_FP_TO_SINT:
3623 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3624 ReplaceNode(Old: Node, New: Tmp1.getNode());
3625 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3626 return true;
3627 }
3628 break;
3629 case ISD::FP_TO_UINT:
3630 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3631 Results.push_back(Elt: Tmp1);
3632 break;
3633 case ISD::STRICT_FP_TO_UINT:
3634 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3635 // Relink the chain.
3636 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3637 // Replace the new UINT result.
3638 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3639 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3640 return true;
3641 }
3642 break;
3643 case ISD::FP_TO_SINT_SAT:
3644 case ISD::FP_TO_UINT_SAT:
3645 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3646 break;
3647 case ISD::LROUND:
3648 case ISD::LLROUND: {
3649 SDValue Arg = Node->getOperand(Num: 0);
3650 EVT ArgVT = Arg.getValueType();
3651 EVT ResVT = Node->getValueType(ResNo: 0);
3652 SDLoc dl(Node);
3653 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3654 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3655 break;
3656 }
3657 case ISD::VAARG:
3658 Results.push_back(Elt: DAG.expandVAArg(Node));
3659 Results.push_back(Elt: Results[0].getValue(R: 1));
3660 break;
3661 case ISD::VACOPY:
3662 Results.push_back(Elt: DAG.expandVACopy(Node));
3663 break;
3664 case ISD::EXTRACT_VECTOR_ELT:
3665 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3666 // This must be an access of the only element. Return it.
3667 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3668 Operand: Node->getOperand(Num: 0));
3669 else
3670 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3671 Results.push_back(Elt: Tmp1);
3672 break;
3673 case ISD::EXTRACT_SUBVECTOR:
3674 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3675 break;
3676 case ISD::INSERT_SUBVECTOR:
3677 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3678 break;
3679 case ISD::CONCAT_VECTORS:
3680 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3681 VectorValueType.isScalableVector() ||
3682 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3683 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3684 else
3685 Results.push_back(Elt: ExpandConcatVectors(Node));
3686 break;
3687 case ISD::SCALAR_TO_VECTOR:
3688 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3689 break;
3690 case ISD::INSERT_VECTOR_ELT:
3691 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3692 break;
3693 case ISD::VECTOR_SHUFFLE: {
3694 SmallVector<int, 32> NewMask;
3695 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3696
3697 EVT VT = Node->getValueType(ResNo: 0);
3698 EVT EltVT = VT.getVectorElementType();
3699 SDValue Op0 = Node->getOperand(Num: 0);
3700 SDValue Op1 = Node->getOperand(Num: 1);
3701 if (!TLI.isTypeLegal(VT: EltVT)) {
3702 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3703
3704 // BUILD_VECTOR operands are allowed to be wider than the element type.
3705 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3706 // it.
3707 if (NewEltVT.bitsLT(VT: EltVT)) {
3708 // Convert shuffle node.
3709 // If original node was v4i64 and the new EltVT is i32,
3710 // cast operands to v8i32 and re-build the mask.
3711
3712 // Calculate new VT, the size of the new VT should be equal to original.
3713 EVT NewVT =
3714 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3715 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3716 assert(NewVT.bitsEq(VT));
3717
3718 // cast operands to new VT
3719 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3720 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3721
3722 // Convert the shuffle mask
3723 unsigned int factor =
3724 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3725
3726 // EltVT gets smaller
3727 assert(factor > 0);
3728
3729 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3730 if (Mask[i] < 0) {
3731 for (unsigned fi = 0; fi < factor; ++fi)
3732 NewMask.push_back(Elt: Mask[i]);
3733 }
3734 else {
3735 for (unsigned fi = 0; fi < factor; ++fi)
3736 NewMask.push_back(Elt: Mask[i]*factor+fi);
3737 }
3738 }
3739 Mask = NewMask;
3740 VT = NewVT;
3741 }
3742 EltVT = NewEltVT;
3743 }
3744 unsigned NumElems = VT.getVectorNumElements();
3745 SmallVector<SDValue, 16> Ops;
3746 for (unsigned i = 0; i != NumElems; ++i) {
3747 if (Mask[i] < 0) {
3748 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3749 continue;
3750 }
3751 unsigned Idx = Mask[i];
3752 if (Idx < NumElems)
3753 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3754 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3755 else
3756 Ops.push_back(
3757 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3758 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3759 }
3760
3761 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3762 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3763 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3764 Results.push_back(Elt: Tmp1);
3765 break;
3766 }
3767 case ISD::VECTOR_SPLICE_LEFT:
3768 case ISD::VECTOR_SPLICE_RIGHT: {
3769 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3770 break;
3771 }
3772 case ISD::VECTOR_DEINTERLEAVE: {
3773 unsigned Factor = Node->getNumOperands();
3774 if (Factor <= 2 || Factor % 2 != 0)
3775 break;
3776 SmallVector<SDValue, 8> Ops(Node->ops());
3777 EVT VecVT = Node->getValueType(ResNo: 0);
3778 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3779 // Deinterleave at Factor/2 so each result contains two factors interleaved:
3780 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
3781 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3782 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
3783 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3784 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
3785 Results.resize(N: Factor);
3786 // Deinterleave the 2 factors out:
3787 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
3788 for (unsigned I = 0; I < Factor / 2; I++) {
3789 SDValue Deinterleave =
3790 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3791 Ops: {L.getValue(R: I), R.getValue(R: I)});
3792 Results[I] = Deinterleave.getValue(R: 0);
3793 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
3794 }
3795 break;
3796 }
3797 case ISD::VECTOR_INTERLEAVE: {
3798 unsigned Factor = Node->getNumOperands();
3799 if (Factor <= 2 || Factor % 2 != 0)
3800 break;
3801 EVT VecVT = Node->getValueType(ResNo: 0);
3802 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3803 SmallVector<SDValue, 8> LOps, ROps;
3804 // Interleave so we have 2 factors per result:
3805 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
3806 for (unsigned I = 0; I < Factor / 2; I++) {
3807 SDValue Interleave =
3808 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3809 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
3810 LOps.push_back(Elt: Interleave.getValue(R: 0));
3811 ROps.push_back(Elt: Interleave.getValue(R: 1));
3812 }
3813 // Interleave at Factor/2:
3814 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
3815 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: LOps);
3816 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: ROps);
3817 for (unsigned I = 0; I < Factor / 2; I++)
3818 Results.push_back(Elt: L.getValue(R: I));
3819 for (unsigned I = 0; I < Factor / 2; I++)
3820 Results.push_back(Elt: R.getValue(R: I));
3821 break;
3822 }
3823 case ISD::EXTRACT_ELEMENT: {
3824 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3825 if (Node->getConstantOperandVal(Num: 1)) {
3826 // 1 -> Hi
3827 Tmp1 = DAG.getNode(
3828 Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3829 N2: DAG.getShiftAmountConstant(Val: OpTy.getSizeInBits() / 2, VT: OpTy, DL: dl));
3830 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3831 } else {
3832 // 0 -> Lo
3833 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3834 Operand: Node->getOperand(Num: 0));
3835 }
3836 Results.push_back(Elt: Tmp1);
3837 break;
3838 }
3839 case ISD::STACKADDRESS:
3840 case ISD::STACKSAVE:
3841 // Expand to CopyFromReg if the target set
3842 // StackPointerRegisterToSaveRestore.
3843 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3844 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3845 VT: Node->getValueType(ResNo: 0)));
3846 Results.push_back(Elt: Results[0].getValue(R: 1));
3847 } else {
3848 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3849 Results.push_back(Elt: Node->getOperand(Num: 0));
3850
3851 StringRef IntrinsicName = Node->getOpcode() == ISD::STACKADDRESS
3852 ? "llvm.stackaddress"
3853 : "llvm.stacksave";
3854 DAG.getContext()->diagnose(DI: DiagnosticInfoLegalizationFailure(
3855 Twine(IntrinsicName) + " is not supported on this target.",
3856 DAG.getMachineFunction().getFunction(), dl.getDebugLoc()));
3857 }
3858 break;
3859 case ISD::STACKRESTORE:
3860 // Expand to CopyToReg if the target set
3861 // StackPointerRegisterToSaveRestore.
3862 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3863 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3864 N: Node->getOperand(Num: 1)));
3865 } else {
3866 Results.push_back(Elt: Node->getOperand(Num: 0));
3867 }
3868 break;
3869 case ISD::GET_DYNAMIC_AREA_OFFSET:
3870 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3871 Results.push_back(Elt: Results[0].getValue(R: 0));
3872 break;
3873 case ISD::FCOPYSIGN:
3874 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3875 break;
3876 case ISD::FNEG:
3877 Results.push_back(Elt: ExpandFNEG(Node));
3878 break;
3879 case ISD::FABS:
3880 Results.push_back(Elt: ExpandFABS(Node));
3881 break;
3882 case ISD::IS_FPCLASS: {
3883 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3884 if (SDValue Expanded =
3885 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3886 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3887 Results.push_back(Elt: Expanded);
3888 break;
3889 }
3890 case ISD::SMIN:
3891 case ISD::SMAX:
3892 case ISD::UMIN:
3893 case ISD::UMAX: {
3894 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3895 ISD::CondCode Pred;
3896 switch (Node->getOpcode()) {
3897 default: llvm_unreachable("How did we get here?");
3898 case ISD::SMAX: Pred = ISD::SETGT; break;
3899 case ISD::SMIN: Pred = ISD::SETLT; break;
3900 case ISD::UMAX: Pred = ISD::SETUGT; break;
3901 case ISD::UMIN: Pred = ISD::SETULT; break;
3902 }
3903 Tmp1 = Node->getOperand(Num: 0);
3904 Tmp2 = Node->getOperand(Num: 1);
3905 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3906 Results.push_back(Elt: Tmp1);
3907 break;
3908 }
3909 case ISD::FMINNUM:
3910 case ISD::FMAXNUM: {
3911 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3912 Results.push_back(Elt: Expanded);
3913 break;
3914 }
3915 case ISD::FMINIMUM:
3916 case ISD::FMAXIMUM: {
3917 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
3918 Results.push_back(Elt: Expanded);
3919 break;
3920 }
3921 case ISD::FMINIMUMNUM:
3922 case ISD::FMAXIMUMNUM: {
3923 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
3924 break;
3925 }
3926 case ISD::FSIN:
3927 case ISD::FCOS: {
3928 EVT VT = Node->getValueType(ResNo: 0);
3929 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3930 // fcos which share the same operand and both are used.
3931 if ((TLI.isOperationLegal(Op: ISD::FSINCOS, VT) ||
3932 isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls())) &&
3933 useSinCos(Node)) {
3934 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3935 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3936 if (Node->getOpcode() == ISD::FCOS)
3937 Tmp1 = Tmp1.getValue(R: 1);
3938 Results.push_back(Elt: Tmp1);
3939 }
3940 break;
3941 }
3942 case ISD::FLDEXP:
3943 case ISD::STRICT_FLDEXP: {
3944 EVT VT = Node->getValueType(ResNo: 0);
3945 RTLIB::Libcall LC = RTLIB::getLDEXP(VT);
3946 // Use the LibCall instead, it is very likely faster
3947 // FIXME: Use separate LibCall action.
3948 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3949 break;
3950
3951 if (SDValue Expanded = expandLdexp(Node)) {
3952 Results.push_back(Elt: Expanded);
3953 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3954 Results.push_back(Elt: Expanded.getValue(R: 1));
3955 }
3956
3957 break;
3958 }
3959 case ISD::FFREXP: {
3960 RTLIB::Libcall LC = RTLIB::getFREXP(VT: Node->getValueType(ResNo: 0));
3961 // Use the LibCall instead, it is very likely faster
3962 // FIXME: Use separate LibCall action.
3963 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3964 break;
3965
3966 if (SDValue Expanded = expandFrexp(Node)) {
3967 Results.push_back(Elt: Expanded);
3968 Results.push_back(Elt: Expanded.getValue(R: 1));
3969 }
3970 break;
3971 }
3972 case ISD::FMODF: {
3973 RTLIB::Libcall LC = RTLIB::getMODF(VT: Node->getValueType(ResNo: 0));
3974 // Use the LibCall instead, it is very likely faster
3975 // FIXME: Use separate LibCall action.
3976 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3977 break;
3978
3979 if (SDValue Expanded = expandModf(Node)) {
3980 Results.push_back(Elt: Expanded);
3981 Results.push_back(Elt: Expanded.getValue(R: 1));
3982 }
3983 break;
3984 }
3985 case ISD::FSINCOS: {
3986 if (isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls()))
3987 break;
3988 EVT VT = Node->getValueType(ResNo: 0);
3989 SDValue Op = Node->getOperand(Num: 0);
3990 SDNodeFlags Flags = Node->getFlags();
3991 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
3992 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
3993 Results.append(IL: {Tmp1, Tmp2});
3994 break;
3995 }
3996 case ISD::FMAD:
3997 llvm_unreachable("Illegal fmad should never be formed");
3998
3999 case ISD::FP16_TO_FP:
4000 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4001 // We can extend to types bigger than f32 in two steps without changing
4002 // the result. Since "f16 -> f32" is much more commonly available, give
4003 // CodeGen the option of emitting that before resorting to a libcall.
4004 SDValue Res =
4005 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
4006 Results.push_back(
4007 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
4008 }
4009 break;
4010 case ISD::STRICT_BF16_TO_FP:
4011 case ISD::STRICT_FP16_TO_FP:
4012 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4013 // We can extend to types bigger than f32 in two steps without changing
4014 // the result. Since "f16 -> f32" is much more commonly available, give
4015 // CodeGen the option of emitting that before resorting to a libcall.
4016 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
4017 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
4018 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
4019 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
4020 Ops: {Res.getValue(R: 1), Res});
4021 Results.push_back(Elt: Res);
4022 Results.push_back(Elt: Res.getValue(R: 1));
4023 }
4024 break;
4025 case ISD::FP_TO_FP16:
4026 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
4027 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
4028 SDValue Op = Node->getOperand(Num: 0);
4029 MVT SVT = Op.getSimpleValueType();
4030 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
4031 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
4032 // Under fastmath, we can expand this node into a fround followed by
4033 // a float-half conversion.
4034 SDValue FloatVal =
4035 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
4036 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
4037 Results.push_back(
4038 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
4039 }
4040 }
4041 break;
4042 case ISD::ConstantFP: {
4043 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
4044 // Check to see if this FP immediate is already legal.
4045 // If this is a legal constant, turn it into a TargetConstantFP node.
4046 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
4047 ForCodeSize: DAG.shouldOptForSize()))
4048 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
4049 break;
4050 }
4051 case ISD::Constant: {
4052 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
4053 Results.push_back(Elt: ExpandConstant(CP));
4054 break;
4055 }
4056 case ISD::FSUB: {
4057 EVT VT = Node->getValueType(ResNo: 0);
4058 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
4059 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
4060 const SDNodeFlags Flags = Node->getFlags();
4061 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
4062 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
4063 Results.push_back(Elt: Tmp1);
4064 }
4065 break;
4066 }
4067 case ISD::SUB: {
4068 EVT VT = Node->getValueType(ResNo: 0);
4069 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
4070 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
4071 "Don't know how to expand this subtraction!");
4072 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
4073 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
4074 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
4075 break;
4076 }
4077 case ISD::UREM:
4078 case ISD::SREM:
4079 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
4080 Results.push_back(Elt: Tmp1);
4081 break;
4082 case ISD::UDIV:
4083 case ISD::SDIV: {
4084 bool isSigned = Node->getOpcode() == ISD::SDIV;
4085 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4086 EVT VT = Node->getValueType(ResNo: 0);
4087 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
4088 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4089 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4090 N2: Node->getOperand(Num: 1));
4091 Results.push_back(Elt: Tmp1);
4092 }
4093 break;
4094 }
4095 case ISD::MULHU:
4096 case ISD::MULHS: {
4097 unsigned ExpandOpcode =
4098 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
4099 EVT VT = Node->getValueType(ResNo: 0);
4100 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4101
4102 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4103 N2: Node->getOperand(Num: 1));
4104 Results.push_back(Elt: Tmp1.getValue(R: 1));
4105 break;
4106 }
4107 case ISD::UMUL_LOHI:
4108 case ISD::SMUL_LOHI: {
4109 SDValue LHS = Node->getOperand(Num: 0);
4110 SDValue RHS = Node->getOperand(Num: 1);
4111 EVT VT = LHS.getValueType();
4112 bool IsSigned = Node->getOpcode() == ISD::SMUL_LOHI;
4113 unsigned MULHOpcode = IsSigned ? ISD::MULHS : ISD::MULHU;
4114
4115 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
4116 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
4117 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
4118 break;
4119 }
4120
4121 SmallVector<SDValue, 4> Halves;
4122 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4123 if (TLI.isTypeLegal(VT: HalfType) &&
4124 TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
4125 HiLoVT: HalfType, DAG,
4126 Kind: TargetLowering::MulExpansionKind::Always)) {
4127 for (unsigned i = 0; i < 2; ++i) {
4128 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
4129 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
4130 SDValue Shift =
4131 DAG.getShiftAmountConstant(Val: HalfType.getScalarSizeInBits(), VT, DL: dl);
4132 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4133 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4134 }
4135 break;
4136 }
4137
4138 SDValue Lo, Hi;
4139 TLI.forceExpandWideMUL(DAG, dl, Signed: IsSigned, LHS, RHS, Lo, Hi);
4140 Results.push_back(Elt: Lo);
4141 Results.push_back(Elt: Hi);
4142 break;
4143 }
4144 case ISD::MUL: {
4145 EVT VT = Node->getValueType(ResNo: 0);
4146 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4147 // See if multiply or divide can be lowered using two-result operations.
4148 // We just need the low half of the multiply; try both the signed
4149 // and unsigned forms. If the target supports both SMUL_LOHI and
4150 // UMUL_LOHI, form a preference by checking which forms of plain
4151 // MULH it supports.
4152 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
4153 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
4154 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
4155 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
4156 unsigned OpToUse = 0;
4157 if (HasSMUL_LOHI && !HasMULHS) {
4158 OpToUse = ISD::SMUL_LOHI;
4159 } else if (HasUMUL_LOHI && !HasMULHU) {
4160 OpToUse = ISD::UMUL_LOHI;
4161 } else if (HasSMUL_LOHI) {
4162 OpToUse = ISD::SMUL_LOHI;
4163 } else if (HasUMUL_LOHI) {
4164 OpToUse = ISD::UMUL_LOHI;
4165 }
4166 if (OpToUse) {
4167 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4168 N2: Node->getOperand(Num: 1)));
4169 break;
4170 }
4171
4172 SDValue Lo, Hi;
4173 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4174 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
4175 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
4176 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
4177 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
4178 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
4179 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4180 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
4181 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
4182 SDValue Shift =
4183 DAG.getShiftAmountConstant(Val: HalfType.getSizeInBits(), VT, DL: dl);
4184 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4185 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4186 }
4187 break;
4188 }
4189 case ISD::FSHL:
4190 case ISD::FSHR:
4191 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
4192 Results.push_back(Elt: Expanded);
4193 break;
4194 case ISD::ROTL:
4195 case ISD::ROTR:
4196 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
4197 Results.push_back(Elt: Expanded);
4198 break;
4199 case ISD::CLMUL:
4200 case ISD::CLMULR:
4201 case ISD::CLMULH:
4202 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG))
4203 Results.push_back(Elt: Expanded);
4204 break;
4205 case ISD::PEXT:
4206 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
4207 break;
4208 case ISD::PDEP:
4209 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
4210 break;
4211 case ISD::SADDSAT:
4212 case ISD::UADDSAT:
4213 case ISD::SSUBSAT:
4214 case ISD::USUBSAT:
4215 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
4216 break;
4217 case ISD::SCMP:
4218 case ISD::UCMP:
4219 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
4220 break;
4221 case ISD::SSHLSAT:
4222 case ISD::USHLSAT:
4223 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
4224 break;
4225 case ISD::SMULFIX:
4226 case ISD::SMULFIXSAT:
4227 case ISD::UMULFIX:
4228 case ISD::UMULFIXSAT:
4229 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
4230 break;
4231 case ISD::SDIVFIX:
4232 case ISD::SDIVFIXSAT:
4233 case ISD::UDIVFIX:
4234 case ISD::UDIVFIXSAT:
4235 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
4236 LHS: Node->getOperand(Num: 0),
4237 RHS: Node->getOperand(Num: 1),
4238 Scale: Node->getConstantOperandVal(Num: 2),
4239 DAG)) {
4240 Results.push_back(Elt: V);
4241 break;
4242 }
4243 // FIXME: We might want to retry here with a wider type if we fail, if that
4244 // type is legal.
4245 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4246 // <= 128 (which is the case for all of the default Embedded-C types),
4247 // we will only get here with types and scales that we could always expand
4248 // if we were allowed to generate libcalls to division functions of illegal
4249 // type. But we cannot do that.
4250 llvm_unreachable("Cannot expand DIVFIX!");
4251 case ISD::UADDO_CARRY:
4252 case ISD::USUBO_CARRY: {
4253 SDValue LHS = Node->getOperand(Num: 0);
4254 SDValue RHS = Node->getOperand(Num: 1);
4255 SDValue Carry = Node->getOperand(Num: 2);
4256
4257 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4258
4259 // Initial add of the 2 operands.
4260 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4261 EVT VT = LHS.getValueType();
4262 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4263
4264 // Initial check for overflow.
4265 EVT CarryType = Node->getValueType(ResNo: 1);
4266 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4267 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4268 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4269
4270 // Add of the sum and the carry.
4271 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4272 SDValue CarryExt =
4273 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4274 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4275
4276 // Second check for overflow. If we are adding, we can only overflow if the
4277 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4278 // If we are subtracting, we can only overflow if the initial sum is 0 and
4279 // the carry is set, resulting in a new sum of all 1s.
4280 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4281 SDValue Overflow2 =
4282 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4283 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4284 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4285 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4286
4287 SDValue ResultCarry =
4288 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4289
4290 Results.push_back(Elt: Sum2);
4291 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4292 break;
4293 }
4294 case ISD::SADDO:
4295 case ISD::SSUBO: {
4296 SDValue Result, Overflow;
4297 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4298 Results.push_back(Elt: Result);
4299 Results.push_back(Elt: Overflow);
4300 break;
4301 }
4302 case ISD::UADDO:
4303 case ISD::USUBO: {
4304 SDValue Result, Overflow;
4305 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4306 Results.push_back(Elt: Result);
4307 Results.push_back(Elt: Overflow);
4308 break;
4309 }
4310 case ISD::UMULO:
4311 case ISD::SMULO: {
4312 SDValue Result, Overflow;
4313 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4314 Results.push_back(Elt: Result);
4315 Results.push_back(Elt: Overflow);
4316 }
4317 break;
4318 }
4319 case ISD::BUILD_PAIR: {
4320 EVT PairTy = Node->getValueType(ResNo: 0);
4321 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4322 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4323 Tmp2 = DAG.getNode(
4324 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4325 N2: DAG.getShiftAmountConstant(Val: PairTy.getSizeInBits() / 2, VT: PairTy, DL: dl));
4326 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4327 break;
4328 }
4329 case ISD::SELECT:
4330 Tmp1 = Node->getOperand(Num: 0);
4331 Tmp2 = Node->getOperand(Num: 1);
4332 Tmp3 = Node->getOperand(Num: 2);
4333 if (Tmp1.getOpcode() == ISD::SETCC) {
4334 Tmp1 = DAG.getSelectCC(
4335 DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), True: Tmp2, False: Tmp3,
4336 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get(), Flags: Node->getFlags());
4337 } else {
4338 Tmp1 =
4339 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4340 True: Tmp2, False: Tmp3, Cond: ISD::SETNE, Flags: Node->getFlags());
4341 }
4342 Results.push_back(Elt: Tmp1);
4343 break;
4344 case ISD::BR_JT: {
4345 SDValue Chain = Node->getOperand(Num: 0);
4346 SDValue Table = Node->getOperand(Num: 1);
4347 SDValue Index = Node->getOperand(Num: 2);
4348 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4349
4350 const DataLayout &TD = DAG.getDataLayout();
4351 EVT PTy = TLI.getPointerTy(DL: TD);
4352
4353 unsigned EntrySize =
4354 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4355
4356 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4357 // This transformation needs to be done here since otherwise the MIPS
4358 // backend will end up emitting a three instruction multiply sequence
4359 // instead of a single shift and MSP430 will call a runtime function.
4360 if (llvm::isPowerOf2_32(Value: EntrySize))
4361 Index = DAG.getNode(
4362 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4363 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4364 else
4365 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4366 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4367 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4368
4369 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4370 SDValue LD = DAG.getExtLoad(
4371 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4372 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4373 Addr = LD;
4374 if (TLI.isJumpTableRelative()) {
4375 // For PIC, the sequence is:
4376 // BRIND(RelocBase + load(Jumptable + index))
4377 // RelocBase can be JumpTable, GOT or some sort of global base.
4378 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4379 Offset: Addr, DL: dl);
4380 }
4381
4382 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4383 Results.push_back(Elt: Tmp1);
4384 break;
4385 }
4386 case ISD::BRCOND:
4387 // Expand brcond's setcc into its constituent parts and create a BR_CC
4388 // Node.
4389 Tmp1 = Node->getOperand(Num: 0);
4390 Tmp2 = Node->getOperand(Num: 1);
4391 if (Tmp2.getOpcode() == ISD::SETCC &&
4392 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4393 VT: Tmp2.getOperand(i: 0).getValueType())) {
4394 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4395 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4396 N5: Node->getOperand(Num: 2));
4397 } else {
4398 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4399 if (Tmp2.isUndef() ||
4400 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4401 Tmp3 = Tmp2;
4402 else
4403 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4404 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4405 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4406 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4407 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4408 N5: Node->getOperand(Num: 2));
4409 }
4410 Results.push_back(Elt: Tmp1);
4411 break;
4412 case ISD::SETCC:
4413 case ISD::STRICT_FSETCC:
4414 case ISD::STRICT_FSETCCS: {
4415 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4416 Node->getOpcode() == ISD::STRICT_FSETCCS;
4417 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4418 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4419 unsigned Offset = IsStrict ? 1 : 0;
4420 Tmp1 = Node->getOperand(Num: 0 + Offset);
4421 Tmp2 = Node->getOperand(Num: 1 + Offset);
4422 Tmp3 = Node->getOperand(Num: 2 + Offset);
4423 bool Legalized =
4424 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3,
4425 NeedInvert, dl, Chain, IsSignaling);
4426
4427 if (Legalized) {
4428 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4429 // condition code, create a new SETCC node.
4430 if (Tmp3.getNode()) {
4431 if (IsStrict) {
4432 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4433 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4434 Chain = Tmp1.getValue(R: 1);
4435 } else {
4436 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4437 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4438 }
4439 }
4440
4441 // If we expanded the SETCC by inverting the condition code, then wrap
4442 // the existing SETCC in a NOT to restore the intended condition.
4443 if (NeedInvert) {
4444 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4445 }
4446
4447 Results.push_back(Elt: Tmp1);
4448 if (IsStrict)
4449 Results.push_back(Elt: Chain);
4450
4451 break;
4452 }
4453
4454 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4455 // understand if this code is useful for strict nodes.
4456 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4457
4458 // Otherwise, SETCC for the given comparison type must be completely
4459 // illegal; expand it into a SELECT_CC.
4460 EVT VT = Node->getValueType(ResNo: 0);
4461 EVT Tmp1VT = Tmp1.getValueType();
4462 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4463 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4464 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3,
4465 Flags: Node->getFlags());
4466 Results.push_back(Elt: Tmp1);
4467 break;
4468 }
4469 case ISD::SELECT_CC: {
4470 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4471 Tmp1 = Node->getOperand(Num: 0); // LHS
4472 Tmp2 = Node->getOperand(Num: 1); // RHS
4473 Tmp3 = Node->getOperand(Num: 2); // True
4474 Tmp4 = Node->getOperand(Num: 3); // False
4475 EVT VT = Node->getValueType(ResNo: 0);
4476 SDValue Chain;
4477 SDValue CC = Node->getOperand(Num: 4);
4478 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4479
4480 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4481 // If the condition code is legal, then we need to expand this
4482 // node using SETCC and SELECT.
4483 EVT CmpVT = Tmp1.getValueType();
4484 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4485 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4486 "expanded.");
4487 EVT CCVT = getSetCCResultType(VT: CmpVT);
4488 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4489 Results.push_back(
4490 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4491 break;
4492 }
4493
4494 // SELECT_CC is legal, so the condition code must not be.
4495 bool Legalized = false;
4496 // Try to legalize by inverting the condition. This is for targets that
4497 // might support an ordered version of a condition, but not the unordered
4498 // version (or vice versa).
4499 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4500 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4501 // Use the new condition code and swap true and false
4502 Legalized = true;
4503 Tmp1 =
4504 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC, Flags: Node->getFlags());
4505 } else {
4506 // If The inverse is not legal, then try to swap the arguments using
4507 // the inverse condition code.
4508 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4509 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4510 // The swapped inverse condition is legal, so swap true and false,
4511 // lhs and rhs.
4512 Legalized = true;
4513 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC,
4514 Flags: Node->getFlags());
4515 }
4516 }
4517
4518 if (!Legalized) {
4519 Legalized = TLI.LegalizeSetCCCondCode(
4520 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4521 NeedInvert, dl, Chain);
4522
4523 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4524
4525 // If we expanded the SETCC by inverting the condition code, then swap
4526 // the True/False operands to match.
4527 if (NeedInvert)
4528 std::swap(a&: Tmp3, b&: Tmp4);
4529
4530 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4531 // condition code, create a new SELECT_CC node.
4532 if (CC.getNode()) {
4533 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4534 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4535 } else {
4536 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4537 CC = DAG.getCondCode(Cond: ISD::SETNE);
4538 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4539 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4540 }
4541 }
4542 Results.push_back(Elt: Tmp1);
4543 break;
4544 }
4545 case ISD::BR_CC: {
4546 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4547 SDValue Chain;
4548 Tmp1 = Node->getOperand(Num: 0); // Chain
4549 Tmp2 = Node->getOperand(Num: 2); // LHS
4550 Tmp3 = Node->getOperand(Num: 3); // RHS
4551 Tmp4 = Node->getOperand(Num: 1); // CC
4552
4553 bool Legalized =
4554 TLI.LegalizeSetCCCondCode(DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()),
4555 LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4, NeedInvert, dl, Chain);
4556 (void)Legalized;
4557 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4558
4559 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4560 // node.
4561 if (Tmp4.getNode()) {
4562 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4563
4564 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4565 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4566 } else {
4567 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4568 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4569 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4570 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4571 }
4572 Results.push_back(Elt: Tmp1);
4573 break;
4574 }
4575 case ISD::BUILD_VECTOR:
4576 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4577 break;
4578 case ISD::SPLAT_VECTOR:
4579 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4580 break;
4581 case ISD::SRA:
4582 case ISD::SRL:
4583 case ISD::SHL: {
4584 // Scalarize vector SRA/SRL/SHL.
4585 EVT VT = Node->getValueType(ResNo: 0);
4586 assert(VT.isVector() && "Unable to legalize non-vector shift");
4587 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4588 unsigned NumElem = VT.getVectorNumElements();
4589
4590 SmallVector<SDValue, 8> Scalars;
4591 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4592 SDValue Ex =
4593 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4594 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4595 SDValue Sh =
4596 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4597 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4598 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4599 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4600 }
4601
4602 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4603 Results.push_back(Elt: Result);
4604 break;
4605 }
4606 case ISD::VECREDUCE_FADD:
4607 case ISD::VECREDUCE_FMUL:
4608 case ISD::VECREDUCE_ADD:
4609 case ISD::VECREDUCE_MUL:
4610 case ISD::VECREDUCE_AND:
4611 case ISD::VECREDUCE_OR:
4612 case ISD::VECREDUCE_XOR:
4613 case ISD::VECREDUCE_SMAX:
4614 case ISD::VECREDUCE_SMIN:
4615 case ISD::VECREDUCE_UMAX:
4616 case ISD::VECREDUCE_UMIN:
4617 case ISD::VECREDUCE_FMAX:
4618 case ISD::VECREDUCE_FMIN:
4619 case ISD::VECREDUCE_FMAXIMUM:
4620 case ISD::VECREDUCE_FMINIMUM:
4621 case ISD::VECREDUCE_FMAXIMUMNUM:
4622 case ISD::VECREDUCE_FMINIMUMNUM:
4623 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4624 break;
4625 case ISD::VP_CTTZ_ELTS:
4626 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4627 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4628 break;
4629 case ISD::CLEAR_CACHE:
4630 // The default expansion of llvm.clear_cache is simply a no-op for those
4631 // targets where it is not needed.
4632 Results.push_back(Elt: Node->getOperand(Num: 0));
4633 break;
4634 case ISD::LRINT:
4635 case ISD::LLRINT: {
4636 SDValue Arg = Node->getOperand(Num: 0);
4637 EVT ArgVT = Arg.getValueType();
4638 EVT ResVT = Node->getValueType(ResNo: 0);
4639 SDLoc DL(Node);
4640 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL, VT: ArgVT, Operand: Arg);
4641 SDValue ConvertNode = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: ResVT, Operand: RoundNode);
4642 // Non-deterministic results are equivalent to freeze poison.
4643 Results.push_back(Elt: DAG.getFreeze(V: ConvertNode));
4644 break;
4645 }
4646 case ISD::ADDRSPACECAST:
4647 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4648 break;
4649 case ISD::GLOBAL_OFFSET_TABLE:
4650 case ISD::GlobalAddress:
4651 case ISD::GlobalTLSAddress:
4652 case ISD::ExternalSymbol:
4653 case ISD::ConstantPool:
4654 case ISD::JumpTable:
4655 case ISD::INTRINSIC_W_CHAIN:
4656 case ISD::INTRINSIC_WO_CHAIN:
4657 case ISD::INTRINSIC_VOID:
4658 // FIXME: Custom lowering for these operations shouldn't return null!
4659 // Return true so that we don't call ConvertNodeToLibcall which also won't
4660 // do anything.
4661 return true;
4662 }
4663
4664 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4665 // FIXME: We were asked to expand a strict floating-point operation,
4666 // but there is currently no expansion implemented that would preserve
4667 // the "strict" properties. For now, we just fall back to the non-strict
4668 // version if that is legal on the target. The actual mutation of the
4669 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4670 switch (Node->getOpcode()) {
4671 default:
4672 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4673 VT: Node->getValueType(ResNo: 0))
4674 == TargetLowering::Legal)
4675 return true;
4676 break;
4677 case ISD::STRICT_FSUB: {
4678 if (TLI.getStrictFPOperationAction(
4679 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4680 return true;
4681 if (TLI.getStrictFPOperationAction(
4682 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4683 break;
4684
4685 EVT VT = Node->getValueType(ResNo: 0);
4686 const SDNodeFlags Flags = Node->getFlags();
4687 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4688 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4689 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4690 Flags);
4691
4692 Results.push_back(Elt: Fadd);
4693 Results.push_back(Elt: Fadd.getValue(R: 1));
4694 break;
4695 }
4696 case ISD::STRICT_SINT_TO_FP:
4697 case ISD::STRICT_UINT_TO_FP:
4698 case ISD::STRICT_LRINT:
4699 case ISD::STRICT_LLRINT:
4700 case ISD::STRICT_LROUND:
4701 case ISD::STRICT_LLROUND:
4702 // These are registered by the operand type instead of the value
4703 // type. Reflect that here.
4704 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4705 VT: Node->getOperand(Num: 1).getValueType())
4706 == TargetLowering::Legal)
4707 return true;
4708 break;
4709 }
4710 }
4711
4712 // Replace the original node with the legalized result.
4713 if (Results.empty()) {
4714 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4715 return false;
4716 }
4717
4718 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4719 ReplaceNode(Old: Node, New: Results.data());
4720 return true;
4721}
4722
4723/// Return if we can use the FAST_* variant of a math libcall for the node.
4724/// FIXME: This is just guessing, we probably should have unique specific sets
4725/// flags required per libcall.
4726static bool canUseFastMathLibcall(const SDNode *Node) {
4727 // FIXME: Probably should define fast to respect nan/inf and only be
4728 // approximate functions.
4729
4730 SDNodeFlags Flags = Node->getFlags();
4731 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4732 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4733}
4734
4735void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4736 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4737 SmallVector<SDValue, 8> Results;
4738 SDLoc dl(Node);
4739 TargetLowering::MakeLibCallOptions CallOptions;
4740 CallOptions.IsPostTypeLegalization = true;
4741 // FIXME: Check flags on the node to see if we can use a finite call.
4742 unsigned Opc = Node->getOpcode();
4743 switch (Opc) {
4744 case ISD::ATOMIC_FENCE: {
4745 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4746 // FIXME: handle "fence singlethread" more efficiently.
4747 TargetLowering::ArgListTy Args;
4748
4749 TargetLowering::CallLoweringInfo CLI(DAG);
4750 CLI.setDebugLoc(dl)
4751 .setChain(Node->getOperand(Num: 0))
4752 .setLibCallee(
4753 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4754 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4755 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4756 ArgsList: std::move(Args));
4757
4758 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4759
4760 Results.push_back(Elt: CallResult.second);
4761 break;
4762 }
4763 // By default, atomic intrinsics are marked Legal and lowered. Targets
4764 // which don't support them directly, however, may want libcalls, in which
4765 // case they mark them Expand, and we get here.
4766 case ISD::ATOMIC_SWAP:
4767 case ISD::ATOMIC_LOAD_ADD:
4768 case ISD::ATOMIC_LOAD_SUB:
4769 case ISD::ATOMIC_LOAD_AND:
4770 case ISD::ATOMIC_LOAD_CLR:
4771 case ISD::ATOMIC_LOAD_OR:
4772 case ISD::ATOMIC_LOAD_XOR:
4773 case ISD::ATOMIC_LOAD_NAND:
4774 case ISD::ATOMIC_LOAD_MIN:
4775 case ISD::ATOMIC_LOAD_MAX:
4776 case ISD::ATOMIC_LOAD_UMIN:
4777 case ISD::ATOMIC_LOAD_UMAX:
4778 case ISD::ATOMIC_CMP_SWAP: {
4779 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4780 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4781 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4782 EVT RetVT = Node->getValueType(ResNo: 0);
4783 SmallVector<SDValue, 4> Ops;
4784 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported) {
4785 // If outline atomic available, prepare its arguments and expand.
4786 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4787 Ops.push_back(Elt: Node->getOperand(Num: 1));
4788
4789 } else {
4790 LC = RTLIB::getSYNC(Opc, VT);
4791 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4792 "Unexpected atomic op or value type!");
4793 // Arguments for expansion to sync libcall
4794 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4795 }
4796 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4797 Ops, CallOptions,
4798 dl: SDLoc(Node),
4799 Chain: Node->getOperand(Num: 0));
4800 Results.push_back(Elt: Tmp.first);
4801 Results.push_back(Elt: Tmp.second);
4802 break;
4803 }
4804 case ISD::TRAP: {
4805 // If this operation is not supported, lower it to 'abort()' call
4806 TargetLowering::ArgListTy Args;
4807 TargetLowering::CallLoweringInfo CLI(DAG);
4808 CLI.setDebugLoc(dl)
4809 .setChain(Node->getOperand(Num: 0))
4810 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4811 Target: DAG.getExternalSymbol(
4812 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4813 ArgsList: std::move(Args));
4814 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4815
4816 Results.push_back(Elt: CallResult.second);
4817 break;
4818 }
4819 case ISD::CLEAR_CACHE: {
4820 SDValue InputChain = Node->getOperand(Num: 0);
4821 SDValue StartVal = Node->getOperand(Num: 1);
4822 SDValue EndVal = Node->getOperand(Num: 2);
4823 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4824 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
4825 dl: SDLoc(Node), Chain: InputChain);
4826 Results.push_back(Elt: Tmp.second);
4827 break;
4828 }
4829 case ISD::FMINNUM:
4830 case ISD::STRICT_FMINNUM:
4831 ExpandFPLibCall(Node, LC: RTLIB::getFMIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4832 break;
4833 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4834 // libcall legalization for these nodes, but there is no default expasion for
4835 // these nodes either (see PR63267 for example).
4836 case ISD::FMAXNUM:
4837 case ISD::STRICT_FMAXNUM:
4838 ExpandFPLibCall(Node, LC: RTLIB::getFMAX(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4839 break;
4840 case ISD::FMINIMUMNUM:
4841 ExpandFPLibCall(Node, LC: RTLIB::getFMINIMUM_NUM(VT: Node->getSimpleValueType(ResNo: 0)),
4842 Results);
4843 break;
4844 case ISD::FMAXIMUMNUM:
4845 ExpandFPLibCall(Node, LC: RTLIB::getFMAXIMUM_NUM(VT: Node->getSimpleValueType(ResNo: 0)),
4846 Results);
4847 break;
4848 case ISD::FSQRT:
4849 case ISD::STRICT_FSQRT: {
4850 // FIXME: Probably should define fast to respect nan/inf and only be
4851 // approximate functions.
4852 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
4853 Call_F32: {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
4854 Call_F64: {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
4855 Call_F80: {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
4856 Call_F128: {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
4857 Call_PPCF128: {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
4858 Results);
4859 break;
4860 }
4861 case ISD::FCBRT:
4862 ExpandFPLibCall(Node, LC: RTLIB::getCBRT(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4863 break;
4864 case ISD::FSIN:
4865 case ISD::STRICT_FSIN:
4866 ExpandFPLibCall(Node, LC: RTLIB::getSIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4867 break;
4868 case ISD::FCOS:
4869 case ISD::STRICT_FCOS:
4870 ExpandFPLibCall(Node, LC: RTLIB::getCOS(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4871 break;
4872 case ISD::FTAN:
4873 case ISD::STRICT_FTAN:
4874 ExpandFPLibCall(Node, LC: RTLIB::getTAN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4875 break;
4876 case ISD::FASIN:
4877 case ISD::STRICT_FASIN:
4878 ExpandFPLibCall(Node, LC: RTLIB::getASIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4879 break;
4880 case ISD::FACOS:
4881 case ISD::STRICT_FACOS:
4882 ExpandFPLibCall(Node, LC: RTLIB::getACOS(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4883 break;
4884 case ISD::FATAN:
4885 case ISD::STRICT_FATAN:
4886 ExpandFPLibCall(Node, LC: RTLIB::getATAN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4887 break;
4888 case ISD::FATAN2:
4889 case ISD::STRICT_FATAN2:
4890 ExpandFPLibCall(Node, LC: RTLIB::getATAN2(VT: Node->getSimpleValueType(ResNo: 0)),
4891 Results);
4892 break;
4893 case ISD::FSINH:
4894 case ISD::STRICT_FSINH:
4895 ExpandFPLibCall(Node, LC: RTLIB::getSINH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4896 break;
4897 case ISD::FCOSH:
4898 case ISD::STRICT_FCOSH:
4899 ExpandFPLibCall(Node, LC: RTLIB::getCOSH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4900 break;
4901 case ISD::FTANH:
4902 case ISD::STRICT_FTANH:
4903 ExpandFPLibCall(Node, LC: RTLIB::getTANH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4904 break;
4905 case ISD::FSINCOS:
4906 case ISD::FSINCOSPI: {
4907 EVT VT = Node->getValueType(ResNo: 0);
4908
4909 if (Node->getOpcode() == ISD::FSINCOS) {
4910 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(VT);
4911 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
4912 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
4913 Results.push_back(Elt: Expanded);
4914 Results.push_back(Elt: Expanded.getValue(R: 1));
4915 break;
4916 }
4917 }
4918 }
4919
4920 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4921 ? RTLIB::getSINCOS(VT)
4922 : RTLIB::getSINCOSPI(VT);
4923 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);
4924 if (!Expanded) {
4925 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
4926 Node->getOperationName(G: &DAG));
4927 SDValue Poison = DAG.getPOISON(VT);
4928 Results.push_back(Elt: Poison);
4929 Results.push_back(Elt: Poison);
4930 }
4931
4932 break;
4933 }
4934 case ISD::FLOG:
4935 case ISD::STRICT_FLOG:
4936 ExpandFPLibCall(Node, LC: RTLIB::getLOG(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4937 break;
4938 case ISD::FLOG2:
4939 case ISD::STRICT_FLOG2:
4940 ExpandFPLibCall(Node, LC: RTLIB::getLOG2(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4941 break;
4942 case ISD::FLOG10:
4943 case ISD::STRICT_FLOG10:
4944 ExpandFPLibCall(Node, LC: RTLIB::getLOG10(VT: Node->getSimpleValueType(ResNo: 0)),
4945 Results);
4946 break;
4947 case ISD::FEXP:
4948 case ISD::STRICT_FEXP:
4949 ExpandFPLibCall(Node, LC: RTLIB::getEXP(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4950 break;
4951 case ISD::FEXP2:
4952 case ISD::STRICT_FEXP2:
4953 ExpandFPLibCall(Node, LC: RTLIB::getEXP2(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4954 break;
4955 case ISD::FEXP10:
4956 ExpandFPLibCall(Node, LC: RTLIB::getEXP10(VT: Node->getSimpleValueType(ResNo: 0)),
4957 Results);
4958 break;
4959 case ISD::FTRUNC:
4960 case ISD::STRICT_FTRUNC:
4961 ExpandFPLibCall(Node, LC: RTLIB::getTRUNC(VT: Node->getSimpleValueType(ResNo: 0)),
4962 Results);
4963 break;
4964 case ISD::FFLOOR:
4965 case ISD::STRICT_FFLOOR:
4966 ExpandFPLibCall(Node, LC: RTLIB::getFLOOR(VT: Node->getSimpleValueType(ResNo: 0)),
4967 Results);
4968 break;
4969 case ISD::FCEIL:
4970 case ISD::STRICT_FCEIL:
4971 ExpandFPLibCall(Node, LC: RTLIB::getCEIL(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4972 break;
4973 case ISD::FRINT:
4974 case ISD::STRICT_FRINT:
4975 ExpandFPLibCall(Node, LC: RTLIB::getRINT(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4976 break;
4977 case ISD::FNEARBYINT:
4978 case ISD::STRICT_FNEARBYINT:
4979 ExpandFPLibCall(Node, LC: RTLIB::getNEARBYINT(VT: Node->getSimpleValueType(ResNo: 0)),
4980 Results);
4981 break;
4982 case ISD::FROUND:
4983 case ISD::STRICT_FROUND:
4984 ExpandFPLibCall(Node, LC: RTLIB::getROUND(VT: Node->getSimpleValueType(ResNo: 0)),
4985 Results);
4986 break;
4987 case ISD::FROUNDEVEN:
4988 case ISD::STRICT_FROUNDEVEN:
4989 ExpandFPLibCall(Node, LC: RTLIB::getROUNDEVEN(VT: Node->getSimpleValueType(ResNo: 0)),
4990 Results);
4991 break;
4992 case ISD::FLDEXP:
4993 case ISD::STRICT_FLDEXP:
4994 ExpandFPLibCall(Node, LC: RTLIB::getLDEXP(VT: Node->getSimpleValueType(ResNo: 0)),
4995 Results);
4996 break;
4997 case ISD::FMODF:
4998 case ISD::FFREXP: {
4999 EVT VT = Node->getValueType(ResNo: 0);
5000 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
5001 : RTLIB::getFREXP(VT);
5002 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
5003 /*CallRetResNo=*/0);
5004 if (!Expanded) {
5005 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
5006 Node->getOperationName(G: &DAG));
5007 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
5008 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: I)));
5009 }
5010 break;
5011 }
5012 case ISD::FPOWI:
5013 case ISD::STRICT_FPOWI: {
5014 RTLIB::Libcall LC = RTLIB::getPOWI(VT: Node->getSimpleValueType(ResNo: 0));
5015 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
5016 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
5017 // Some targets don't have a powi libcall; use pow instead.
5018 if (Node->isStrictFPOpcode()) {
5019 SDValue Exponent =
5020 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
5021 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5022 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5023 SDValue FPOW =
5024 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
5025 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5026 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
5027 Results.push_back(Elt: FPOW);
5028 Results.push_back(Elt: FPOW.getValue(R: 1));
5029 } else {
5030 SDValue Exponent =
5031 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
5032 Operand: Node->getOperand(Num: 1));
5033 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
5034 VT: Node->getValueType(ResNo: 0),
5035 N1: Node->getOperand(Num: 0), N2: Exponent));
5036 }
5037 break;
5038 }
5039 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
5040 bool ExponentHasSizeOfInt =
5041 DAG.getLibInfo().getIntSize() ==
5042 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
5043 if (!ExponentHasSizeOfInt) {
5044 // If the exponent does not match with sizeof(int) a libcall to
5045 // RTLIB::POWI would use the wrong type for the argument.
5046 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
5047 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
5048 break;
5049 }
5050 ExpandFPLibCall(Node, LC, Results);
5051 break;
5052 }
5053 case ISD::FPOW:
5054 case ISD::STRICT_FPOW:
5055 ExpandFPLibCall(Node, LC: RTLIB::getPOW(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5056 break;
5057 case ISD::LROUND:
5058 case ISD::STRICT_LROUND:
5059 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
5060 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
5061 Call_F128: RTLIB::LROUND_F128,
5062 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
5063 break;
5064 case ISD::LLROUND:
5065 case ISD::STRICT_LLROUND:
5066 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
5067 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
5068 Call_F128: RTLIB::LLROUND_F128,
5069 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
5070 break;
5071 case ISD::LRINT:
5072 case ISD::STRICT_LRINT:
5073 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
5074 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
5075 Call_F128: RTLIB::LRINT_F128,
5076 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
5077 break;
5078 case ISD::LLRINT:
5079 case ISD::STRICT_LLRINT:
5080 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
5081 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
5082 Call_F128: RTLIB::LLRINT_F128,
5083 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
5084 break;
5085 case ISD::FDIV:
5086 case ISD::STRICT_FDIV: {
5087 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5088 Call_F32: {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5089 Call_F64: {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5090 Call_F80: {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5091 Call_F128: {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5092 Call_PPCF128: {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5093 break;
5094 }
5095 case ISD::FREM:
5096 case ISD::STRICT_FREM:
5097 ExpandFPLibCall(Node, LC: RTLIB::getREM(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5098 break;
5099 case ISD::FMA:
5100 case ISD::STRICT_FMA:
5101 ExpandFPLibCall(Node, LC: RTLIB::getFMA(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5102 break;
5103 case ISD::FADD:
5104 case ISD::STRICT_FADD: {
5105 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5106 Call_F32: {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5107 Call_F64: {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5108 Call_F80: {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5109 Call_F128: {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5110 Call_PPCF128: {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5111 break;
5112 }
5113 case ISD::FMUL:
5114 case ISD::STRICT_FMUL: {
5115 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5116 Call_F32: {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5117 Call_F64: {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5118 Call_F80: {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5119 Call_F128: {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5120 Call_PPCF128: {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5121 break;
5122 }
5123 case ISD::FP16_TO_FP:
5124 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5125 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
5126 }
5127 break;
5128 case ISD::STRICT_BF16_TO_FP:
5129 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5130 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5131 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
5132 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5133 Results.push_back(Elt: Tmp.first);
5134 Results.push_back(Elt: Tmp.second);
5135 }
5136 break;
5137 case ISD::STRICT_FP16_TO_FP: {
5138 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5139 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5140 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
5141 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5142 Results.push_back(Elt: Tmp.first);
5143 Results.push_back(Elt: Tmp.second);
5144 }
5145 break;
5146 }
5147 case ISD::FP_TO_FP16: {
5148 RTLIB::Libcall LC =
5149 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
5150 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5151 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5152 break;
5153 }
5154 case ISD::FP_TO_BF16: {
5155 RTLIB::Libcall LC =
5156 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
5157 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5158 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5159 break;
5160 }
5161 case ISD::STRICT_SINT_TO_FP:
5162 case ISD::STRICT_UINT_TO_FP:
5163 case ISD::SINT_TO_FP:
5164 case ISD::UINT_TO_FP: {
5165 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
5166 bool IsStrict = Node->isStrictFPOpcode();
5167 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
5168 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
5169 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
5170 EVT RVT = Node->getValueType(ResNo: 0);
5171 EVT NVT = EVT();
5172 SDLoc dl(Node);
5173
5174 // Even if the input is legal, no libcall may exactly match, eg. we don't
5175 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
5176 // eg: i13 -> fp. Then, look for an appropriate libcall.
5177 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5178 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5179 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5180 ++t) {
5181 NVT = (MVT::SimpleValueType)t;
5182 // The source needs to big enough to hold the operand.
5183 if (NVT.bitsGE(VT: SVT))
5184 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
5185 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
5186 }
5187 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5188
5189 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5190 // Sign/zero extend the argument if the libcall takes a larger type.
5191 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
5192 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
5193 CallOptions.setIsSigned(Signed);
5194 std::pair<SDValue, SDValue> Tmp =
5195 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
5196 Results.push_back(Elt: Tmp.first);
5197 if (IsStrict)
5198 Results.push_back(Elt: Tmp.second);
5199 break;
5200 }
5201 case ISD::FP_TO_SINT:
5202 case ISD::FP_TO_UINT:
5203 case ISD::STRICT_FP_TO_SINT:
5204 case ISD::STRICT_FP_TO_UINT: {
5205 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
5206 bool IsStrict = Node->isStrictFPOpcode();
5207 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
5208 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
5209
5210 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5211 EVT SVT = Op.getValueType();
5212 EVT RVT = Node->getValueType(ResNo: 0);
5213 EVT NVT = EVT();
5214 SDLoc dl(Node);
5215
5216 // Even if the result is legal, no libcall may exactly match, eg. we don't
5217 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
5218 // eg: fp -> i32. Then, look for an appropriate libcall.
5219 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5220 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5221 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5222 ++IntVT) {
5223 NVT = (MVT::SimpleValueType)IntVT;
5224 // The type needs to big enough to hold the result.
5225 if (NVT.bitsGE(VT: RVT))
5226 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
5227 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
5228 }
5229 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5230
5231 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5232 std::pair<SDValue, SDValue> Tmp =
5233 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
5234
5235 // Truncate the result if the libcall returns a larger type.
5236 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
5237 if (IsStrict)
5238 Results.push_back(Elt: Tmp.second);
5239 break;
5240 }
5241
5242 case ISD::FP_ROUND:
5243 case ISD::STRICT_FP_ROUND: {
5244 // X = FP_ROUND(Y, TRUNC)
5245 // TRUNC is a flag, which is always an integer that is zero or one.
5246 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5247 // is known to not change the value of Y.
5248 // We can only expand it into libcall if the TRUNC is 0.
5249 bool IsStrict = Node->isStrictFPOpcode();
5250 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5251 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5252 EVT VT = Node->getValueType(ResNo: 0);
5253 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5254 "Unable to expand as libcall if it is not normal rounding");
5255
5256 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5257 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5258
5259 std::pair<SDValue, SDValue> Tmp =
5260 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5261 Results.push_back(Elt: Tmp.first);
5262 if (IsStrict)
5263 Results.push_back(Elt: Tmp.second);
5264 break;
5265 }
5266 case ISD::FP_EXTEND: {
5267 Results.push_back(
5268 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5269 RetVT: Node->getValueType(ResNo: 0)),
5270 Node, isSigned: false).first);
5271 break;
5272 }
5273 case ISD::STRICT_FP_EXTEND:
5274 case ISD::STRICT_FP_TO_FP16:
5275 case ISD::STRICT_FP_TO_BF16: {
5276 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5277 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5278 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5279 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5280 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5281 else
5282 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5283 RetVT: Node->getValueType(ResNo: 0));
5284
5285 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5286
5287 std::pair<SDValue, SDValue> Tmp =
5288 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5289 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5290 Results.push_back(Elt: Tmp.first);
5291 Results.push_back(Elt: Tmp.second);
5292 break;
5293 }
5294 case ISD::FSUB:
5295 case ISD::STRICT_FSUB: {
5296 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5297 Call_F32: {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5298 Call_F64: {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5299 Call_F80: {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5300 Call_F128: {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5301 Call_PPCF128: {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5302 break;
5303 }
5304 case ISD::SREM:
5305 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5306 Call_I8: RTLIB::SREM_I8,
5307 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5308 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5309 break;
5310 case ISD::UREM:
5311 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5312 Call_I8: RTLIB::UREM_I8,
5313 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5314 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5315 break;
5316 case ISD::SDIV:
5317 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5318 Call_I8: RTLIB::SDIV_I8,
5319 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5320 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5321 break;
5322 case ISD::UDIV:
5323 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5324 Call_I8: RTLIB::UDIV_I8,
5325 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5326 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5327 break;
5328 case ISD::SDIVREM:
5329 case ISD::UDIVREM:
5330 // Expand into divrem libcall
5331 ExpandDivRemLibCall(Node, Results);
5332 break;
5333 case ISD::MUL:
5334 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5335 Call_I8: RTLIB::MUL_I8,
5336 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5337 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5338 break;
5339 case ISD::CTLZ_ZERO_POISON:
5340 Results.push_back(Elt: ExpandBitCountingLibCall(
5341 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5342 break;
5343 case ISD::CTPOP:
5344 Results.push_back(Elt: ExpandBitCountingLibCall(
5345 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5346 break;
5347 case ISD::RESET_FPENV: {
5348 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5349 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5350 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5351 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5352 SDValue Chain = Node->getOperand(Num: 0);
5353 Results.push_back(
5354 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, Node));
5355 break;
5356 }
5357 case ISD::GET_FPENV_MEM: {
5358 SDValue Chain = Node->getOperand(Num: 0);
5359 SDValue EnvPtr = Node->getOperand(Num: 1);
5360 Results.push_back(
5361 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, Node));
5362 break;
5363 }
5364 case ISD::SET_FPENV_MEM: {
5365 SDValue Chain = Node->getOperand(Num: 0);
5366 SDValue EnvPtr = Node->getOperand(Num: 1);
5367 Results.push_back(
5368 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, Node));
5369 break;
5370 }
5371 case ISD::GET_FPMODE: {
5372 // Call fegetmode, which saves control modes into a stack slot. Then load
5373 // the value to return from the stack.
5374 EVT ModeVT = Node->getValueType(ResNo: 0);
5375 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5376 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5377 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5378 InChain: Node->getOperand(Num: 0), Node);
5379 SDValue LdInst = DAG.getLoad(
5380 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5381 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5382 Results.push_back(Elt: LdInst);
5383 Results.push_back(Elt: LdInst.getValue(R: 1));
5384 break;
5385 }
5386 case ISD::SET_FPMODE: {
5387 // Move control modes to stack slot and then call fesetmode with the pointer
5388 // to the slot as argument.
5389 SDValue Mode = Node->getOperand(Num: 1);
5390 EVT ModeVT = Mode.getValueType();
5391 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5392 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5393 SDValue StInst = DAG.getStore(
5394 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5395 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5396 Results.push_back(
5397 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, Node));
5398 break;
5399 }
5400 case ISD::RESET_FPMODE: {
5401 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5402 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5403 // target must provide custom lowering.
5404 const DataLayout &DL = DAG.getDataLayout();
5405 EVT PtrTy = TLI.getPointerTy(DL);
5406 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5407 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5408 InChain: Node->getOperand(Num: 0), Node));
5409 break;
5410 }
5411 }
5412
5413 // Replace the original node with the legalized result.
5414 if (!Results.empty()) {
5415 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5416 ReplaceNode(Old: Node, New: Results.data());
5417 } else
5418 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5419}
5420
5421// Determine the vector type to use in place of an original scalar element when
5422// promoting equally sized vectors.
5423static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5424 MVT EltVT, MVT NewEltVT) {
5425 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5426 MVT MidVT = OldEltsPerNewElt == 1
5427 ? NewEltVT
5428 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5429 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5430 return MidVT;
5431}
5432
5433void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5434 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5435 SmallVector<SDValue, 8> Results;
5436 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5437 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5438 Node->getOpcode() == ISD::SINT_TO_FP || Node->getOpcode() == ISD::SETCC ||
5439 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5440 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5441 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5442 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5443 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5444 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM ||
5445 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUMNUM ||
5446 Node->getOpcode() == ISD::VECREDUCE_FMINIMUMNUM) {
5447 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5448 }
5449 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5450 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5451 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5452 Node->getOpcode() == ISD::STRICT_FSETCC ||
5453 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5454 Node->getOpcode() == ISD::STRICT_LRINT ||
5455 Node->getOpcode() == ISD::STRICT_LLRINT ||
5456 Node->getOpcode() == ISD::STRICT_LROUND ||
5457 Node->getOpcode() == ISD::STRICT_LLROUND ||
5458 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5459 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5460 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5461 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5462 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5463 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5464 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5465 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5466 if (Node->getOpcode() == ISD::BR_CC ||
5467 Node->getOpcode() == ISD::SELECT_CC)
5468 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5469 // Preserve fast math flags
5470 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5471 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5472 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5473 SDLoc dl(Node);
5474 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5475 switch (Node->getOpcode()) {
5476 case ISD::CTTZ:
5477 case ISD::CTTZ_ZERO_POISON:
5478 case ISD::CTLZ:
5479 case ISD::CTPOP: {
5480 // Zero extend the argument unless its cttz, then use any_extend.
5481 if (Node->getOpcode() == ISD::CTTZ ||
5482 Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
5483 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5484 else
5485 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5486
5487 unsigned NewOpc = Node->getOpcode();
5488 if (NewOpc == ISD::CTTZ) {
5489 // The count is the same in the promoted type except if the original
5490 // value was zero. This can be handled by setting the bit just off
5491 // the top of the original type.
5492 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5493 BitNo: OVT.getSizeInBits());
5494 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5495 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5496 NewOpc = ISD::CTTZ_ZERO_POISON;
5497 }
5498 // Perform the larger operation. For CTPOP and CTTZ_ZERO_POISON, this is
5499 // already the correct result.
5500 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5501 if (NewOpc == ISD::CTLZ) {
5502 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5503 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5504 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5505 OVT.getSizeInBits(), DL: dl, VT: NVT));
5506 }
5507 Results.push_back(
5508 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5509 break;
5510 }
5511 case ISD::CTLZ_ZERO_POISON: {
5512 // We know that the argument is unlikely to be zero, hence we can take a
5513 // different approach as compared to ISD::CTLZ
5514
5515 // Any Extend the argument
5516 auto AnyExtendedNode =
5517 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5518
5519 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5520 auto ShiftConstant = DAG.getShiftAmountConstant(
5521 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5522 auto LeftShiftResult =
5523 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5524
5525 // Perform the larger operation
5526 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5527 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5528 break;
5529 }
5530 case ISD::PEXT: {
5531 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5532 Tmp2 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5533 Tmp1 = DAG.getNode(Opcode: ISD::PEXT, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5534 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5535 break;
5536 }
5537 case ISD::PDEP: {
5538 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5539 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5540 Tmp1 = DAG.getNode(Opcode: ISD::PDEP, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5541 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5542 break;
5543 }
5544 case ISD::BITREVERSE:
5545 case ISD::BSWAP: {
5546 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5547 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5548 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5549 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5550 N2: DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl));
5551
5552 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5553 break;
5554 }
5555 case ISD::FP_TO_UINT:
5556 case ISD::STRICT_FP_TO_UINT:
5557 case ISD::FP_TO_SINT:
5558 case ISD::STRICT_FP_TO_SINT:
5559 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5560 break;
5561 case ISD::FP_TO_UINT_SAT:
5562 case ISD::FP_TO_SINT_SAT:
5563 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5564 break;
5565 case ISD::UINT_TO_FP:
5566 case ISD::STRICT_UINT_TO_FP:
5567 case ISD::SINT_TO_FP:
5568 case ISD::STRICT_SINT_TO_FP:
5569 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5570 break;
5571 case ISD::VAARG: {
5572 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5573 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5574
5575 unsigned TruncOp;
5576 if (OVT.isVector()) {
5577 TruncOp = ISD::BITCAST;
5578 } else {
5579 assert(OVT.isInteger()
5580 && "VAARG promotion is supported only for vectors or integer types");
5581 TruncOp = ISD::TRUNCATE;
5582 }
5583
5584 // Perform the larger operation, then convert back
5585 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5586 Align: Node->getConstantOperandVal(Num: 3));
5587 Chain = Tmp1.getValue(R: 1);
5588
5589 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5590
5591 // Modified the chain result - switch anything that used the old chain to
5592 // use the new one.
5593 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5594 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5595 if (UpdatedNodes) {
5596 UpdatedNodes->insert(X: Tmp2.getNode());
5597 UpdatedNodes->insert(X: Chain.getNode());
5598 }
5599 ReplacedNode(N: Node);
5600 break;
5601 }
5602 case ISD::MUL:
5603 case ISD::SDIV:
5604 case ISD::SREM:
5605 case ISD::UDIV:
5606 case ISD::UREM:
5607 case ISD::SMIN:
5608 case ISD::SMAX:
5609 case ISD::UMIN:
5610 case ISD::UMAX:
5611 case ISD::AND:
5612 case ISD::OR:
5613 case ISD::XOR: {
5614 unsigned ExtOp, TruncOp;
5615 if (OVT.isVector()) {
5616 ExtOp = ISD::BITCAST;
5617 TruncOp = ISD::BITCAST;
5618 } else {
5619 assert(OVT.isInteger() && "Cannot promote logic operation");
5620
5621 switch (Node->getOpcode()) {
5622 default:
5623 ExtOp = ISD::ANY_EXTEND;
5624 break;
5625 case ISD::SDIV:
5626 case ISD::SREM:
5627 case ISD::SMIN:
5628 case ISD::SMAX:
5629 ExtOp = ISD::SIGN_EXTEND;
5630 break;
5631 case ISD::UDIV:
5632 case ISD::UREM:
5633 ExtOp = ISD::ZERO_EXTEND;
5634 break;
5635 case ISD::UMIN:
5636 case ISD::UMAX:
5637 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5638 ExtOp = ISD::SIGN_EXTEND;
5639 else
5640 ExtOp = ISD::ZERO_EXTEND;
5641 break;
5642 }
5643 TruncOp = ISD::TRUNCATE;
5644 }
5645 // Promote each of the values to the new type.
5646 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5647 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5648 // Perform the larger operation, then convert back
5649 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5650 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5651 break;
5652 }
5653 case ISD::UMUL_LOHI:
5654 case ISD::SMUL_LOHI: {
5655 // Promote to a multiply in a wider integer type.
5656 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5657 : ISD::SIGN_EXTEND;
5658 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5659 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5660 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5661
5662 unsigned OriginalSize = OVT.getScalarSizeInBits();
5663 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5664 N2: DAG.getShiftAmountConstant(Val: OriginalSize, VT: NVT, DL: dl));
5665 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5666 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5667 break;
5668 }
5669 case ISD::SELECT: {
5670 unsigned ExtOp, TruncOp;
5671 if (Node->getValueType(ResNo: 0).isVector() ||
5672 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5673 ExtOp = ISD::BITCAST;
5674 TruncOp = ISD::BITCAST;
5675 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5676 ExtOp = ISD::ANY_EXTEND;
5677 TruncOp = ISD::TRUNCATE;
5678 } else {
5679 ExtOp = ISD::FP_EXTEND;
5680 TruncOp = ISD::FP_ROUND;
5681 }
5682 Tmp1 = Node->getOperand(Num: 0);
5683 // Promote each of the values to the new type.
5684 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5685 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5686 // Perform the larger operation, then round down.
5687 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5688 if (TruncOp != ISD::FP_ROUND)
5689 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5690 else
5691 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5692 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5693 Results.push_back(Elt: Tmp1);
5694 break;
5695 }
5696 case ISD::VECTOR_SHUFFLE: {
5697 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5698
5699 // Cast the two input vectors.
5700 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5701 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5702
5703 // Convert the shuffle mask to the right # elements.
5704 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5705 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5706 Results.push_back(Elt: Tmp1);
5707 break;
5708 }
5709 case ISD::VECTOR_SPLICE_LEFT:
5710 case ISD::VECTOR_SPLICE_RIGHT: {
5711 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5712 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5713 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5714 N3: Node->getOperand(Num: 2));
5715 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5716 break;
5717 }
5718 case ISD::SELECT_CC: {
5719 SDValue Cond = Node->getOperand(Num: 4);
5720 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5721 // Type of the comparison operands.
5722 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5723 assert(CVT == OVT && "not handled");
5724
5725 unsigned ExtOp = ISD::FP_EXTEND;
5726 if (NVT.isInteger()) {
5727 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5728 }
5729
5730 // Promote the comparison operands, if needed.
5731 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5732 Tmp1 = Node->getOperand(Num: 0);
5733 Tmp2 = Node->getOperand(Num: 1);
5734 } else {
5735 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5736 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5737 }
5738 // Cast the true/false operands.
5739 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5740 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5741
5742 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5743 Flags: Node->getFlags());
5744
5745 // Cast the result back to the original type.
5746 if (ExtOp != ISD::FP_EXTEND)
5747 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5748 else
5749 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5750 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5751
5752 Results.push_back(Elt: Tmp1);
5753 break;
5754 }
5755 case ISD::SETCC:
5756 case ISD::STRICT_FSETCC:
5757 case ISD::STRICT_FSETCCS: {
5758 unsigned ExtOp = ISD::FP_EXTEND;
5759 if (NVT.isInteger()) {
5760 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5761 if (isSignedIntSetCC(Code: CCCode) ||
5762 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5763 ExtOp = ISD::SIGN_EXTEND;
5764 else
5765 ExtOp = ISD::ZERO_EXTEND;
5766 }
5767 if (Node->isStrictFPOpcode()) {
5768 SDValue InChain = Node->getOperand(Num: 0);
5769 std::tie(args&: Tmp1, args: std::ignore) =
5770 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5771 std::tie(args&: Tmp2, args: std::ignore) =
5772 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5773 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5774 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5775 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
5776 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5777 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5778 Flags: Node->getFlags()));
5779 Results.push_back(Elt: Results.back().getValue(R: 1));
5780 break;
5781 }
5782 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5783 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5784 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5785 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5786 break;
5787 }
5788 case ISD::BR_CC: {
5789 unsigned ExtOp = ISD::FP_EXTEND;
5790 if (NVT.isInteger()) {
5791 ISD::CondCode CCCode =
5792 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5793 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5794 }
5795 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5796 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5797 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5798 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5799 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5800 break;
5801 }
5802 case ISD::FADD:
5803 case ISD::FSUB:
5804 case ISD::FMUL:
5805 case ISD::FDIV:
5806 case ISD::FREM:
5807 case ISD::FMINNUM:
5808 case ISD::FMAXNUM:
5809 case ISD::FMINIMUM:
5810 case ISD::FMAXIMUM:
5811 case ISD::FMINIMUMNUM:
5812 case ISD::FMAXIMUMNUM:
5813 case ISD::FPOW:
5814 case ISD::FATAN2:
5815 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5816 if (!OVT.isVector() && NVT.isVector() &&
5817 NVT.getVectorElementType() == OVT) {
5818 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5819 Tmp2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5820 Tmp3 =
5821 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, Flags: Node->getFlags());
5822 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Tmp3,
5823 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5824 break;
5825 }
5826 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5827 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5828 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5829 Results.push_back(
5830 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5831 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5832 break;
5833
5834 case ISD::STRICT_FMINIMUM:
5835 case ISD::STRICT_FMAXIMUM: {
5836 SDValue InChain = Node->getOperand(Num: 0);
5837 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
5838 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5839 N2: Node->getOperand(Num: 1));
5840 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5841 N2: Node->getOperand(Num: 2));
5842 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
5843 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
5844 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
5845 N1: InChain, N2: Tmp3,
5846 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5847 Results.push_back(Elt: Tmp4);
5848 Results.push_back(Elt: Tmp4.getValue(R: 1));
5849 break;
5850 }
5851
5852 case ISD::STRICT_FADD:
5853 case ISD::STRICT_FSUB:
5854 case ISD::STRICT_FMUL:
5855 case ISD::STRICT_FDIV:
5856 case ISD::STRICT_FMINNUM:
5857 case ISD::STRICT_FMAXNUM:
5858 case ISD::STRICT_FREM:
5859 case ISD::STRICT_FPOW:
5860 case ISD::STRICT_FATAN2:
5861 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5862 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5863 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5864 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5865 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5866 N2: Tmp2.getValue(R: 1));
5867 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5868 Ops: {Tmp3, Tmp1, Tmp2});
5869 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5870 Ops: {Tmp1.getValue(R: 1), Tmp1,
5871 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5872 Results.push_back(Elt: Tmp1);
5873 Results.push_back(Elt: Tmp1.getValue(R: 1));
5874 break;
5875 case ISD::FMA:
5876 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5877 if (!OVT.isVector() && NVT.isVector() &&
5878 NVT.getVectorElementType() == OVT) {
5879 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5880 Tmp2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5881 Tmp3 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5882 SDValue Result = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3,
5883 Flags: Node->getFlags());
5884 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Result,
5885 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5886 break;
5887 }
5888 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5889 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5890 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5891 Results.push_back(
5892 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5893 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5894 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5895 break;
5896 case ISD::STRICT_FMA:
5897 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5898 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5899 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5900 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5901 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5902 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
5903 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5904 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
5905 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5906 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
5907 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5908 Ops: {Tmp4.getValue(R: 1), Tmp4,
5909 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5910 Results.push_back(Elt: Tmp4);
5911 Results.push_back(Elt: Tmp4.getValue(R: 1));
5912 break;
5913 case ISD::FCOPYSIGN:
5914 case ISD::FLDEXP:
5915 case ISD::FPOWI: {
5916 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5917 Tmp2 = Node->getOperand(Num: 1);
5918 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5919
5920 // fcopysign doesn't change anything but the sign bit, so
5921 // (fp_round (fcopysign (fpext a), b))
5922 // is as precise as
5923 // (fp_round (fpext a))
5924 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5925 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5926 Results.push_back(
5927 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5928 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5929 break;
5930 }
5931 case ISD::STRICT_FLDEXP: {
5932 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5933 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5934 Tmp2 = Node->getOperand(Num: 2);
5935 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
5936 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
5937 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5938 Ops: {Tmp3.getValue(R: 1), Tmp3,
5939 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5940 Results.push_back(Elt: Tmp4);
5941 Results.push_back(Elt: Tmp4.getValue(R: 1));
5942 break;
5943 }
5944 case ISD::STRICT_FPOWI:
5945 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5946 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5947 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5948 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
5949 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5950 Ops: {Tmp2.getValue(R: 1), Tmp2,
5951 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5952 Results.push_back(Elt: Tmp3);
5953 Results.push_back(Elt: Tmp3.getValue(R: 1));
5954 break;
5955 case ISD::FFREXP: {
5956 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5957 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5958
5959 Results.push_back(
5960 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5961 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5962
5963 Results.push_back(Elt: Tmp2.getValue(R: 1));
5964 break;
5965 }
5966 case ISD::FMODF:
5967 case ISD::FSINCOS:
5968 case ISD::FSINCOSPI: {
5969 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5970 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
5971 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
5972 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5973 Results.push_back(
5974 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
5975 break;
5976 }
5977 case ISD::FFLOOR:
5978 case ISD::FCEIL:
5979 case ISD::FRINT:
5980 case ISD::FNEARBYINT:
5981 case ISD::FROUND:
5982 case ISD::FROUNDEVEN:
5983 case ISD::FTRUNC:
5984 case ISD::FNEG:
5985 case ISD::FSQRT:
5986 case ISD::FSIN:
5987 case ISD::FCOS:
5988 case ISD::FTAN:
5989 case ISD::FASIN:
5990 case ISD::FACOS:
5991 case ISD::FATAN:
5992 case ISD::FSINH:
5993 case ISD::FCOSH:
5994 case ISD::FTANH:
5995 case ISD::FLOG:
5996 case ISD::FLOG2:
5997 case ISD::FLOG10:
5998 case ISD::FABS:
5999 case ISD::FEXP:
6000 case ISD::FEXP2:
6001 case ISD::FEXP10:
6002 case ISD::FCANONICALIZE:
6003 // Promote scalar operations to vector using SCALAR_TO_VECTOR
6004 if (!OVT.isVector() && NVT.isVector() &&
6005 NVT.getVectorElementType() == OVT) {
6006 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6007 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1, Flags: Node->getFlags());
6008 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Tmp2,
6009 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
6010 break;
6011 }
6012 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6013 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6014 Results.push_back(
6015 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6016 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6017 break;
6018 case ISD::STRICT_FFLOOR:
6019 case ISD::STRICT_FCEIL:
6020 case ISD::STRICT_FRINT:
6021 case ISD::STRICT_FNEARBYINT:
6022 case ISD::STRICT_FROUND:
6023 case ISD::STRICT_FROUNDEVEN:
6024 case ISD::STRICT_FTRUNC:
6025 case ISD::STRICT_FSQRT:
6026 case ISD::STRICT_FSIN:
6027 case ISD::STRICT_FCOS:
6028 case ISD::STRICT_FTAN:
6029 case ISD::STRICT_FASIN:
6030 case ISD::STRICT_FACOS:
6031 case ISD::STRICT_FATAN:
6032 case ISD::STRICT_FSINH:
6033 case ISD::STRICT_FCOSH:
6034 case ISD::STRICT_FTANH:
6035 case ISD::STRICT_FLOG:
6036 case ISD::STRICT_FLOG2:
6037 case ISD::STRICT_FLOG10:
6038 case ISD::STRICT_FEXP:
6039 case ISD::STRICT_FEXP2:
6040 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6041 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6042 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6043 Ops: {Tmp1.getValue(R: 1), Tmp1});
6044 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6045 Ops: {Tmp2.getValue(R: 1), Tmp2,
6046 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6047 Results.push_back(Elt: Tmp3);
6048 Results.push_back(Elt: Tmp3.getValue(R: 1));
6049 break;
6050 case ISD::LLROUND:
6051 case ISD::LROUND:
6052 case ISD::LRINT:
6053 case ISD::LLRINT:
6054 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6055 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
6056 Results.push_back(Elt: Tmp2);
6057 break;
6058 case ISD::STRICT_LLROUND:
6059 case ISD::STRICT_LROUND:
6060 case ISD::STRICT_LRINT:
6061 case ISD::STRICT_LLRINT:
6062 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6063 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6064 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6065 Ops: {Tmp1.getValue(R: 1), Tmp1});
6066 Results.push_back(Elt: Tmp2);
6067 Results.push_back(Elt: Tmp2.getValue(R: 1));
6068 break;
6069 case ISD::BUILD_VECTOR: {
6070 MVT EltVT = OVT.getVectorElementType();
6071 MVT NewEltVT = NVT.getVectorElementType();
6072
6073 // Handle bitcasts to a different vector type with the same total bit size
6074 //
6075 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
6076 // =>
6077 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
6078
6079 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6080 "Invalid promote type for build_vector");
6081 assert(NewEltVT.bitsLE(EltVT) && "not handled");
6082
6083 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6084
6085 SmallVector<SDValue, 8> NewOps;
6086 for (const SDValue &Op : Node->op_values())
6087 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
6088
6089 SDLoc SL(Node);
6090 SDValue Concat =
6091 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
6092 DL: SL, VT: NVT, Ops: NewOps);
6093 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6094 Results.push_back(Elt: CvtVec);
6095 break;
6096 }
6097 case ISD::EXTRACT_VECTOR_ELT: {
6098 MVT EltVT = OVT.getVectorElementType();
6099 MVT NewEltVT = NVT.getVectorElementType();
6100
6101 // Handle bitcasts to a different vector type with the same total bit size.
6102 //
6103 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
6104 // =>
6105 // v4i32:castx = bitcast x:v2i64
6106 //
6107 // i64 = bitcast
6108 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
6109 // (i32 (extract_vector_elt castx, (2 * y + 1)))
6110 //
6111
6112 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6113 "Invalid promote type for extract_vector_elt");
6114 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6115
6116 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6117 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6118
6119 SDValue Idx = Node->getOperand(Num: 1);
6120 EVT IdxVT = Idx.getValueType();
6121 SDLoc SL(Node);
6122 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
6123 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6124
6125 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6126
6127 SmallVector<SDValue, 8> NewOps;
6128 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6129 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6130 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6131
6132 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6133 N1: CastVec, N2: TmpIdx);
6134 NewOps.push_back(Elt);
6135 }
6136
6137 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
6138 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
6139 break;
6140 }
6141 case ISD::INSERT_VECTOR_ELT: {
6142 MVT EltVT = OVT.getVectorElementType();
6143 MVT NewEltVT = NVT.getVectorElementType();
6144
6145 // Handle bitcasts to a different vector type with the same total bit size
6146 //
6147 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
6148 // =>
6149 // v4i32:castx = bitcast x:v2i64
6150 // v2i32:casty = bitcast y:i64
6151 //
6152 // v2i64 = bitcast
6153 // (v4i32 insert_vector_elt
6154 // (v4i32 insert_vector_elt v4i32:castx,
6155 // (extract_vector_elt casty, 0), 2 * z),
6156 // (extract_vector_elt casty, 1), (2 * z + 1))
6157
6158 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6159 "Invalid promote type for insert_vector_elt");
6160 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6161
6162 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6163 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6164
6165 SDValue Val = Node->getOperand(Num: 1);
6166 SDValue Idx = Node->getOperand(Num: 2);
6167 EVT IdxVT = Idx.getValueType();
6168 SDLoc SL(Node);
6169
6170 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
6171 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6172
6173 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6174 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6175
6176 SDValue NewVec = CastVec;
6177 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6178 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6179 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6180
6181 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6182 N1: CastVal, N2: IdxOffset);
6183
6184 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
6185 N1: NewVec, N2: Elt, N3: InEltIdx);
6186 }
6187
6188 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
6189 break;
6190 }
6191 case ISD::SCALAR_TO_VECTOR: {
6192 MVT EltVT = OVT.getVectorElementType();
6193 MVT NewEltVT = NVT.getVectorElementType();
6194
6195 // Handle bitcasts to different vector type with the same total bit size.
6196 //
6197 // e.g. v2i64 = scalar_to_vector x:i64
6198 // =>
6199 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
6200 //
6201
6202 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6203 SDValue Val = Node->getOperand(Num: 0);
6204 SDLoc SL(Node);
6205
6206 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6207 SDValue Undef = DAG.getUNDEF(VT: MidVT);
6208
6209 SmallVector<SDValue, 8> NewElts;
6210 NewElts.push_back(Elt: CastVal);
6211 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
6212 NewElts.push_back(Elt: Undef);
6213
6214 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
6215 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6216 Results.push_back(Elt: CvtVec);
6217 break;
6218 }
6219 case ISD::ATOMIC_SWAP:
6220 case ISD::ATOMIC_STORE: {
6221 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6222 SDLoc SL(Node);
6223 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
6224 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6225 "unexpected promotion type");
6226 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6227 "unexpected atomic_swap with illegal type");
6228
6229 SDValue Op0 = AM->getBasePtr();
6230 SDValue Op1 = CastVal;
6231
6232 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
6233 // but really it should merge with ISD::STORE.
6234 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6235 std::swap(a&: Op0, b&: Op1);
6236
6237 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
6238 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
6239
6240 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6241 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6242 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6243 } else
6244 Results.push_back(Elt: NewAtomic);
6245 break;
6246 }
6247 case ISD::ATOMIC_LOAD: {
6248 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6249 SDLoc SL(Node);
6250 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6251 "unexpected promotion type");
6252 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6253 "unexpected atomic_load with illegal type");
6254
6255 SDValue NewAtomic =
6256 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
6257 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
6258 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6259 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6260 break;
6261 }
6262 case ISD::SPLAT_VECTOR: {
6263 SDValue Scalar = Node->getOperand(Num: 0);
6264 MVT ScalarType = Scalar.getSimpleValueType();
6265 MVT NewScalarType = NVT.getVectorElementType();
6266 if (ScalarType.isInteger()) {
6267 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6268 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6269 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
6270 break;
6271 }
6272 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6273 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6274 Results.push_back(
6275 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6276 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6277 break;
6278 }
6279 case ISD::VECREDUCE_FMAX:
6280 case ISD::VECREDUCE_FMIN:
6281 case ISD::VECREDUCE_FMAXIMUM:
6282 case ISD::VECREDUCE_FMINIMUM:
6283 case ISD::VECREDUCE_FMAXIMUMNUM:
6284 case ISD::VECREDUCE_FMINIMUMNUM:
6285 case ISD::VP_REDUCE_FMAX:
6286 case ISD::VP_REDUCE_FMIN:
6287 case ISD::VP_REDUCE_FMAXIMUM:
6288 case ISD::VP_REDUCE_FMINIMUM:
6289 Results.push_back(Elt: PromoteReduction(Node));
6290 break;
6291 }
6292
6293 // Replace the original node with the legalized result.
6294 if (!Results.empty()) {
6295 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6296 ReplaceNode(Old: Node, New: Results.data());
6297 } else
6298 LLVM_DEBUG(dbgs() << "Could not promote node\n");
6299}
6300
6301/// This is the entry point for the file.
6302void SelectionDAG::Legalize() {
6303 AssignTopologicalOrder();
6304
6305 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6306 // Use a delete listener to remove nodes which were deleted during
6307 // legalization from LegalizeNodes. This is needed to handle the situation
6308 // where a new node is allocated by the object pool to the same address of a
6309 // previously deleted node.
6310 DAGNodeDeletedListener DeleteListener(
6311 *this,
6312 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
6313
6314 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6315
6316 // Visit all the nodes. We start in topological order, so that we see
6317 // nodes with their original operands intact. Legalization can produce
6318 // new nodes which may themselves need to be legalized. Iterate until all
6319 // nodes have been legalized.
6320 while (true) {
6321 bool AnyLegalized = false;
6322 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6323 --NI;
6324
6325 SDNode *N = &*NI;
6326 if (N->use_empty() && N != getRoot().getNode()) {
6327 ++NI;
6328 DeleteNode(N);
6329 continue;
6330 }
6331
6332 if (LegalizedNodes.insert(Ptr: N).second) {
6333 AnyLegalized = true;
6334 Legalizer.LegalizeOp(Node: N);
6335
6336 if (N->use_empty() && N != getRoot().getNode()) {
6337 ++NI;
6338 DeleteNode(N);
6339 }
6340 }
6341 }
6342 if (!AnyLegalized)
6343 break;
6344
6345 }
6346
6347 // Remove dead nodes now.
6348 RemoveDeadNodes();
6349}
6350
6351bool SelectionDAG::LegalizeOp(SDNode *N,
6352 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6353 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6354 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6355
6356 // Directly insert the node in question, and legalize it. This will recurse
6357 // as needed through operands.
6358 LegalizedNodes.insert(Ptr: N);
6359 Legalizer.LegalizeOp(Node: N);
6360
6361 return LegalizedNodes.count(Ptr: N);
6362}
6363