1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SelectionDAGNodes.h"
32#include "llvm/CodeGen/TargetFrameLowering.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/CodeGenTypes/MachineValueType.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Compiler.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/MathExtras.h"
49#include "llvm/Support/raw_ostream.h"
50#include "llvm/Target/TargetMachine.h"
51#include "llvm/Target/TargetOptions.h"
52#include <cassert>
53#include <cstdint>
54#include <tuple>
55#include <utility>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "legalizedag"
60
61namespace {
62
63/// Keeps track of state when getting the sign of a floating-point value as an
64/// integer.
65struct FloatSignAsInt {
66 EVT FloatVT;
67 SDValue Chain;
68 SDValue FloatPtr;
69 SDValue IntPtr;
70 MachinePointerInfo IntPointerInfo;
71 MachinePointerInfo FloatPointerInfo;
72 SDValue IntValue;
73 APInt SignMask;
74 uint8_t SignBit;
75};
76
77//===----------------------------------------------------------------------===//
78/// This takes an arbitrary SelectionDAG as input and
79/// hacks on it until the target machine can handle it. This involves
80/// eliminating value sizes the machine cannot handle (promoting small sizes to
81/// large sizes or splitting up large values into small values) as well as
82/// eliminating operations the machine cannot handle.
83///
84/// This code also does a small amount of optimization and recognition of idioms
85/// as part of its processing. For example, if a target does not support a
86/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
87/// will attempt merge setcc and brc instructions into brcc's.
88class SelectionDAGLegalize {
89 const TargetMachine &TM;
90 const TargetLowering &TLI;
91 SelectionDAG &DAG;
92
93 /// The set of nodes which have already been legalized. We hold a
94 /// reference to it in order to update as necessary on node deletion.
95 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
96
97 /// A set of all the nodes updated during legalization.
98 SmallSetVector<SDNode *, 16> *UpdatedNodes;
99
100 EVT getSetCCResultType(EVT VT) const {
101 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
102 }
103
104 // Libcall insertion helpers.
105
106public:
107 SelectionDAGLegalize(SelectionDAG &DAG,
108 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
109 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
110 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
112
113 /// Legalizes the given operation.
114 void LegalizeOp(SDNode *Node);
115
116private:
117 SDValue OptimizeFloatStore(StoreSDNode *ST);
118
119 void LegalizeLoadOps(SDNode *Node);
120 void LegalizeStoreOps(SDNode *Node);
121
122 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
123
124 /// Return a vector shuffle operation which
125 /// performs the same shuffe in terms of order or result bytes, but on a type
126 /// whose vector element type is narrower than the original shuffle type.
127 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
128 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
129 SDValue N1, SDValue N2,
130 ArrayRef<int> Mask) const;
131
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 TargetLowering::ArgListTy &&Args,
134 bool IsSigned, EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
136
137 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 SmallVectorImpl<SDValue> &Results);
139
140 void
141 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
142 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
143 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
144 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
145 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
146 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
147 SmallVectorImpl<SDValue> &Results);
148
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
150 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
151 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
152 void ExpandArgFPLibCall(SDNode *Node,
153 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
154 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
155 RTLIB::Libcall Call_PPCF128,
156 SmallVectorImpl<SDValue> &Results);
157 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
158 RTLIB::Libcall CallI64,
159 RTLIB::Libcall CallI128);
160 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161
162 SDValue ExpandSincosStretLibCall(SDNode *Node) const;
163
164 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
165 const SDLoc &dl);
166 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
167 const SDLoc &dl, SDValue ChainIn);
168 SDValue ExpandBUILD_VECTOR(SDNode *Node);
169 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
170 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
171 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
172 SmallVectorImpl<SDValue> &Results);
173 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
174 SDValue Value) const;
175 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
176 SDValue NewIntValue) const;
177 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
178 SDValue ExpandFABS(SDNode *Node) const;
179 SDValue ExpandFNEG(SDNode *Node) const;
180 SDValue expandLdexp(SDNode *Node) const;
181 SDValue expandFrexp(SDNode *Node) const;
182 SDValue expandModf(SDNode *Node) const;
183
184 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
185 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
186 SmallVectorImpl<SDValue> &Results);
187 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
188 SmallVectorImpl<SDValue> &Results);
189 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
190
191 /// Implements vector reduce operation promotion.
192 ///
193 /// All vector operands are promoted to a vector type with larger element
194 /// type, and the start value is promoted to a larger scalar type. Then the
195 /// result is truncated back to the original scalar type.
196 SDValue PromoteReduction(SDNode *Node);
197
198 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
199
200 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
201 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
202 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
203 SDValue ExpandConcatVectors(SDNode *Node);
204
205 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
206 SDValue ExpandConstant(ConstantSDNode *CP);
207
208 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
209 bool ExpandNode(SDNode *Node);
210 void ConvertNodeToLibcall(SDNode *Node);
211 void PromoteNode(SDNode *Node);
212
213public:
214 // Node replacement helpers
215
216 void ReplacedNode(SDNode *N) {
217 LegalizedNodes.erase(Ptr: N);
218 if (UpdatedNodes)
219 UpdatedNodes->insert(X: N);
220 }
221
222 void ReplaceNode(SDNode *Old, SDNode *New) {
223 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
224 dbgs() << " with: "; New->dump(&DAG));
225
226 assert(Old->getNumValues() == New->getNumValues() &&
227 "Replacing one node with another that produces a different number "
228 "of values!");
229 DAG.ReplaceAllUsesWith(From: Old, To: New);
230 if (UpdatedNodes)
231 UpdatedNodes->insert(X: New);
232 ReplacedNode(N: Old);
233 }
234
235 void ReplaceNode(SDValue Old, SDValue New) {
236 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
237 dbgs() << " with: "; New->dump(&DAG));
238
239 DAG.ReplaceAllUsesWith(From: Old, To: New);
240 if (UpdatedNodes)
241 UpdatedNodes->insert(X: New.getNode());
242 ReplacedNode(N: Old.getNode());
243 }
244
245 void ReplaceNode(SDNode *Old, const SDValue *New) {
246 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
247
248 DAG.ReplaceAllUsesWith(From: Old, To: New);
249 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
250 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
251 New[i]->dump(&DAG));
252 if (UpdatedNodes)
253 UpdatedNodes->insert(X: New[i].getNode());
254 }
255 ReplacedNode(N: Old);
256 }
257
258 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
259 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
260 dbgs() << " with: "; New->dump(&DAG));
261
262 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
263 if (UpdatedNodes)
264 UpdatedNodes->insert(X: New.getNode());
265 ReplacedNode(N: Old.getNode());
266 }
267};
268
269} // end anonymous namespace
270
271// Helper function that generates an MMO that considers the alignment of the
272// stack, and the size of the stack object
273static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
274 MachineFunction &MF,
275 bool isObjectScalable) {
276 auto &MFI = MF.getFrameInfo();
277 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
278 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
279 LocationSize ObjectSize = isObjectScalable
280 ? LocationSize::beforeOrAfterPointer()
281 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
282 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
283 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
284}
285
286/// Return a vector shuffle operation which
287/// performs the same shuffle in terms of order or result bytes, but on a type
288/// whose vector element type is narrower than the original shuffle type.
289/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
290SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
291 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
292 ArrayRef<int> Mask) const {
293 unsigned NumMaskElts = VT.getVectorNumElements();
294 unsigned NumDestElts = NVT.getVectorNumElements();
295 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
296
297 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
298
299 if (NumEltsGrowth == 1)
300 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
301
302 SmallVector<int, 8> NewMask;
303 for (unsigned i = 0; i != NumMaskElts; ++i) {
304 int Idx = Mask[i];
305 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
306 if (Idx < 0)
307 NewMask.push_back(Elt: -1);
308 else
309 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
310 }
311 }
312 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
313 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
314 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
315}
316
317/// Expands the ConstantFP node to an integer constant or
318/// a load from the constant pool.
319SDValue
320SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
321 bool Extend = false;
322 SDLoc dl(CFP);
323
324 // If a FP immediate is precise when represented as a float and if the
325 // target can do an extending load from float to double, we put it into
326 // the constant pool as a float, even if it's is statically typed as a
327 // double. This shrinks FP constants and canonicalizes them for targets where
328 // an FP extending load is the same cost as a normal load (such as on the x87
329 // fp stack or PPC FP unit).
330 EVT VT = CFP->getValueType(ResNo: 0);
331 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
332 if (!UseCP) {
333 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
334 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
335 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
336 }
337
338 APFloat APF = CFP->getValueAPF();
339 EVT OrigVT = VT;
340 EVT SVT = VT;
341
342 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
343 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
344 if (!APF.isSignaling()) {
345 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
346 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
347 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
348 // Only do this if the target has a native EXTLOAD instruction from
349 // smaller type.
350 TLI.isLoadLegal(
351 ValVT: OrigVT, MemVT: SVT,
352 Alignment: Align(DAG.getDataLayout().getPrefTypeAlign(
353 Ty: SVT.getTypeForEVT(Context&: *DAG.getContext()))),
354 AddrSpace: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction())
355 .getAddrSpace(),
356 ExtType: ISD::EXTLOAD, Atomic: false) &&
357 TLI.ShouldShrinkFPConstant(OrigVT)) {
358 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
359 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
360 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
361 VT = SVT;
362 Extend = true;
363 }
364 }
365 }
366
367 SDValue CPIdx =
368 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
369 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
370 if (Extend) {
371 SDValue Result = DAG.getExtLoad(
372 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
373 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
374 Alignment);
375 return Result;
376 }
377 SDValue Result = DAG.getLoad(
378 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
379 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
380 return Result;
381}
382
383/// Expands the Constant node to a load from the constant pool.
384SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
385 SDLoc dl(CP);
386 EVT VT = CP->getValueType(ResNo: 0);
387 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
388 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
389 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
390 SDValue Result = DAG.getLoad(
391 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
392 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
393 return Result;
394}
395
396SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
397 SDValue Vec = Op.getOperand(i: 0);
398 SDValue Val = Op.getOperand(i: 1);
399 SDValue Idx = Op.getOperand(i: 2);
400 SDLoc dl(Op);
401
402 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
403 // SCALAR_TO_VECTOR requires that the type of the value being inserted
404 // match the element type of the vector being created, except for
405 // integers in which case the inserted value can be over width.
406 EVT EltVT = Vec.getValueType().getVectorElementType();
407 if (Val.getValueType() == EltVT ||
408 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
409 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
410 VT: Vec.getValueType(), Operand: Val);
411
412 unsigned NumElts = Vec.getValueType().getVectorNumElements();
413 // We generate a shuffle of InVec and ScVec, so the shuffle mask
414 // should be 0,1,2,3,4,5... with the appropriate element replaced with
415 // elt 0 of the RHS.
416 SmallVector<int, 8> ShufOps;
417 for (unsigned i = 0; i != NumElts; ++i)
418 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
419
420 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
421 }
422 }
423 return ExpandInsertToVectorThroughStack(Op);
424}
425
426SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
427 if (!ISD::isNormalStore(N: ST))
428 return SDValue();
429
430 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
431 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
432 // FIXME: move this to the DAG Combiner! Note that we can't regress due
433 // to phase ordering between legalized code and the dag combiner. This
434 // probably means that we need to integrate dag combiner and legalizer
435 // together.
436 // We generally can't do this one for long doubles.
437 SDValue Chain = ST->getChain();
438 SDValue Ptr = ST->getBasePtr();
439 SDValue Value = ST->getValue();
440 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
441 AAMDNodes AAInfo = ST->getAAInfo();
442 SDLoc dl(ST);
443
444 // Don't optimise TargetConstantFP
445 if (Value.getOpcode() == ISD::TargetConstantFP)
446 return SDValue();
447
448 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
449 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
450 TLI.isTypeLegal(VT: MVT::i32)) {
451 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
452 bitcastToAPInt().zextOrTrunc(width: 32),
453 DL: SDLoc(CFP), VT: MVT::i32);
454 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
455 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
456 }
457
458 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
459 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
460 // If this target supports 64-bit registers, do a single 64-bit store.
461 if (TLI.isTypeLegal(VT: MVT::i64)) {
462 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
463 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
464 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
465 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
466 }
467
468 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
469 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
470 // stores. If the target supports neither 32- nor 64-bits, this
471 // xform is certainly not worth it.
472 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
473 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
474 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
475 if (DAG.getDataLayout().isBigEndian())
476 std::swap(a&: Lo, b&: Hi);
477
478 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
479 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
480 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
481 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
482 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
483 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
484
485 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
486 }
487 }
488 }
489 return SDValue();
490}
491
492void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
493 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
494 SDValue Chain = ST->getChain();
495 SDValue Ptr = ST->getBasePtr();
496 SDLoc dl(Node);
497
498 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
499 AAMDNodes AAInfo = ST->getAAInfo();
500
501 if (!ST->isTruncatingStore()) {
502 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
503 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
504 ReplaceNode(Old: ST, New: OptStore);
505 return;
506 }
507
508 SDValue Value = ST->getValue();
509 MVT VT = Value.getSimpleValueType();
510 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
511 default: llvm_unreachable("This action is not supported yet!");
512 case TargetLowering::Legal: {
513 // If this is an unaligned store and the target doesn't support it,
514 // expand it.
515 EVT MemVT = ST->getMemoryVT();
516 const DataLayout &DL = DAG.getDataLayout();
517 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
518 MMO: *ST->getMemOperand())) {
519 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
520 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
521 ReplaceNode(Old: SDValue(ST, 0), New: Result);
522 } else
523 LLVM_DEBUG(dbgs() << "Legal store\n");
524 break;
525 }
526 case TargetLowering::Custom: {
527 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
528 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
529 if (Res && Res != SDValue(Node, 0))
530 ReplaceNode(Old: SDValue(Node, 0), New: Res);
531 return;
532 }
533 case TargetLowering::Promote: {
534 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
535 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
536 "Can only promote stores to same size type");
537 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
538 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
539 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
540 ReplaceNode(Old: SDValue(Node, 0), New: Result);
541 break;
542 }
543 }
544 return;
545 }
546
547 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
548 SDValue Value = ST->getValue();
549 EVT StVT = ST->getMemoryVT();
550 TypeSize StWidth = StVT.getSizeInBits();
551 TypeSize StSize = StVT.getStoreSizeInBits();
552 auto &DL = DAG.getDataLayout();
553
554 if (StWidth != StSize) {
555 // Promote to a byte-sized store with upper bits zero if not
556 // storing an integral number of bytes. For example, promote
557 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
558 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
559 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
560 SDValue Result =
561 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
562 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
563 ReplaceNode(Old: SDValue(Node, 0), New: Result);
564 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
565 // If not storing a power-of-2 number of bits, expand as two stores.
566 assert(!StVT.isVector() && "Unsupported truncstore!");
567 unsigned StWidthBits = StWidth.getFixedValue();
568 unsigned LogStWidth = Log2_32(Value: StWidthBits);
569 assert(LogStWidth < 32);
570 unsigned RoundWidth = 1 << LogStWidth;
571 assert(RoundWidth < StWidthBits);
572 unsigned ExtraWidth = StWidthBits - RoundWidth;
573 assert(ExtraWidth < RoundWidth);
574 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
575 "Store size not an integral number of bytes!");
576 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
577 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
578 SDValue Lo, Hi;
579 unsigned IncrementSize;
580
581 if (DL.isLittleEndian()) {
582 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
583 // Store the bottom RoundWidth bits.
584 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
585 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
586
587 // Store the remaining ExtraWidth bits.
588 IncrementSize = RoundWidth / 8;
589 Ptr =
590 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
591 Hi = DAG.getNode(
592 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
593 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Value.getValueType(), DL: dl));
594 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
595 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
596 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
597 } else {
598 // Big endian - avoid unaligned stores.
599 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
600 // Store the top RoundWidth bits.
601 Hi = DAG.getNode(
602 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
603 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Value.getValueType(), DL: dl));
604 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
605 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
606
607 // Store the remaining ExtraWidth bits.
608 IncrementSize = RoundWidth / 8;
609 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
610 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
611 VT: Ptr.getValueType()));
612 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
613 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
614 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
615 }
616
617 // The order of the stores doesn't matter.
618 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
619 ReplaceNode(Old: SDValue(Node, 0), New: Result);
620 } else {
621 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT,
622 Alignment: ST->getAlign(), AddrSpace: ST->getAddressSpace())) {
623 default:
624 llvm_unreachable("This action is not supported yet!");
625 case TargetLowering::Legal: {
626 EVT MemVT = ST->getMemoryVT();
627 // If this is an unaligned store and the target doesn't support it,
628 // expand it.
629 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
630 MMO: *ST->getMemOperand())) {
631 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
632 ReplaceNode(Old: SDValue(ST, 0), New: Result);
633 }
634 break;
635 }
636 case TargetLowering::Custom: {
637 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
638 if (Res && Res != SDValue(Node, 0))
639 ReplaceNode(Old: SDValue(Node, 0), New: Res);
640 return;
641 }
642 case TargetLowering::Expand:
643 assert(!StVT.isVector() &&
644 "Vector Stores are handled in LegalizeVectorOps");
645
646 SDValue Result;
647
648 // TRUNCSTORE:i16 i32 -> STORE i16
649 if (TLI.isTypeLegal(VT: StVT)) {
650 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
651 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
652 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
653 } else {
654 // The in-memory type isn't legal. Truncate to the type it would promote
655 // to, and then do a truncstore.
656 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
657 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
658 Operand: Value);
659 Result = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
660 SVT: StVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
661 }
662
663 ReplaceNode(Old: SDValue(Node, 0), New: Result);
664 break;
665 }
666 }
667}
668
669void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
670 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
671 SDValue Chain = LD->getChain(); // The chain.
672 SDValue Ptr = LD->getBasePtr(); // The base pointer.
673 SDValue Value; // The value returned by the load op.
674 SDLoc dl(Node);
675
676 ISD::LoadExtType ExtType = LD->getExtensionType();
677 if (ExtType == ISD::NON_EXTLOAD) {
678 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
679 MVT VT = Node->getSimpleValueType(ResNo: 0);
680 SDValue RVal = SDValue(Node, 0);
681 SDValue RChain = SDValue(Node, 1);
682
683 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
684 default: llvm_unreachable("This action is not supported yet!");
685 case TargetLowering::Legal: {
686 EVT MemVT = LD->getMemoryVT();
687 const DataLayout &DL = DAG.getDataLayout();
688 // If this is an unaligned load and the target doesn't support it,
689 // expand it.
690 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
691 MMO: *LD->getMemOperand())) {
692 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
693 }
694 break;
695 }
696 case TargetLowering::Custom:
697 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
698 RVal = Res;
699 RChain = Res.getValue(R: 1);
700 }
701 break;
702
703 case TargetLowering::Promote: {
704 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
705 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
706 "Can only promote loads to same size type");
707
708 // If the range metadata type does not match the legalized memory
709 // operation type, remove the range metadata.
710 if (const MDNode *MD = LD->getRanges()) {
711 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
712 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
713 !NVT.isInteger())
714 LD->getMemOperand()->clearRanges();
715 }
716 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
717 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
718 RChain = Res.getValue(R: 1);
719 break;
720 }
721 }
722 if (RChain.getNode() != Node) {
723 assert(RVal.getNode() != Node && "Load must be completely replaced");
724 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
725 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
726 if (UpdatedNodes) {
727 UpdatedNodes->insert(X: RVal.getNode());
728 UpdatedNodes->insert(X: RChain.getNode());
729 }
730 ReplacedNode(N: Node);
731 }
732 return;
733 }
734
735 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
736 EVT SrcVT = LD->getMemoryVT();
737 TypeSize SrcWidth = SrcVT.getSizeInBits();
738 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
739 AAMDNodes AAInfo = LD->getAAInfo();
740
741 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
742 // Some targets pretend to have an i1 loading operation, and actually
743 // load an i8. This trick is correct for ZEXTLOAD because the top 7
744 // bits are guaranteed to be zero; it helps the optimizers understand
745 // that these bits are zero. It is also useful for EXTLOAD, since it
746 // tells the optimizers that those bits are undefined. It would be
747 // nice to have an effective generic way of getting these benefits...
748 // Until such a way is found, don't insist on promoting i1 here.
749 (SrcVT != MVT::i1 ||
750 TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1, Alignment: LD->getAlign(),
751 AddrSpace: LD->getAddressSpace(), ExtType,
752 Atomic: false) == TargetLowering::Promote)) {
753 // Promote to a byte-sized load if not loading an integral number of
754 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
755 unsigned NewWidth = SrcVT.getStoreSizeInBits();
756 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
757 SDValue Ch;
758
759 // The extra bits are guaranteed to be zero, since we stored them that
760 // way. A zext load from NVT thus automatically gives zext from SrcVT.
761
762 ISD::LoadExtType NewExtType =
763 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
764
765 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
766 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
767 Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
768
769 Ch = Result.getValue(R: 1); // The chain.
770
771 if (ExtType == ISD::SEXTLOAD)
772 // Having the top bits zero doesn't help when sign extending.
773 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
774 VT: Result.getValueType(),
775 N1: Result, N2: DAG.getValueType(SrcVT));
776 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
777 // All the top bits are guaranteed to be zero - inform the optimizers.
778 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
779 VT: Result.getValueType(), N1: Result,
780 N2: DAG.getValueType(SrcVT));
781
782 Value = Result;
783 Chain = Ch;
784 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
785 // If not loading a power-of-2 number of bits, expand as two loads.
786 assert(!SrcVT.isVector() && "Unsupported extload!");
787 unsigned SrcWidthBits = SrcWidth.getFixedValue();
788 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
789 assert(LogSrcWidth < 32);
790 unsigned RoundWidth = 1 << LogSrcWidth;
791 assert(RoundWidth < SrcWidthBits);
792 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
793 assert(ExtraWidth < RoundWidth);
794 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
795 "Load size not an integral number of bytes!");
796 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
797 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
798 SDValue Lo, Hi, Ch;
799 unsigned IncrementSize;
800 auto &DL = DAG.getDataLayout();
801
802 if (DL.isLittleEndian()) {
803 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
804 // Load the bottom RoundWidth bits.
805 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
806 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
807 MMOFlags, Metadata: AAInfo);
808
809 // Load the remaining ExtraWidth bits.
810 IncrementSize = RoundWidth / 8;
811 Ptr =
812 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
813 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
814 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
815 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
816
817 // Build a factor node to remember that this load is independent of
818 // the other one.
819 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
820 N2: Hi.getValue(R: 1));
821
822 // Move the top bits to the right place.
823 Hi = DAG.getNode(
824 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
825 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Hi.getValueType(), DL: dl));
826
827 // Join the hi and lo parts.
828 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
829 } else {
830 // Big endian - avoid unaligned loads.
831 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
832 // Load the top RoundWidth bits.
833 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
834 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
835 MMOFlags, Metadata: AAInfo);
836
837 // Load the remaining ExtraWidth bits.
838 IncrementSize = RoundWidth / 8;
839 Ptr =
840 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
841 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
842 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
843 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
844
845 // Build a factor node to remember that this load is independent of
846 // the other one.
847 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
848 N2: Hi.getValue(R: 1));
849
850 // Move the top bits to the right place.
851 Hi = DAG.getNode(
852 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
853 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Hi.getValueType(), DL: dl));
854
855 // Join the hi and lo parts.
856 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
857 }
858
859 Chain = Ch;
860 } else {
861 bool isCustom = false;
862 switch (TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: SrcVT.getSimpleVT(),
863 Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(), ExtType,
864 Atomic: false)) {
865 default:
866 llvm_unreachable("This action is not supported yet!");
867 case TargetLowering::Custom:
868 isCustom = true;
869 [[fallthrough]];
870 case TargetLowering::Legal:
871 Value = SDValue(Node, 0);
872 Chain = SDValue(Node, 1);
873
874 if (isCustom) {
875 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
876 Value = Res;
877 Chain = Res.getValue(R: 1);
878 }
879 } else {
880 // If this is an unaligned load and the target doesn't support it,
881 // expand it.
882 EVT MemVT = LD->getMemoryVT();
883 const DataLayout &DL = DAG.getDataLayout();
884 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
885 MMO: *LD->getMemOperand())) {
886 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
887 }
888 }
889 break;
890
891 case TargetLowering::Expand: {
892 EVT DestVT = Node->getValueType(ResNo: 0);
893 if (!TLI.isLoadLegal(ValVT: DestVT, MemVT: SrcVT, Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(),
894 ExtType: ISD::EXTLOAD, Atomic: false)) {
895 // If the source type is not legal, see if there is a legal extload to
896 // an intermediate type that we can then extend further.
897 EVT LoadVT =
898 TLI.getRegisterType(Context&: *DAG.getContext(), VT: SrcVT.getSimpleVT());
899 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
900 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
901 TLI.isLoadLegal(ValVT: LoadVT, MemVT: SrcVT, Alignment: LD->getAlign(),
902 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false))) {
903 // If we are loading a legal type, this is a non-extload followed by a
904 // full extend.
905 ISD::LoadExtType MidExtType =
906 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
907
908 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
909 MemVT: SrcVT, MMO: LD->getMemOperand());
910 unsigned ExtendOp =
911 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
912 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
913 Chain = Load.getValue(R: 1);
914 break;
915 }
916
917 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
918 // normal undefined upper bits behavior to allow using an in-reg extend
919 // with the illegal FP type, so load as an integer and do the
920 // from-integer conversion.
921 EVT SVT = SrcVT.getScalarType();
922 if (SVT == MVT::f16 || SVT == MVT::bf16) {
923 EVT ISrcVT = SrcVT.changeTypeToInteger();
924 EVT IDestVT = DestVT.changeTypeToInteger();
925 EVT ILoadVT =
926 TLI.getRegisterType(Context&: *DAG.getContext(), VT: IDestVT.getSimpleVT());
927
928 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
929 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
930 Value =
931 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
932 DL: dl, VT: DestVT, Operand: Result);
933 Chain = Result.getValue(R: 1);
934 break;
935 }
936 }
937
938 assert(!SrcVT.isVector() &&
939 "Vector Loads are handled in LegalizeVectorOps");
940
941 // FIXME: This does not work for vectors on most targets. Sign-
942 // and zero-extend operations are currently folded into extending
943 // loads, whether they are legal or not, and then we end up here
944 // without any support for legalizing them.
945 assert(ExtType != ISD::EXTLOAD &&
946 "EXTLOAD should always be supported!");
947 // Turn the unsupported load into an EXTLOAD followed by an
948 // explicit zero/sign extend inreg.
949 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
950 VT: Node->getValueType(ResNo: 0),
951 Chain, Ptr, MemVT: SrcVT,
952 MMO: LD->getMemOperand());
953 SDValue ValRes;
954 if (ExtType == ISD::SEXTLOAD)
955 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
956 VT: Result.getValueType(),
957 N1: Result, N2: DAG.getValueType(SrcVT));
958 else
959 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
960 Value = ValRes;
961 Chain = Result.getValue(R: 1);
962 break;
963 }
964 }
965 }
966
967 // Since loads produce two values, make sure to remember that we legalized
968 // both of them.
969 if (Chain.getNode() != Node) {
970 assert(Value.getNode() != Node && "Load must be completely replaced");
971 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
972 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
973 if (UpdatedNodes) {
974 UpdatedNodes->insert(X: Value.getNode());
975 UpdatedNodes->insert(X: Chain.getNode());
976 }
977 ReplacedNode(N: Node);
978 }
979}
980
981/// Return a legal replacement for the given operation, with all legal operands.
982void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
983 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
984
985 // Allow illegal target nodes and illegal registers.
986 if (Node->getOpcode() == ISD::TargetConstant ||
987 Node->getOpcode() == ISD::Register)
988 return;
989
990#ifndef NDEBUG
991 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
992 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
993 TargetLowering::TypeLegal &&
994 "Unexpected illegal type!");
995
996 for (const SDValue &Op : Node->op_values())
997 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
998 TargetLowering::TypeLegal ||
999 Op.getOpcode() == ISD::TargetConstant ||
1000 Op.getOpcode() == ISD::Register) &&
1001 "Unexpected illegal type!");
1002#endif
1003
1004 // Figure out the correct action; the way to query this varies by opcode
1005 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1006 bool SimpleFinishLegalizing = true;
1007 switch (Node->getOpcode()) {
1008 case ISD::INTRINSIC_W_CHAIN:
1009 case ISD::INTRINSIC_WO_CHAIN:
1010 case ISD::INTRINSIC_VOID:
1011 case ISD::STACKSAVE:
1012 case ISD::STACKADDRESS:
1013 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1014 break;
1015 case ISD::GET_DYNAMIC_AREA_OFFSET:
1016 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1017 VT: Node->getValueType(ResNo: 0));
1018 break;
1019 case ISD::VAARG:
1020 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1021 VT: Node->getValueType(ResNo: 0));
1022 if (Action != TargetLowering::Promote)
1023 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1024 break;
1025 case ISD::SET_FPENV:
1026 case ISD::SET_FPMODE:
1027 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1028 VT: Node->getOperand(Num: 1).getValueType());
1029 break;
1030 case ISD::FP_TO_FP16:
1031 case ISD::FP_TO_BF16:
1032 case ISD::SINT_TO_FP:
1033 case ISD::UINT_TO_FP:
1034 case ISD::EXTRACT_VECTOR_ELT:
1035 case ISD::LROUND:
1036 case ISD::LLROUND:
1037 case ISD::LRINT:
1038 case ISD::LLRINT:
1039 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1040 VT: Node->getOperand(Num: 0).getValueType());
1041 break;
1042 case ISD::STRICT_FP_TO_FP16:
1043 case ISD::STRICT_FP_TO_BF16:
1044 case ISD::STRICT_SINT_TO_FP:
1045 case ISD::STRICT_UINT_TO_FP:
1046 case ISD::STRICT_LRINT:
1047 case ISD::STRICT_LLRINT:
1048 case ISD::STRICT_LROUND:
1049 case ISD::STRICT_LLROUND:
1050 // These pseudo-ops are the same as the other STRICT_ ops except
1051 // they are registered with setOperationAction() using the input type
1052 // instead of the output type.
1053 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1054 VT: Node->getOperand(Num: 1).getValueType());
1055 break;
1056 case ISD::SIGN_EXTEND_INREG: {
1057 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1058 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1059 break;
1060 }
1061 case ISD::ATOMIC_STORE:
1062 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1063 VT: Node->getOperand(Num: 1).getValueType());
1064 break;
1065 case ISD::SELECT_CC:
1066 case ISD::STRICT_FSETCC:
1067 case ISD::STRICT_FSETCCS:
1068 case ISD::SETCC:
1069 case ISD::SETCCCARRY:
1070 case ISD::BR_CC: {
1071 unsigned Opc = Node->getOpcode();
1072 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1073 : Opc == ISD::STRICT_FSETCC ? 3
1074 : Opc == ISD::STRICT_FSETCCS ? 3
1075 : Opc == ISD::SETCCCARRY ? 3
1076 : Opc == ISD::SETCC ? 2
1077 : 1;
1078 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1079 : Opc == ISD::STRICT_FSETCC ? 1
1080 : Opc == ISD::STRICT_FSETCCS ? 1
1081 : 0;
1082 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1083 ISD::CondCode CCCode =
1084 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1085 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1086 if (Action == TargetLowering::Legal) {
1087 if (Node->getOpcode() == ISD::SELECT_CC)
1088 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1089 VT: Node->getValueType(ResNo: 0));
1090 else
1091 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1092 }
1093 break;
1094 }
1095 case ISD::LOAD:
1096 case ISD::STORE:
1097 // FIXME: Model these properly. LOAD and STORE are complicated, and
1098 // STORE expects the unlegalized operand in some cases.
1099 SimpleFinishLegalizing = false;
1100 break;
1101 case ISD::CALLSEQ_START:
1102 case ISD::CALLSEQ_END:
1103 // FIXME: This shouldn't be necessary. These nodes have special properties
1104 // dealing with the recursive nature of legalization. Removing this
1105 // special case should be done as part of making LegalizeDAG non-recursive.
1106 SimpleFinishLegalizing = false;
1107 break;
1108 case ISD::EXTRACT_ELEMENT:
1109 case ISD::GET_ROUNDING:
1110 case ISD::MERGE_VALUES:
1111 case ISD::EH_RETURN:
1112 case ISD::FRAME_TO_ARGS_OFFSET:
1113 case ISD::EH_DWARF_CFA:
1114 case ISD::EH_SJLJ_SETJMP:
1115 case ISD::EH_SJLJ_LONGJMP:
1116 case ISD::EH_SJLJ_SETUP_DISPATCH:
1117 // These operations lie about being legal: when they claim to be legal,
1118 // they should actually be expanded.
1119 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1120 if (Action == TargetLowering::Legal)
1121 Action = TargetLowering::Expand;
1122 break;
1123 case ISD::INIT_TRAMPOLINE:
1124 case ISD::ADJUST_TRAMPOLINE:
1125 case ISD::FRAMEADDR:
1126 case ISD::RETURNADDR:
1127 case ISD::ADDROFRETURNADDR:
1128 case ISD::SPONENTRY:
1129 // These operations lie about being legal: when they claim to be legal,
1130 // they should actually be custom-lowered.
1131 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1132 if (Action == TargetLowering::Legal)
1133 Action = TargetLowering::Custom;
1134 break;
1135 case ISD::CLEAR_CACHE:
1136 // This operation is typically going to be LibCall unless the target wants
1137 // something differrent.
1138 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1139 break;
1140 case ISD::READCYCLECOUNTER:
1141 case ISD::READSTEADYCOUNTER:
1142 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1143 // legalization might have expanded that to several smaller types.
1144 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1145 break;
1146 case ISD::READ_REGISTER:
1147 case ISD::WRITE_REGISTER:
1148 // Named register is legal in the DAG, but blocked by register name
1149 // selection if not implemented by target (to chose the correct register)
1150 // They'll be converted to Copy(To/From)Reg.
1151 Action = TargetLowering::Legal;
1152 break;
1153 case ISD::UBSANTRAP:
1154 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1155 if (Action == TargetLowering::Expand) {
1156 // replace ISD::UBSANTRAP with ISD::TRAP
1157 SDValue NewVal;
1158 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1159 N: Node->getOperand(Num: 0));
1160 ReplaceNode(Old: Node, New: NewVal.getNode());
1161 LegalizeOp(Node: NewVal.getNode());
1162 return;
1163 }
1164 break;
1165 case ISD::DEBUGTRAP:
1166 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1167 if (Action == TargetLowering::Expand) {
1168 // replace ISD::DEBUGTRAP with ISD::TRAP
1169 SDValue NewVal;
1170 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1171 N: Node->getOperand(Num: 0));
1172 ReplaceNode(Old: Node, New: NewVal.getNode());
1173 LegalizeOp(Node: NewVal.getNode());
1174 return;
1175 }
1176 break;
1177 case ISD::SADDSAT:
1178 case ISD::UADDSAT:
1179 case ISD::SSUBSAT:
1180 case ISD::USUBSAT:
1181 case ISD::SSHLSAT:
1182 case ISD::USHLSAT:
1183 case ISD::SCMP:
1184 case ISD::UCMP:
1185 case ISD::FP_TO_SINT_SAT:
1186 case ISD::FP_TO_UINT_SAT:
1187 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1188 break;
1189 case ISD::SMULFIX:
1190 case ISD::SMULFIXSAT:
1191 case ISD::UMULFIX:
1192 case ISD::UMULFIXSAT:
1193 case ISD::SDIVFIX:
1194 case ISD::SDIVFIXSAT:
1195 case ISD::UDIVFIX:
1196 case ISD::UDIVFIXSAT: {
1197 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1198 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1199 VT: Node->getValueType(ResNo: 0), Scale);
1200 break;
1201 }
1202 case ISD::MSCATTER:
1203 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1204 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1205 break;
1206 case ISD::MSTORE:
1207 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1208 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1209 break;
1210 case ISD::VP_SCATTER:
1211 Action = TLI.getOperationAction(
1212 Op: Node->getOpcode(),
1213 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1214 break;
1215 case ISD::VP_STORE:
1216 Action = TLI.getOperationAction(
1217 Op: Node->getOpcode(),
1218 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1219 break;
1220 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1221 Action = TLI.getOperationAction(
1222 Op: Node->getOpcode(),
1223 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1224 break;
1225 case ISD::VECREDUCE_FADD:
1226 case ISD::VECREDUCE_FMUL:
1227 case ISD::VECREDUCE_ADD:
1228 case ISD::VECREDUCE_MUL:
1229 case ISD::VECREDUCE_AND:
1230 case ISD::VECREDUCE_OR:
1231 case ISD::VECREDUCE_XOR:
1232 case ISD::VECREDUCE_SMAX:
1233 case ISD::VECREDUCE_SMIN:
1234 case ISD::VECREDUCE_UMAX:
1235 case ISD::VECREDUCE_UMIN:
1236 case ISD::VECREDUCE_FMAX:
1237 case ISD::VECREDUCE_FMIN:
1238 case ISD::VECREDUCE_FMAXIMUM:
1239 case ISD::VECREDUCE_FMINIMUM:
1240 case ISD::VECREDUCE_FMAXIMUMNUM:
1241 case ISD::VECREDUCE_FMINIMUMNUM:
1242 case ISD::IS_FPCLASS:
1243 Action = TLI.getOperationAction(
1244 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1245 break;
1246 case ISD::VECREDUCE_SEQ_FADD:
1247 case ISD::VECREDUCE_SEQ_FMUL:
1248 case ISD::VP_REDUCE_FADD:
1249 case ISD::VP_REDUCE_FMUL:
1250 case ISD::VP_REDUCE_ADD:
1251 case ISD::VP_REDUCE_MUL:
1252 case ISD::VP_REDUCE_AND:
1253 case ISD::VP_REDUCE_OR:
1254 case ISD::VP_REDUCE_XOR:
1255 case ISD::VP_REDUCE_SMAX:
1256 case ISD::VP_REDUCE_SMIN:
1257 case ISD::VP_REDUCE_UMAX:
1258 case ISD::VP_REDUCE_UMIN:
1259 case ISD::VP_REDUCE_FMAX:
1260 case ISD::VP_REDUCE_FMIN:
1261 case ISD::VP_REDUCE_FMAXIMUM:
1262 case ISD::VP_REDUCE_FMINIMUM:
1263 case ISD::VP_REDUCE_SEQ_FADD:
1264 case ISD::VP_REDUCE_SEQ_FMUL:
1265 Action = TLI.getOperationAction(
1266 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1267 break;
1268 case ISD::CTTZ_ELTS:
1269 case ISD::CTTZ_ELTS_ZERO_POISON:
1270 case ISD::VP_CTTZ_ELTS:
1271 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
1272 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1273 VT: Node->getOperand(Num: 0).getValueType());
1274 break;
1275 case ISD::VECTOR_INTERLEAVE:
1276 case ISD::VECTOR_DEINTERLEAVE:
1277 Action = TLI.getVectorInterleaveAction(
1278 Opc: Node->getOpcode(), Factor: Node->getNumOperands(), VT: Node->getValueType(ResNo: 0));
1279 break;
1280 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1281 Action = TLI.getOperationAction(
1282 Op: Node->getOpcode(),
1283 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1284 break;
1285 default:
1286 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1287 Action = TLI.getCustomOperationAction(Op&: *Node);
1288 } else {
1289 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1290 }
1291 break;
1292 }
1293
1294 if (SimpleFinishLegalizing) {
1295 SDNode *NewNode = Node;
1296 switch (Node->getOpcode()) {
1297 default: break;
1298 case ISD::SHL:
1299 case ISD::SRL:
1300 case ISD::SRA:
1301 case ISD::ROTL:
1302 case ISD::ROTR:
1303 case ISD::SSHLSAT:
1304 case ISD::USHLSAT: {
1305 // Legalizing shifts/rotates requires adjusting the shift amount
1306 // to the appropriate width.
1307 SDValue Op0 = Node->getOperand(Num: 0);
1308 SDValue Op1 = Node->getOperand(Num: 1);
1309 if (!Op1.getValueType().isVector()) {
1310 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1311 // The getShiftAmountOperand() may create a new operand node or
1312 // return the existing one. If new operand is created we need
1313 // to update the parent node.
1314 // Do not try to legalize SAO here! It will be automatically legalized
1315 // in the next round.
1316 if (SAO != Op1)
1317 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1318 }
1319 break;
1320 }
1321 case ISD::FSHL:
1322 case ISD::FSHR:
1323 case ISD::SRL_PARTS:
1324 case ISD::SRA_PARTS:
1325 case ISD::SHL_PARTS: {
1326 // Legalizing shifts/rotates requires adjusting the shift amount
1327 // to the appropriate width.
1328 SDValue Op0 = Node->getOperand(Num: 0);
1329 SDValue Op1 = Node->getOperand(Num: 1);
1330 SDValue Op2 = Node->getOperand(Num: 2);
1331 if (!Op2.getValueType().isVector()) {
1332 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1333 // The getShiftAmountOperand() may create a new operand node or
1334 // return the existing one. If new operand is created we need
1335 // to update the parent node.
1336 if (SAO != Op2)
1337 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1338 }
1339 break;
1340 }
1341 }
1342
1343 if (NewNode != Node) {
1344 ReplaceNode(Old: Node, New: NewNode);
1345 Node = NewNode;
1346 }
1347 switch (Action) {
1348 case TargetLowering::Legal:
1349 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1350 return;
1351 case TargetLowering::Custom:
1352 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1353 // FIXME: The handling for custom lowering with multiple results is
1354 // a complete mess.
1355 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1356 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1357 return;
1358
1359 if (Node->getNumValues() == 1) {
1360 // Verify the new types match the original. Glue is waived because
1361 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1362 assert((Res.getValueType() == Node->getValueType(0) ||
1363 Node->getValueType(0) == MVT::Glue) &&
1364 "Type mismatch for custom legalized operation");
1365 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1366 // We can just directly replace this node with the lowered value.
1367 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1368 return;
1369 }
1370
1371 SmallVector<SDValue, 8> ResultVals;
1372 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1373 // Verify the new types match the original. Glue is waived because
1374 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1375 assert((Res->getValueType(i) == Node->getValueType(i) ||
1376 Node->getValueType(i) == MVT::Glue) &&
1377 "Type mismatch for custom legalized operation");
1378 ResultVals.push_back(Elt: Res.getValue(R: i));
1379 }
1380 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1381 ReplaceNode(Old: Node, New: ResultVals.data());
1382 return;
1383 }
1384 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1385 [[fallthrough]];
1386 case TargetLowering::Expand:
1387 if (ExpandNode(Node))
1388 return;
1389 [[fallthrough]];
1390 case TargetLowering::LibCall:
1391 ConvertNodeToLibcall(Node);
1392 return;
1393 case TargetLowering::Promote:
1394 PromoteNode(Node);
1395 return;
1396 }
1397 }
1398
1399 switch (Node->getOpcode()) {
1400 default:
1401#ifndef NDEBUG
1402 dbgs() << "NODE: ";
1403 Node->dump( &DAG);
1404 dbgs() << "\n";
1405#endif
1406 llvm_unreachable("Do not know how to legalize this operator!");
1407
1408 case ISD::CALLSEQ_START:
1409 case ISD::CALLSEQ_END:
1410 break;
1411 case ISD::LOAD:
1412 return LegalizeLoadOps(Node);
1413 case ISD::STORE:
1414 return LegalizeStoreOps(Node);
1415 }
1416}
1417
1418SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1419 SDValue Vec = Op.getOperand(i: 0);
1420 SDValue Idx = Op.getOperand(i: 1);
1421 SDLoc dl(Op);
1422
1423 // Before we generate a new store to a temporary stack slot, see if there is
1424 // already one that we can use. There often is because when we scalarize
1425 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1426 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1427 // the vector. If all are expanded here, we don't want one store per vector
1428 // element.
1429
1430 // Caches for hasPredecessorHelper
1431 SmallPtrSet<const SDNode *, 32> Visited;
1432 SmallVector<const SDNode *, 16> Worklist;
1433 Visited.insert(Ptr: Op.getNode());
1434 Worklist.push_back(Elt: Idx.getNode());
1435 SDValue StackPtr, Ch;
1436 for (SDNode *User : Vec.getNode()->users()) {
1437 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1438 if (ST->isIndexed() || ST->isTruncatingStore() ||
1439 ST->getValue() != Vec)
1440 continue;
1441
1442 // Make sure that nothing else could have stored into the destination of
1443 // this store.
1444 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1445 continue;
1446
1447 // If the index is dependent on the store we will introduce a cycle when
1448 // creating the load (the load uses the index, and by replacing the chain
1449 // we will make the index dependent on the load). Also, the store might be
1450 // dependent on the extractelement and introduce a cycle when creating
1451 // the load.
1452 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1453 ST->hasPredecessor(N: Op.getNode()))
1454 continue;
1455
1456 StackPtr = ST->getBasePtr();
1457 Ch = SDValue(ST, 0);
1458 break;
1459 }
1460 }
1461
1462 EVT VecVT = Vec.getValueType();
1463
1464 if (!Ch.getNode()) {
1465 // Store the value to a temporary stack slot, then LOAD the returned part.
1466 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1467 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1468 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1469 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1470 }
1471
1472 SDValue NewLoad;
1473 Align ElementAlignment =
1474 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1475 b: DAG.getDataLayout().getPrefTypeAlign(
1476 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1477
1478 if (Op.getValueType().isVector()) {
1479 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1480 SubVecVT: Op.getValueType(), Index: Idx);
1481 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1482 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1483 } else {
1484 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1485 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1486 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1487 Alignment: ElementAlignment);
1488 }
1489
1490 // Replace the chain going out of the store, by the one out of the load.
1491 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1492
1493 // We introduced a cycle though, so update the loads operands, making sure
1494 // to use the original store's chain as an incoming chain.
1495 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1496 NewLoadOperands[0] = Ch;
1497 NewLoad =
1498 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1499 return NewLoad;
1500}
1501
1502SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1503 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1504
1505 SDValue Vec = Op.getOperand(i: 0);
1506 SDValue Part = Op.getOperand(i: 1);
1507 SDValue Idx = Op.getOperand(i: 2);
1508 SDLoc dl(Op);
1509
1510 // Store the value to a temporary stack slot, then LOAD the returned part.
1511 EVT VecVT = Vec.getValueType();
1512 EVT PartVT = Part.getValueType();
1513 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1514 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1515 MachinePointerInfo PtrInfo =
1516 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1517
1518 // First store the whole vector.
1519 Align BaseVecAlignment =
1520 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1521 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1522 Alignment: BaseVecAlignment);
1523
1524 // Freeze the index so we don't poison the clamping code we're about to emit.
1525 Idx = DAG.getFreeze(V: Idx);
1526
1527 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1528 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1529
1530 // Then store the inserted part.
1531 if (PartVT.isVector()) {
1532 SDValue SubStackPtr =
1533 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1534
1535 // Store the subvector.
1536 Ch = DAG.getStore(
1537 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1538 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1539 Alignment: PartAlignment);
1540 } else {
1541 SDValue SubStackPtr =
1542 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1543
1544 // Store the scalar value.
1545 Ch = DAG.getTruncStore(
1546 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1547 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1548 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1549 }
1550
1551 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1552 "ElementAlignment does not match!");
1553
1554 // Finally, load the updated vector.
1555 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1556 Alignment: BaseVecAlignment);
1557}
1558
1559SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1560 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1561 SDLoc DL(Node);
1562 SmallVector<SDValue, 16> Ops;
1563 unsigned NumOperands = Node->getNumOperands();
1564 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1565 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1566 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1567 EVT ElementValueType = TLI.getTypeToTransformTo(
1568 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1569 for (unsigned I = 0; I < NumOperands; ++I) {
1570 SDValue SubOp = Node->getOperand(Num: I);
1571 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1572 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1573 N1: SubOp,
1574 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1575 }
1576 }
1577 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1578}
1579
1580SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1581 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1582 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1583 "Unexpected opcode!");
1584
1585 // We can't handle this case efficiently. Allocate a sufficiently
1586 // aligned object on the stack, store each operand into it, then load
1587 // the result as a vector.
1588 // Create the stack frame object.
1589 EVT VT = Node->getValueType(ResNo: 0);
1590 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1591 : Node->getOperand(Num: 0).getValueType();
1592 SDLoc dl(Node);
1593 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1594 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1595 MachinePointerInfo PtrInfo =
1596 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1597
1598 // Emit a store of each element to the stack slot.
1599 SmallVector<SDValue, 8> Stores;
1600 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1601 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1602
1603 // If the destination vector element type of a BUILD_VECTOR is narrower than
1604 // the source element type, only store the bits necessary.
1605 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1606 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1607
1608 // Store (in the right endianness) the elements to memory.
1609 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1610 // Ignore undef elements.
1611 if (Node->getOperand(Num: i).isUndef()) continue;
1612
1613 unsigned Offset = TypeByteSize*i;
1614
1615 SDValue Idx =
1616 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1617
1618 if (Truncate)
1619 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1620 Val: Node->getOperand(Num: i), Ptr: Idx,
1621 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1622 else
1623 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1624 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1625 }
1626
1627 SDValue StoreChain;
1628 if (!Stores.empty()) // Not all undef elements?
1629 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1630 else
1631 StoreChain = DAG.getEntryNode();
1632
1633 // Result is a load from the stack slot.
1634 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1635}
1636
1637/// Bitcast a floating-point value to an integer value. Only bitcast the part
1638/// containing the sign bit if the target has no integer value capable of
1639/// holding all bits of the floating-point value.
1640void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1641 const SDLoc &DL,
1642 SDValue Value) const {
1643 EVT FloatVT = Value.getValueType();
1644 unsigned NumBits = FloatVT.getScalarSizeInBits();
1645 State.FloatVT = FloatVT;
1646 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1647 // Convert to an integer of the same size.
1648 if (TLI.isTypeLegal(VT: IVT)) {
1649 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1650 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1651 State.SignBit = NumBits - 1;
1652 return;
1653 }
1654
1655 auto &DataLayout = DAG.getDataLayout();
1656 // Store the float to memory, then load the sign part out as an integer.
1657 MVT LoadTy = TLI.getRegisterType(Context&: *DAG.getContext(), VT: MVT::i8);
1658 // First create a temporary that is aligned for both the load and store.
1659 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1660 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1661 // Then store the float to it.
1662 State.FloatPtr = StackPtr;
1663 MachineFunction &MF = DAG.getMachineFunction();
1664 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1665 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1666 PtrInfo: State.FloatPointerInfo);
1667
1668 SDValue IntPtr;
1669 if (DataLayout.isBigEndian()) {
1670 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1671 // Load out a legal integer with the same sign bit as the float.
1672 IntPtr = StackPtr;
1673 State.IntPointerInfo = State.FloatPointerInfo;
1674 } else {
1675 // Advance the pointer so that the loaded byte will contain the sign bit.
1676 unsigned ByteOffset = (NumBits / 8) - 1;
1677 IntPtr =
1678 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1679 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1680 Offset: ByteOffset);
1681 }
1682
1683 State.IntPtr = IntPtr;
1684 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1685 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1686 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1687 State.SignBit = 7;
1688}
1689
1690/// Replace the integer value produced by getSignAsIntValue() with a new value
1691/// and cast the result back to a floating-point type.
1692SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1693 const SDLoc &DL,
1694 SDValue NewIntValue) const {
1695 if (!State.Chain)
1696 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1697
1698 // Override the part containing the sign bit in the value stored on the stack.
1699 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1700 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1701 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1702 PtrInfo: State.FloatPointerInfo);
1703}
1704
1705SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1706 SDLoc DL(Node);
1707 SDValue Mag = Node->getOperand(Num: 0);
1708 SDValue Sign = Node->getOperand(Num: 1);
1709
1710 if (Sign.getValueType().isVector())
1711 return DAG.UnrollVectorOp(N: Node);
1712
1713 // Get sign bit into an integer value.
1714 FloatSignAsInt SignAsInt;
1715 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1716
1717 EVT IntVT = SignAsInt.IntValue.getValueType();
1718 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1719 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1720 N2: SignMask);
1721
1722 // If FABS is legal transform
1723 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1724 EVT FloatVT = Mag.getValueType();
1725 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1726 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1727 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1728 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1729 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1730 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1731 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1732 }
1733
1734 // Transform Mag value to integer, and clear the sign bit.
1735 FloatSignAsInt MagAsInt;
1736 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1737 EVT MagVT = MagAsInt.IntValue.getValueType();
1738 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1739 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1740 N2: ClearSignMask);
1741
1742 // Get the signbit at the right position for MagAsInt.
1743 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1744 EVT ShiftVT = IntVT;
1745 if (SignBit.getScalarValueSizeInBits() <
1746 ClearedSign.getScalarValueSizeInBits()) {
1747 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1748 ShiftVT = MagVT;
1749 }
1750 if (ShiftAmount > 0) {
1751 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1752 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1753 } else if (ShiftAmount < 0) {
1754 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1755 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1756 }
1757 if (SignBit.getScalarValueSizeInBits() >
1758 ClearedSign.getScalarValueSizeInBits()) {
1759 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1760 }
1761
1762 // Store the part with the modified sign and convert back to float.
1763 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1764 Flags: SDNodeFlags::Disjoint);
1765
1766 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1767}
1768
1769SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1770 // Get the sign bit as an integer.
1771 SDLoc DL(Node);
1772 if (Node->getValueType(ResNo: 0).isVector())
1773 return DAG.UnrollVectorOp(N: Node);
1774
1775 FloatSignAsInt SignAsInt;
1776 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1777 EVT IntVT = SignAsInt.IntValue.getValueType();
1778
1779 // Flip the sign.
1780 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1781 SDValue SignFlip =
1782 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1783
1784 // Convert back to float.
1785 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1786}
1787
1788SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1789 SDLoc DL(Node);
1790 SDValue Value = Node->getOperand(Num: 0);
1791
1792 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1793 EVT FloatVT = Value.getValueType();
1794 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1795 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1796 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1797 }
1798
1799 if (FloatVT.isVector())
1800 return DAG.UnrollVectorOp(N: Node);
1801
1802 // Transform value to integer, clear the sign bit and transform back.
1803 FloatSignAsInt ValueAsInt;
1804 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1805 EVT IntVT = ValueAsInt.IntValue.getValueType();
1806 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1807 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1808 N2: ClearSignMask);
1809 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1810}
1811
1812void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1813 SmallVectorImpl<SDValue> &Results) {
1814 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1815 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1816 " not tell us which reg is the stack pointer!");
1817 SDLoc dl(Node);
1818 EVT VT = Node->getValueType(ResNo: 0);
1819 SDValue Tmp1 = SDValue(Node, 0);
1820 SDValue Tmp2 = SDValue(Node, 1);
1821 SDValue Tmp3 = Node->getOperand(Num: 2);
1822 SDValue Chain = Tmp1.getOperand(i: 0);
1823
1824 // Chain the dynamic stack allocation so that it doesn't modify the stack
1825 // pointer when other instructions are using the stack.
1826 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1827
1828 SDValue Size = Tmp2.getOperand(i: 1);
1829 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1830 Chain = SP.getValue(R: 1);
1831 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1832 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1833 unsigned Opc =
1834 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1835 ISD::ADD : ISD::SUB;
1836
1837 Align StackAlign = TFL->getStackAlign();
1838 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1839 if (Alignment > StackAlign)
1840 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1841 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1842 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1843
1844 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1845
1846 Results.push_back(Elt: Tmp1);
1847 Results.push_back(Elt: Tmp2);
1848}
1849
1850SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1851 EVT DestVT, const SDLoc &dl) {
1852 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1853}
1854
1855SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1856 EVT DestVT, const SDLoc &dl,
1857 SDValue Chain) {
1858 EVT SrcVT = SrcOp.getValueType();
1859 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1860 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1861 // Don't convert with stack if the load/store is expensive.
1862 if ((SrcVT.bitsGT(VT: SlotVT) && !TLI.isTruncStoreLegalOrCustom(
1863 ValVT: SrcVT, MemVT: SlotVT, Alignment: DestAlign,
1864 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace())) ||
1865 (SlotVT.bitsLT(VT: DestVT) &&
1866 !TLI.isLoadLegalOrCustom(ValVT: DestVT, MemVT: SlotVT, Alignment: DestAlign,
1867 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace(),
1868 ExtType: ISD::EXTLOAD, Atomic: false)))
1869 return SDValue();
1870
1871 return DAG.emitStackConvert(SrcOp, SlotVT, DestVT, DL: dl, Chain);
1872}
1873
1874SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1875 SDLoc dl(Node);
1876 // Create a vector sized/aligned stack slot, store the value to element #0,
1877 // then load the whole vector back out.
1878 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1879
1880 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1881 int SPFI = StackPtrFI->getIndex();
1882
1883 SDValue Ch = DAG.getTruncStore(
1884 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1885 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1886 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1887 return DAG.getLoad(
1888 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1889 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1890}
1891
1892static bool
1893ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1894 const TargetLowering &TLI, SDValue &Res) {
1895 unsigned NumElems = Node->getNumOperands();
1896 SDLoc dl(Node);
1897 EVT VT = Node->getValueType(ResNo: 0);
1898
1899 // Try to group the scalars into pairs, shuffle the pairs together, then
1900 // shuffle the pairs of pairs together, etc. until the vector has
1901 // been built. This will work only if all of the necessary shuffle masks
1902 // are legal.
1903
1904 // We do this in two phases; first to check the legality of the shuffles,
1905 // and next, assuming that all shuffles are legal, to create the new nodes.
1906 for (int Phase = 0; Phase < 2; ++Phase) {
1907 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1908 NewIntermedVals;
1909 for (unsigned i = 0; i < NumElems; ++i) {
1910 SDValue V = Node->getOperand(Num: i);
1911 if (V.isUndef())
1912 continue;
1913
1914 SDValue Vec;
1915 if (Phase)
1916 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1917 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1918 }
1919
1920 while (IntermedVals.size() > 2) {
1921 NewIntermedVals.clear();
1922 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1923 // This vector and the next vector are shuffled together (simply to
1924 // append the one to the other).
1925 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1926
1927 SmallVector<int, 16> FinalIndices;
1928 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1929 IntermedVals[i+1].second.size());
1930
1931 int k = 0;
1932 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1933 ++j, ++k) {
1934 ShuffleVec[k] = j;
1935 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1936 }
1937 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1938 ++j, ++k) {
1939 ShuffleVec[k] = NumElems + j;
1940 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1941 }
1942
1943 SDValue Shuffle;
1944 if (Phase)
1945 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1946 N2: IntermedVals[i+1].first,
1947 Mask: ShuffleVec);
1948 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1949 return false;
1950 NewIntermedVals.push_back(
1951 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1952 }
1953
1954 // If we had an odd number of defined values, then append the last
1955 // element to the array of new vectors.
1956 if ((IntermedVals.size() & 1) != 0)
1957 NewIntermedVals.push_back(Elt: IntermedVals.back());
1958
1959 IntermedVals.swap(RHS&: NewIntermedVals);
1960 }
1961
1962 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1963 "Invalid number of intermediate vectors");
1964 SDValue Vec1 = IntermedVals[0].first;
1965 SDValue Vec2;
1966 if (IntermedVals.size() > 1)
1967 Vec2 = IntermedVals[1].first;
1968 else if (Phase)
1969 Vec2 = DAG.getPOISON(VT);
1970
1971 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1972 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1973 ShuffleVec[IntermedVals[0].second[i]] = i;
1974 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1975 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1976
1977 if (Phase)
1978 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1979 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1980 return false;
1981 }
1982
1983 return true;
1984}
1985
1986/// Expand a BUILD_VECTOR node on targets that don't
1987/// support the operation, but do support the resultant vector type.
1988SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1989 unsigned NumElems = Node->getNumOperands();
1990 SDValue Value1, Value2;
1991 SDLoc dl(Node);
1992 EVT VT = Node->getValueType(ResNo: 0);
1993 EVT OpVT = Node->getOperand(Num: 0).getValueType();
1994 EVT EltVT = VT.getVectorElementType();
1995
1996 // If the only non-undef value is the low element, turn this into a
1997 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1998 bool isOnlyLowElement = true;
1999 bool MoreThanTwoValues = false;
2000 bool isConstant = true;
2001 for (unsigned i = 0; i < NumElems; ++i) {
2002 SDValue V = Node->getOperand(Num: i);
2003 if (V.isUndef())
2004 continue;
2005 if (i > 0)
2006 isOnlyLowElement = false;
2007 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2008 isConstant = false;
2009
2010 if (!Value1.getNode()) {
2011 Value1 = V;
2012 } else if (!Value2.getNode()) {
2013 if (V != Value1)
2014 Value2 = V;
2015 } else if (V != Value1 && V != Value2) {
2016 MoreThanTwoValues = true;
2017 }
2018 }
2019
2020 if (!Value1.getNode())
2021 return DAG.getUNDEF(VT);
2022
2023 if (isOnlyLowElement)
2024 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2025
2026 // If all elements are constants, create a load from the constant pool.
2027 if (isConstant) {
2028 SmallVector<Constant*, 16> CV;
2029 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2030 if (ConstantFPSDNode *V =
2031 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2032 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2033 } else if (ConstantSDNode *V =
2034 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2035 if (OpVT==EltVT)
2036 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2037 else {
2038 // If OpVT and EltVT don't match, EltVT is not legal and the
2039 // element values have been promoted/truncated earlier. Undo this;
2040 // we don't want a v16i8 to become a v16i32 for example.
2041 const ConstantInt *CI = V->getConstantIntValue();
2042 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2043 V: CI->getZExtValue(), /*IsSigned=*/false,
2044 /*ImplicitTrunc=*/true));
2045 }
2046 } else {
2047 assert(Node->getOperand(i).isUndef());
2048 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2049 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2050 }
2051 }
2052 Constant *CP = ConstantVector::get(V: CV);
2053 SDValue CPIdx =
2054 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2055 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2056 return DAG.getLoad(
2057 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2058 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2059 Alignment);
2060 }
2061
2062 SmallSet<SDValue, 16> DefinedValues;
2063 for (unsigned i = 0; i < NumElems; ++i) {
2064 if (Node->getOperand(Num: i).isUndef())
2065 continue;
2066 DefinedValues.insert(V: Node->getOperand(Num: i));
2067 }
2068
2069 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2070 if (!MoreThanTwoValues) {
2071 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2072 for (unsigned i = 0; i < NumElems; ++i) {
2073 SDValue V = Node->getOperand(Num: i);
2074 if (V.isUndef())
2075 continue;
2076 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2077 }
2078 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2079 // Get the splatted value into the low element of a vector register.
2080 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2081 SDValue Vec2;
2082 if (Value2.getNode())
2083 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2084 else
2085 Vec2 = DAG.getPOISON(VT);
2086
2087 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2088 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2089 }
2090 } else {
2091 SDValue Res;
2092 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2093 return Res;
2094 }
2095 }
2096
2097 // Otherwise, we can't handle this case efficiently.
2098 return ExpandVectorBuildThroughStack(Node);
2099}
2100
2101SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2102 SDLoc DL(Node);
2103 EVT VT = Node->getValueType(ResNo: 0);
2104 SDValue SplatVal = Node->getOperand(Num: 0);
2105
2106 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2107}
2108
2109// Expand a node into a call to a libcall, returning the value as the first
2110// result and the chain as the second. If the result value does not fit into a
2111// register, return the lo part and set the hi part to the by-reg argument in
2112// the first. If it does fit into a single register, return the result and
2113// leave the Hi part unset.
2114std::pair<SDValue, SDValue>
2115SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2116 TargetLowering::ArgListTy &&Args,
2117 bool IsSigned, EVT RetVT) {
2118 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2119 SDValue Callee;
2120 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2121 if (LCImpl != RTLIB::Unsupported)
2122 Callee = DAG.getExternalSymbol(LCImpl, VT: CodePtrTy);
2123 else {
2124 Callee = DAG.getPOISON(VT: CodePtrTy);
2125 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2126 Node->getOperationName(G: &DAG));
2127 }
2128
2129 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2130
2131 // By default, the input chain to this libcall is the entry node of the
2132 // function. If the libcall is going to be emitted as a tail call then
2133 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2134 // node which is being folded has a non-entry input chain.
2135 SDValue InChain = DAG.getEntryNode();
2136
2137 // isTailCall may be true since the callee does not reference caller stack
2138 // frame. Check if it's in the right position and that the return types match.
2139 SDValue TCChain = InChain;
2140 const Function &F = DAG.getMachineFunction().getFunction();
2141 bool isTailCall =
2142 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2143 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()) &&
2144 // Lowering doesn't support tail calling inside a function with
2145 // a swifterror argument yet.
2146 !DAG.hasSwiftErrorArg();
2147 if (isTailCall)
2148 InChain = TCChain;
2149
2150 TargetLowering::CallLoweringInfo CLI(DAG);
2151 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2152 CLI.setDebugLoc(SDLoc(Node))
2153 .setChain(InChain)
2154 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
2155 Target: Callee, ArgsList: std::move(Args))
2156 .setTailCall(isTailCall)
2157 .setSExtResult(signExtend)
2158 .setZExtResult(!signExtend)
2159 .setIsPostTypeLegalization(true);
2160
2161 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2162
2163 if (!CallInfo.second.getNode()) {
2164 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2165 // It's a tailcall, return the chain (which is the DAG root).
2166 return {DAG.getRoot(), DAG.getRoot()};
2167 }
2168
2169 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2170 return CallInfo;
2171}
2172
2173std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2174 bool isSigned) {
2175 TargetLowering::ArgListTy Args;
2176 for (const SDValue &Op : Node->op_values()) {
2177 EVT ArgVT = Op.getValueType();
2178 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2179 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2180 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2181 Entry.IsZExt = !Entry.IsSExt;
2182 Args.push_back(x: Entry);
2183 }
2184
2185 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2186 RetVT: Node->getValueType(ResNo: 0));
2187}
2188
2189void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2190 RTLIB::Libcall LC,
2191 SmallVectorImpl<SDValue> &Results) {
2192 if (LC == RTLIB::UNKNOWN_LIBCALL)
2193 llvm_unreachable("Can't create an unknown libcall!");
2194
2195 if (Node->isStrictFPOpcode()) {
2196 EVT RetVT = Node->getValueType(ResNo: 0);
2197 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2198 if (LCImpl == RTLIB::Unsupported) {
2199 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2200 Node->getOperationName(G: &DAG));
2201 Results.push_back(Elt: DAG.getPOISON(VT: RetVT));
2202 Results.push_back(Elt: Node->getOperand(Num: 0));
2203 return;
2204 }
2205 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2206 TargetLowering::MakeLibCallOptions CallOptions;
2207 CallOptions.IsPostTypeLegalization = true;
2208 // FIXME: This doesn't support tail calls.
2209 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2210 DAG, LibcallImpl: LCImpl, RetVT, Ops, CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
2211 Results.push_back(Elt: Tmp.first);
2212 Results.push_back(Elt: Tmp.second);
2213 } else {
2214 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2215 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2216 Results.push_back(Elt: Tmp);
2217 }
2218}
2219
2220/// Expand the node to a libcall based on the result type.
2221void SelectionDAGLegalize::ExpandFastFPLibCall(
2222 SDNode *Node, bool IsFast,
2223 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2224 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2225 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2226 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2227 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2228 SmallVectorImpl<SDValue> &Results) {
2229
2230 EVT VT = Node->getSimpleValueType(ResNo: 0);
2231
2232 RTLIB::Libcall LC;
2233
2234 // FIXME: Probably should define fast to respect nan/inf and only be
2235 // approximate functions.
2236
2237 if (IsFast) {
2238 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.first, Call_F64: Call_F64.first, Call_F80: Call_F80.first,
2239 Call_F128: Call_F128.first, Call_PPCF128: Call_PPCF128.first);
2240 }
2241
2242 if (!IsFast || DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
2243 // Fall back if we don't have a fast implementation.
2244 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.second, Call_F64: Call_F64.second,
2245 Call_F80: Call_F80.second, Call_F128: Call_F128.second,
2246 Call_PPCF128: Call_PPCF128.second);
2247 }
2248
2249 ExpandFPLibCall(Node, LC, Results);
2250}
2251
2252SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2253 RTLIB::Libcall Call_I8,
2254 RTLIB::Libcall Call_I16,
2255 RTLIB::Libcall Call_I32,
2256 RTLIB::Libcall Call_I64,
2257 RTLIB::Libcall Call_I128) {
2258 RTLIB::Libcall LC;
2259 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2260 default: llvm_unreachable("Unexpected request for libcall!");
2261 case MVT::i8: LC = Call_I8; break;
2262 case MVT::i16: LC = Call_I16; break;
2263 case MVT::i32: LC = Call_I32; break;
2264 case MVT::i64: LC = Call_I64; break;
2265 case MVT::i128: LC = Call_I128; break;
2266 }
2267 return ExpandLibCall(LC, Node, isSigned).first;
2268}
2269
2270/// Expand the node to a libcall based on first argument type (for instance
2271/// lround and its variant).
2272void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2273 RTLIB::Libcall Call_F32,
2274 RTLIB::Libcall Call_F64,
2275 RTLIB::Libcall Call_F80,
2276 RTLIB::Libcall Call_F128,
2277 RTLIB::Libcall Call_PPCF128,
2278 SmallVectorImpl<SDValue> &Results) {
2279 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2280 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2281 Call_F32, Call_F64, Call_F80,
2282 Call_F128, Call_PPCF128);
2283 ExpandFPLibCall(Node, LC, Results);
2284}
2285
2286SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2287 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2288 RTLIB::Libcall CallI128) {
2289 RTLIB::Libcall LC;
2290 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2291 default:
2292 llvm_unreachable("Unexpected request for libcall!");
2293 case MVT::i32:
2294 LC = CallI32;
2295 break;
2296 case MVT::i64:
2297 LC = CallI64;
2298 break;
2299 case MVT::i128:
2300 LC = CallI128;
2301 break;
2302 }
2303
2304 // Bit-counting libcalls have one unsigned argument and return `int`.
2305 // Note that `int` may be illegal on this target; ExpandLibCall will
2306 // take care of promoting it to a legal type.
2307 SDValue Op = Node->getOperand(Num: 0);
2308 EVT IntVT =
2309 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2310
2311 EVT ArgVT = Op.getValueType();
2312 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2313 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2314 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2315 Arg.IsZExt = !Arg.IsSExt;
2316
2317 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2318 /*IsSigned=*/true, RetVT: IntVT)
2319 .first;
2320
2321 // If ExpandLibCall created a tail call, the result was already
2322 // of the correct type. Otherwise, we need to sign extend it.
2323 if (Res.getValueType() != MVT::Other)
2324 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2325 return Res;
2326}
2327
2328/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2329void
2330SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2331 SmallVectorImpl<SDValue> &Results) {
2332 unsigned Opcode = Node->getOpcode();
2333 bool isSigned = Opcode == ISD::SDIVREM;
2334
2335 RTLIB::Libcall LC;
2336 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2337 default: llvm_unreachable("Unexpected request for libcall!");
2338 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2339 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2340 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2341 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2342 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2343 }
2344
2345 // The input chain to this libcall is the entry node of the function.
2346 // Legalizing the call will automatically add the previous call to the
2347 // dependence.
2348 SDValue InChain = DAG.getEntryNode();
2349
2350 EVT RetVT = Node->getValueType(ResNo: 0);
2351 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2352
2353 TargetLowering::ArgListTy Args;
2354 for (const SDValue &Op : Node->op_values()) {
2355 EVT ArgVT = Op.getValueType();
2356 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2357 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2358 Entry.IsSExt = isSigned;
2359 Entry.IsZExt = !isSigned;
2360 Args.push_back(x: Entry);
2361 }
2362
2363 // Also pass the return address of the remainder.
2364 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2365 TargetLowering::ArgListEntry Entry(
2366 FIPtr, PointerType::getUnqual(C&: RetTy->getContext()));
2367 Entry.IsSExt = isSigned;
2368 Entry.IsZExt = !isSigned;
2369 Args.push_back(x: Entry);
2370
2371 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2372 if (LibcallImpl == RTLIB::Unsupported) {
2373 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2374 Node->getOperationName(G: &DAG));
2375 SDValue Poison = DAG.getPOISON(VT: RetVT);
2376 Results.push_back(Elt: Poison);
2377 Results.push_back(Elt: Poison);
2378 return;
2379 }
2380
2381 SDValue Callee =
2382 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2383
2384 SDLoc dl(Node);
2385 TargetLowering::CallLoweringInfo CLI(DAG);
2386 CLI.setDebugLoc(dl)
2387 .setChain(InChain)
2388 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
2389 ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2390 .setSExtResult(isSigned)
2391 .setZExtResult(!isSigned);
2392
2393 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2394
2395 // Remainder is loaded back from the stack frame.
2396 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
2397 MachinePointerInfo PtrInfo =
2398 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2399
2400 SDValue Rem = DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo);
2401 Results.push_back(Elt: CallInfo.first);
2402 Results.push_back(Elt: Rem);
2403}
2404
2405/// Return true if sincos or __sincos_stret libcall is available.
2406static bool isSinCosLibcallAvailable(SDNode *Node,
2407 const LibcallLoweringInfo &Libcalls) {
2408 MVT::SimpleValueType VT = Node->getSimpleValueType(ResNo: 0).SimpleTy;
2409 return Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS(VT)) != RTLIB::Unsupported ||
2410 Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS_STRET(VT)) !=
2411 RTLIB::Unsupported;
2412}
2413
2414/// Only issue sincos libcall if both sin and cos are needed.
2415static bool useSinCos(SDNode *Node) {
2416 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2417 ? ISD::FCOS : ISD::FSIN;
2418
2419 SDValue Op0 = Node->getOperand(Num: 0);
2420 for (const SDNode *User : Op0.getNode()->users()) {
2421 if (User == Node)
2422 continue;
2423 // The other user might have been turned into sincos already.
2424 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2425 return true;
2426 }
2427 return false;
2428}
2429
2430SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2431 // For iOS, we want to call an alternative entry point: __sincos_stret,
2432 // which returns the values in two S / D registers.
2433 SDLoc dl(Node);
2434 SDValue Arg = Node->getOperand(Num: 0);
2435 EVT ArgVT = Arg.getValueType();
2436 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(VT: ArgVT);
2437 RTLIB::LibcallImpl SincosStret = DAG.getLibcalls().getLibcallImpl(Call: LC);
2438 if (SincosStret == RTLIB::Unsupported)
2439 return SDValue();
2440
2441 /// There are 3 different ABI cases to handle:
2442 /// - Direct return of separate fields in registers
2443 /// - Single return as vector elements
2444 /// - sret struct
2445
2446 const RTLIB::RuntimeLibcallsInfo &CallsInfo = TLI.getRuntimeLibcallsInfo();
2447
2448 const DataLayout &DL = DAG.getDataLayout();
2449
2450 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2451 Ctx&: *DAG.getContext(), TT: TM.getTargetTriple(), DL, LibcallImpl: SincosStret);
2452
2453 Type *SincosStretRetTy = FuncTy->getReturnType();
2454 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(Call: SincosStret);
2455
2456 SDValue Callee =
2457 DAG.getExternalSymbol(LCImpl: SincosStret, VT: TLI.getProgramPointerTy(DL));
2458
2459 TargetLowering::ArgListTy Args;
2460 SDValue SRet;
2461
2462 int FrameIdx;
2463 if (FuncTy->getParamType(i: 0)->isPointerTy()) {
2464 // Uses sret
2465 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2466
2467 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(ArgNo: 0);
2468 Type *StructTy = PtrAttrs.getStructRetType();
2469 const uint64_t ByteSize = DL.getTypeAllocSize(Ty: StructTy);
2470 const Align StackAlign = DL.getPrefTypeAlign(Ty: StructTy);
2471
2472 FrameIdx = MFI.CreateStackObject(Size: ByteSize, Alignment: StackAlign, isSpillSlot: false);
2473 SRet = DAG.getFrameIndex(FI: FrameIdx, VT: TLI.getFrameIndexTy(DL));
2474
2475 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(i: 0));
2476 Entry.IsSRet = true;
2477 Entry.IndirectType = StructTy;
2478 Entry.Alignment = StackAlign;
2479
2480 Args.push_back(x: Entry);
2481 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 1));
2482 } else {
2483 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 0));
2484 }
2485
2486 TargetLowering::CallLoweringInfo CLI(DAG);
2487 CLI.setDebugLoc(dl)
2488 .setChain(DAG.getEntryNode())
2489 .setLibCallee(CC: CallConv, ResultType: SincosStretRetTy, Target: Callee, ArgsList: std::move(Args))
2490 .setIsPostTypeLegalization();
2491
2492 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2493
2494 if (SRet) {
2495 MachinePointerInfo PtrInfo =
2496 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2497 SDValue LoadSin = DAG.getLoad(VT: ArgVT, dl, Chain: CallResult.second, Ptr: SRet, PtrInfo);
2498
2499 TypeSize StoreSize = ArgVT.getStoreSize();
2500
2501 // Address of cos field.
2502 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr: SRet, Offset: StoreSize);
2503 SDValue LoadCos = DAG.getLoad(VT: ArgVT, dl, Chain: LoadSin.getValue(R: 1), Ptr: Add,
2504 PtrInfo: PtrInfo.getWithOffset(O: StoreSize));
2505
2506 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2507 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: LoadSin.getValue(R: 0),
2508 N2: LoadCos.getValue(R: 0));
2509 }
2510
2511 if (!CallResult.first.getValueType().isVector())
2512 return CallResult.first;
2513
2514 SDValue SinVal =
2515 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2516 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2517 SDValue CosVal =
2518 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2519 N2: DAG.getVectorIdxConstant(Val: 1, DL: dl));
2520 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2521 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: SinVal, N2: CosVal);
2522}
2523
2524SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2525 SDLoc dl(Node);
2526 EVT VT = Node->getValueType(ResNo: 0);
2527 SDValue X = Node->getOperand(Num: 0);
2528 SDValue N = Node->getOperand(Num: 1);
2529 EVT ExpVT = N.getValueType();
2530 EVT AsIntVT = VT.changeTypeToInteger();
2531 if (AsIntVT == EVT()) // TODO: How to handle f80?
2532 return SDValue();
2533
2534 // The expansion works through the integer-equivalent type; if that is not
2535 // legal, bail out and let the caller use a libcall (or diagnose a missing
2536 // one).
2537 if (!TLI.isTypeLegal(VT: AsIntVT))
2538 return SDValue();
2539
2540 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2541 return SDValue();
2542
2543 SDNodeFlags NSW;
2544 NSW.setNoSignedWrap(true);
2545 SDNodeFlags NUW_NSW;
2546 NUW_NSW.setNoUnsignedWrap(true);
2547 NUW_NSW.setNoSignedWrap(true);
2548
2549 EVT SetCCVT =
2550 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2551 const fltSemantics &FltSem = VT.getFltSemantics();
2552
2553 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2554 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2555 const int Precision = APFloat::semanticsPrecision(FltSem);
2556
2557 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2558 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2559
2560 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2561
2562 const APFloat One(FltSem, "1.0");
2563 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2564
2565 // Offset by precision to avoid denormal range.
2566 APFloat ScaleDownK =
2567 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2568
2569 // TODO: Should really introduce control flow and use a block for the >
2570 // MaxExp, < MinExp cases
2571
2572 // First, handle exponents Exp > MaxExp and scale down.
2573 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2574
2575 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2576 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2577 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2578 SDValue DecN1 =
2579 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2580
2581 SDValue ScaleUpTwice =
2582 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2583
2584 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2585 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2586 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2587
2588 SDValue SelectN_Big =
2589 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2590 SDValue SelectX_Big =
2591 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2592
2593 // Now handle exponents Exp < MinExp
2594 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2595
2596 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2597 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2598
2599 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2600
2601 SDValue ClampMinVal =
2602 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2603 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2604 SDValue IncN1 =
2605 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2606
2607 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2608 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2609 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2610
2611 SDValue ScaleDownTwice = DAG.getSetCC(
2612 DL: dl, VT: SetCCVT, LHS: N,
2613 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2614
2615 SDValue SelectN_Small =
2616 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2617 SDValue SelectX_Small =
2618 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2619
2620 // Now combine the two out of range exponent handling cases with the base
2621 // case.
2622 SDValue NewX = DAG.getNode(
2623 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2624 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2625
2626 SDValue NewN = DAG.getNode(
2627 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2628 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2629
2630 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2631
2632 SDValue ExponentShiftAmt =
2633 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2634 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2635
2636 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2637 N2: ExponentShiftAmt, Flags: NUW_NSW);
2638 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2639 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2640}
2641
2642SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2643 SDLoc dl(Node);
2644 SDValue Val = Node->getOperand(Num: 0);
2645 EVT VT = Val.getValueType();
2646 EVT ExpVT = Node->getValueType(ResNo: 1);
2647 EVT AsIntVT = VT.changeTypeToInteger();
2648 if (AsIntVT == EVT()) // TODO: How to handle f80?
2649 return SDValue();
2650
2651 // The expansion works through the integer-equivalent type; if that is not
2652 // legal, bail out and let the caller use a libcall (or diagnose a missing
2653 // one).
2654 if (!TLI.isTypeLegal(VT: AsIntVT))
2655 return SDValue();
2656
2657 const fltSemantics &FltSem = VT.getFltSemantics();
2658 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2659 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2660 const unsigned BitSize = VT.getScalarSizeInBits();
2661
2662 // TODO: Could introduce control flow and skip over the denormal handling.
2663
2664 // scale_up = fmul value, scalbn(1.0, precision + 1)
2665 // extracted_exp = (bitcast value to uint) >> precision - 1
2666 // biased_exp = extracted_exp + min_exp
2667 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2668 //
2669 // is_denormal = val < smallest_normalized
2670 // computed_fract = is_denormal ? scale_up : extracted_fract
2671 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2672 //
2673 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2674 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2675
2676 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2677 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2678 VT: AsIntVT);
2679
2680 SDValue SmallestNormalizedInt = DAG.getConstant(
2681 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2682 VT: AsIntVT);
2683
2684 // Masks out the exponent bits.
2685 SDValue ExpMask =
2686 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2687
2688 // Mask out the exponent part of the value.
2689 //
2690 // e.g, for f32 FractSignMaskVal = 0x807fffff
2691 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2692 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2693
2694 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2695 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2696
2697 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2698
2699 const APFloat One(FltSem, "1.0");
2700 // Scale a possible denormal input.
2701 // e.g., for f64, 0x1p+54
2702 APFloat ScaleUpKVal =
2703 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2704
2705 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2706 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2707
2708 EVT SetCCVT =
2709 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2710
2711 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2712
2713 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2714
2715 SDValue AddNegSmallestNormal =
2716 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2717 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2718 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2719
2720 SDValue IsDenormal =
2721 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2722
2723 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2724 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2725
2726 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2727 SDValue ScaledSelect =
2728 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2729
2730 SDValue ExpMaskScaled =
2731 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2732
2733 SDValue ScaledValue =
2734 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2735
2736 // Extract the exponent bits.
2737 SDValue ExponentShiftAmt =
2738 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2739 SDValue ShiftedExp =
2740 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2741 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2742
2743 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2744 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2745 SDValue DenormalExpBias =
2746 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2747
2748 SDValue MaskedFractAsInt =
2749 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2750 const APFloat Half(FltSem, "0.5");
2751 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2752 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2753 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2754
2755 SDValue ComputedExp =
2756 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2757
2758 SDValue Result0 =
2759 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2760
2761 SDValue Result1 =
2762 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2763
2764 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2765}
2766
2767SDValue SelectionDAGLegalize::expandModf(SDNode *Node) const {
2768 SDLoc dl(Node);
2769 SDValue Val = Node->getOperand(Num: 0);
2770 EVT VT = Val.getValueType();
2771 SDNodeFlags Flags = Node->getFlags();
2772
2773 SDValue IntPart = DAG.getNode(Opcode: ISD::FTRUNC, DL: dl, VT, Operand: Val, Flags);
2774 SDValue FracPart = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT, N1: Val, N2: IntPart, Flags);
2775
2776 SDValue FracToUse;
2777 if (Flags.hasNoInfs()) {
2778 FracToUse = FracPart;
2779 } else {
2780 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val, Flags);
2781 SDValue Inf =
2782 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
2783 EVT SetCCVT =
2784 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2785 SDValue IsInf = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: Inf, Cond: ISD::SETOEQ);
2786 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: dl, VT);
2787 FracToUse = DAG.getSelect(DL: dl, VT, Cond: IsInf, LHS: Zero, RHS: FracPart);
2788 }
2789
2790 SDValue ResultFrac =
2791 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT, N1: FracToUse, N2: Val, Flags);
2792 return DAG.getMergeValues(Ops: {ResultFrac, IntPart}, dl);
2793}
2794
2795/// This function is responsible for legalizing a
2796/// INT_TO_FP operation of the specified operand when the target requests that
2797/// we expand it. At this point, we know that the result and operand types are
2798/// legal for the target.
2799SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2800 SDValue &Chain) {
2801 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2802 Node->getOpcode() == ISD::SINT_TO_FP);
2803 EVT DestVT = Node->getValueType(ResNo: 0);
2804 SDLoc dl(Node);
2805 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2806 SDValue Op0 = Node->getOperand(Num: OpNo);
2807 EVT SrcVT = Op0.getValueType();
2808
2809 // TODO: Should any fast-math-flags be set for the created nodes?
2810 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2811 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2812 (DestVT.bitsLE(VT: MVT::f64) ||
2813 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2814 : ISD::FP_EXTEND,
2815 VT: DestVT))) {
2816 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2817 "expansion\n");
2818
2819 // Get the stack frame index of a 8 byte buffer.
2820 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2821
2822 SDValue Lo = Op0;
2823 // if signed map to unsigned space
2824 if (isSigned) {
2825 // Invert sign bit (signed to unsigned mapping).
2826 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2827 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2828 }
2829 // Initial hi portion of constructed double.
2830 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2831
2832 // If this a big endian target, swap the lo and high data.
2833 if (DAG.getDataLayout().isBigEndian())
2834 std::swap(a&: Lo, b&: Hi);
2835
2836 SDValue MemChain = DAG.getEntryNode();
2837
2838 // Store the lo of the constructed double.
2839 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2840 PtrInfo: MachinePointerInfo());
2841 // Store the hi of the constructed double.
2842 SDValue HiPtr =
2843 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2844 SDValue Store2 =
2845 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2846 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2847
2848 // load the constructed double
2849 SDValue Load =
2850 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2851 // FP constant to bias correct the final result
2852 SDValue Bias = DAG.getConstantFP(
2853 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2854 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2855 DL: dl, VT: MVT::f64);
2856 // Subtract the bias and get the final result.
2857 SDValue Sub;
2858 SDValue Result;
2859 if (Node->isStrictFPOpcode()) {
2860 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2861 Ops: {Node->getOperand(Num: 0), Load, Bias});
2862 Chain = Sub.getValue(R: 1);
2863 if (DestVT != Sub.getValueType()) {
2864 std::pair<SDValue, SDValue> ResultPair;
2865 ResultPair =
2866 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2867 Result = ResultPair.first;
2868 Chain = ResultPair.second;
2869 }
2870 else
2871 Result = Sub;
2872 } else {
2873 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2874 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2875 }
2876 return Result;
2877 }
2878
2879 if (isSigned)
2880 return SDValue();
2881
2882 // TODO: Generalize this for use with other types.
2883 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2884 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2885 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2886 // For unsigned conversions, convert them to signed conversions using the
2887 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2888 // should be valid for i32->f32 as well.
2889
2890 // More generally this transform should be valid if there are 3 more bits
2891 // in the integer type than the significand. Rounding uses the first bit
2892 // after the width of the significand and the OR of all bits after that. So
2893 // we need to be able to OR the shifted out bit into one of the bits that
2894 // participate in the OR.
2895
2896 // TODO: This really should be implemented using a branch rather than a
2897 // select. We happen to get lucky and machinesink does the right
2898 // thing most of the time. This would be a good candidate for a
2899 // pseudo-op, or, even better, for whole-function isel.
2900 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2901
2902 SDValue SignBitTest = DAG.getSetCC(
2903 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2904
2905 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL: dl);
2906 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2907 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2908 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2909 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2910
2911 SDValue Slow, Fast;
2912 if (Node->isStrictFPOpcode()) {
2913 // In strict mode, we must avoid spurious exceptions, and therefore
2914 // must make sure to only emit a single STRICT_SINT_TO_FP.
2915 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2916 // The STRICT_SINT_TO_FP inherits the exception mode from the
2917 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2918 // never raise any exception.
2919 SDNodeFlags Flags;
2920 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2921 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: {DestVT, MVT::Other},
2922 Ops: {Node->getOperand(Num: 0), InCvt}, Flags);
2923 Flags.setNoFPExcept(true);
2924 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: {DestVT, MVT::Other},
2925 Ops: {Fast.getValue(R: 1), Fast, Fast}, Flags);
2926 Chain = Slow.getValue(R: 1);
2927 } else {
2928 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2929 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2930 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2931 }
2932
2933 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2934 }
2935
2936 // Don't expand it if there isn't cheap fadd.
2937 if (!TLI.isOperationLegalOrCustom(
2938 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2939 return SDValue();
2940
2941 // The following optimization is valid only if every value in SrcVT (when
2942 // treated as signed) is representable in DestVT. Check that the mantissa
2943 // size of DestVT is >= than the number of bits in SrcVT -1.
2944 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2945 SrcVT.getSizeInBits() - 1 &&
2946 "Cannot perform lossless SINT_TO_FP!");
2947
2948 SDValue Tmp1;
2949 if (Node->isStrictFPOpcode()) {
2950 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2951 Ops: { Node->getOperand(Num: 0), Op0 });
2952 } else
2953 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2954
2955 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2956 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2957 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2958 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2959 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2960 Cond: SignSet, LHS: Four, RHS: Zero);
2961
2962 // If the sign bit of the integer is set, the large number will be treated
2963 // as a negative number. To counteract this, the dynamic code adds an
2964 // offset depending on the data type.
2965 uint64_t FF;
2966 switch (SrcVT.getSimpleVT().SimpleTy) {
2967 default:
2968 return SDValue();
2969 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2970 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2971 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2972 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2973 }
2974 if (DAG.getDataLayout().isLittleEndian())
2975 FF <<= 32;
2976 Constant *FudgeFactor = ConstantInt::get(
2977 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2978
2979 SDValue CPIdx =
2980 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2981 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2982 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2983 Alignment = commonAlignment(A: Alignment, Offset: 4);
2984 SDValue FudgeInReg;
2985 if (DestVT == MVT::f32)
2986 FudgeInReg = DAG.getLoad(
2987 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2988 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2989 Alignment);
2990 else {
2991 SDValue Load = DAG.getExtLoad(
2992 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2993 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
2994 Alignment);
2995 HandleSDNode Handle(Load);
2996 LegalizeOp(Node: Load.getNode());
2997 FudgeInReg = Handle.getValue();
2998 }
2999
3000 if (Node->isStrictFPOpcode()) {
3001 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
3002 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
3003 Chain = Result.getValue(R: 1);
3004 return Result;
3005 }
3006
3007 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
3008}
3009
3010/// This function is responsible for legalizing a
3011/// *INT_TO_FP operation of the specified operand when the target requests that
3012/// we promote it. At this point, we know that the result and operand types are
3013/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
3014/// operation that takes a larger input.
3015void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
3016 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
3017 bool IsStrict = N->isStrictFPOpcode();
3018 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
3019 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
3020 EVT DestVT = N->getValueType(ResNo: 0);
3021 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3022 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
3023 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
3024
3025 // First step, figure out the appropriate *INT_TO_FP operation to use.
3026 EVT NewInTy = LegalOp.getValueType();
3027
3028 unsigned OpToUse = 0;
3029
3030 // Scan for the appropriate larger type to use.
3031 while (true) {
3032 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
3033 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3034
3035 // If the target supports SINT_TO_FP of this type, use it.
3036 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
3037 OpToUse = SIntOp;
3038 break;
3039 }
3040 if (IsSigned)
3041 continue;
3042
3043 // If the target supports UINT_TO_FP of this type, use it.
3044 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
3045 OpToUse = UIntOp;
3046 break;
3047 }
3048
3049 // Otherwise, try a larger type.
3050 }
3051
3052 // Okay, we found the operation and type to use. Zero extend our input to the
3053 // desired type then run the operation on it.
3054 if (IsStrict) {
3055 SDValue Res =
3056 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
3057 Ops: {N->getOperand(Num: 0),
3058 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3059 DL: dl, VT: NewInTy, Operand: LegalOp)});
3060 Results.push_back(Elt: Res);
3061 Results.push_back(Elt: Res.getValue(R: 1));
3062 return;
3063 }
3064
3065 Results.push_back(
3066 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
3067 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3068 DL: dl, VT: NewInTy, Operand: LegalOp)));
3069}
3070
3071/// This function is responsible for legalizing a
3072/// FP_TO_*INT operation of the specified operand when the target requests that
3073/// we promote it. At this point, we know that the result and operand types are
3074/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
3075/// operation that returns a larger result.
3076void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3077 SmallVectorImpl<SDValue> &Results) {
3078 bool IsStrict = N->isStrictFPOpcode();
3079 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3080 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3081 EVT DestVT = N->getValueType(ResNo: 0);
3082 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3083 // First step, figure out the appropriate FP_TO*INT operation to use.
3084 EVT NewOutTy = DestVT;
3085
3086 unsigned OpToUse = 0;
3087
3088 // Scan for the appropriate larger type to use.
3089 while (true) {
3090 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
3091 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3092
3093 // A larger signed type can hold all unsigned values of the requested type,
3094 // so using FP_TO_SINT is valid
3095 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
3096 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3097 break;
3098
3099 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
3100 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
3101 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3102 break;
3103
3104 // Otherwise, try a larger type.
3105 }
3106
3107 // Okay, we found the operation and type to use.
3108 SDValue Operation;
3109 if (IsStrict) {
3110 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
3111 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
3112 } else
3113 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
3114
3115 // Truncate the result of the extended FP_TO_*INT operation to the desired
3116 // size.
3117 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
3118 Results.push_back(Elt: Trunc);
3119 if (IsStrict)
3120 Results.push_back(Elt: Operation.getValue(R: 1));
3121}
3122
3123/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
3124/// the result and operand types are legal and there must be a legal
3125/// FP_TO_*INT_SAT operation for a larger result type.
3126SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3127 const SDLoc &dl) {
3128 unsigned Opcode = Node->getOpcode();
3129
3130 // Scan for the appropriate larger type to use.
3131 EVT NewOutTy = Node->getValueType(ResNo: 0);
3132 while (true) {
3133 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
3134 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3135
3136 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
3137 break;
3138 }
3139
3140 // Saturation width is determined by second operand, so we don't have to
3141 // perform any fixup and can directly truncate the result.
3142 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
3143 N2: Node->getOperand(Num: 1));
3144 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
3145}
3146
3147/// Open code the operations for PARITY of the specified operation.
3148SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3149 EVT VT = Op.getValueType();
3150 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3151 unsigned Sz = VT.getScalarSizeInBits();
3152
3153 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3154 SDValue Result;
3155 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3156 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3157 } else {
3158 Result = Op;
3159 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3160 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3161 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3162 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3163 }
3164 }
3165
3166 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3167}
3168
3169SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3170 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3171 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3172 : Node->getOperand(Num: 0).getSimpleValueType();
3173 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3174 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3175 MVT NewScalarVT = NewVecVT.getVectorElementType();
3176
3177 SDLoc DL(Node);
3178 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3179
3180 // FIXME: Support integer.
3181 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3182 "Only FP promotion is supported");
3183
3184 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3185 if (Node->getOperand(Num: j).getValueType().isVector() &&
3186 !(IsVPOpcode &&
3187 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3188 // promote the vector operand.
3189 // FIXME: Support integer.
3190 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3191 "Only FP promotion is supported");
3192 Operands[j] =
3193 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3194 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3195 // promote the initial value.
3196 Operands[j] =
3197 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3198 } else {
3199 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3200 }
3201
3202 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3203 Flags: Node->getFlags());
3204
3205 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3206 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3207 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3208}
3209
3210bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3211 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3212 SmallVector<SDValue, 8> Results;
3213 SDLoc dl(Node);
3214 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3215 bool NeedInvert;
3216 switch (Node->getOpcode()) {
3217 case ISD::ABS:
3218 case ISD::ABS_MIN_POISON:
3219 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3220 Results.push_back(Elt: Tmp1);
3221 break;
3222 case ISD::ABDS:
3223 case ISD::ABDU:
3224 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3225 Results.push_back(Elt: Tmp1);
3226 break;
3227 case ISD::AVGCEILS:
3228 case ISD::AVGCEILU:
3229 case ISD::AVGFLOORS:
3230 case ISD::AVGFLOORU:
3231 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3232 Results.push_back(Elt: Tmp1);
3233 break;
3234 case ISD::CTPOP:
3235 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3236 Results.push_back(Elt: Tmp1);
3237 break;
3238 case ISD::CTLZ:
3239 case ISD::CTLZ_ZERO_POISON:
3240 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3241 Results.push_back(Elt: Tmp1);
3242 break;
3243 case ISD::CTLS:
3244 if ((Tmp1 = TLI.expandCTLS(N: Node, DAG)))
3245 Results.push_back(Elt: Tmp1);
3246 break;
3247 case ISD::CTTZ:
3248 case ISD::CTTZ_ZERO_POISON:
3249 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3250 Results.push_back(Elt: Tmp1);
3251 break;
3252 case ISD::BITREVERSE:
3253 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3254 Results.push_back(Elt: Tmp1);
3255 break;
3256 case ISD::BSWAP:
3257 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3258 Results.push_back(Elt: Tmp1);
3259 break;
3260 case ISD::PARITY:
3261 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3262 break;
3263 case ISD::FRAMEADDR:
3264 case ISD::RETURNADDR:
3265 case ISD::FRAME_TO_ARGS_OFFSET:
3266 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3267 break;
3268 case ISD::EH_DWARF_CFA: {
3269 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3270 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3271 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3272 VT: CfaArg.getValueType(),
3273 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3274 VT: CfaArg.getValueType()),
3275 N2: CfaArg);
3276 SDValue FA = DAG.getNode(
3277 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3278 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3279 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3280 N1: FA, N2: Offset));
3281 break;
3282 }
3283 case ISD::GET_ROUNDING:
3284 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3285 Results.push_back(Elt: Node->getOperand(Num: 0));
3286 break;
3287 case ISD::EH_RETURN:
3288 case ISD::EH_LABEL:
3289 case ISD::PREFETCH:
3290 case ISD::VAEND:
3291 case ISD::EH_SJLJ_LONGJMP:
3292 // If the target didn't expand these, there's nothing to do, so just
3293 // preserve the chain and be done.
3294 Results.push_back(Elt: Node->getOperand(Num: 0));
3295 break;
3296 case ISD::READCYCLECOUNTER:
3297 case ISD::READSTEADYCOUNTER:
3298 // If the target didn't expand this, just return 'zero' and preserve the
3299 // chain.
3300 Results.append(NumInputs: Node->getNumValues() - 1,
3301 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3302 Results.push_back(Elt: Node->getOperand(Num: 0));
3303 break;
3304 case ISD::EH_SJLJ_SETJMP:
3305 // If the target didn't expand this, just return 'zero' and preserve the
3306 // chain.
3307 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3308 Results.push_back(Elt: Node->getOperand(Num: 0));
3309 break;
3310 case ISD::ATOMIC_LOAD: {
3311 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3312 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3313 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3314 SDValue Swap = DAG.getAtomicCmpSwap(
3315 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3316 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3317 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3318 Results.push_back(Elt: Swap.getValue(R: 0));
3319 Results.push_back(Elt: Swap.getValue(R: 1));
3320 break;
3321 }
3322 case ISD::ATOMIC_STORE: {
3323 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3324 SDValue Swap = DAG.getAtomic(
3325 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3326 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3327 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3328 Results.push_back(Elt: Swap.getValue(R: 1));
3329 break;
3330 }
3331 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3332 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3333 // splits out the success value as a comparison. Expanding the resulting
3334 // ATOMIC_CMP_SWAP will produce a libcall.
3335 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3336 SDValue Res = DAG.getAtomicCmpSwap(
3337 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3338 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3339 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3340
3341 SDValue ExtRes = Res;
3342 SDValue LHS = Res;
3343 SDValue RHS = Node->getOperand(Num: 1);
3344
3345 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3346 EVT OuterType = Node->getValueType(ResNo: 0);
3347 switch (TLI.getExtendForAtomicOps()) {
3348 case ISD::SIGN_EXTEND:
3349 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3350 N2: DAG.getValueType(AtomicType));
3351 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3352 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3353 ExtRes = LHS;
3354 break;
3355 case ISD::ZERO_EXTEND:
3356 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3357 N2: DAG.getValueType(AtomicType));
3358 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3359 ExtRes = LHS;
3360 break;
3361 case ISD::ANY_EXTEND:
3362 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3363 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3364 break;
3365 default:
3366 llvm_unreachable("Invalid atomic op extension");
3367 }
3368
3369 SDValue Success =
3370 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3371
3372 Results.push_back(Elt: ExtRes.getValue(R: 0));
3373 Results.push_back(Elt: Success);
3374 Results.push_back(Elt: Res.getValue(R: 1));
3375 break;
3376 }
3377 case ISD::ATOMIC_LOAD_SUB: {
3378 SDLoc DL(Node);
3379 EVT VT = Node->getValueType(ResNo: 0);
3380 SDValue RHS = Node->getOperand(Num: 2);
3381 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3382 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3383 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3384 RHS = RHS->getOperand(Num: 0);
3385 SDValue NewRHS =
3386 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3387 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3388 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3389 Val: NewRHS, MMO: AN->getMemOperand());
3390 Results.push_back(Elt: Res);
3391 Results.push_back(Elt: Res.getValue(R: 1));
3392 break;
3393 }
3394 case ISD::ATOMIC_LOAD_FSUB: {
3395 SDLoc DL(Node);
3396 EVT VT = Node->getValueType(ResNo: 0);
3397 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3398 SDValue NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Node->getOperand(Num: 2));
3399 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_FADD, dl: DL, MemVT: AN->getMemoryVT(),
3400 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3401 Val: NewRHS, MMO: AN->getMemOperand());
3402 Results.push_back(Elt: Res);
3403 Results.push_back(Elt: Res.getValue(R: 1));
3404 break;
3405 }
3406 case ISD::DYNAMIC_STACKALLOC:
3407 ExpandDYNAMIC_STACKALLOC(Node, Results);
3408 break;
3409 case ISD::MERGE_VALUES:
3410 for (unsigned i = 0; i < Node->getNumValues(); i++)
3411 Results.push_back(Elt: Node->getOperand(Num: i));
3412 break;
3413 case ISD::POISON:
3414 case ISD::UNDEF: {
3415 EVT VT = Node->getValueType(ResNo: 0);
3416 if (VT.isInteger())
3417 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3418 else {
3419 assert(VT.isFloatingPoint() && "Unknown value type!");
3420 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3421 }
3422 break;
3423 }
3424 case ISD::STRICT_FP_ROUND:
3425 // When strict mode is enforced we can't do expansion because it
3426 // does not honor the "strict" properties. Only libcall is allowed.
3427 if (TLI.isStrictFPEnabled())
3428 break;
3429 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3430 // since this operation is more efficient than stack operation.
3431 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3432 VT: Node->getValueType(ResNo: 0))
3433 == TargetLowering::Legal)
3434 break;
3435 // We fall back to use stack operation when the FP_ROUND operation
3436 // isn't available.
3437 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3438 DestVT: Node->getValueType(ResNo: 0), dl,
3439 Chain: Node->getOperand(Num: 0)))) {
3440 ReplaceNode(Old: Node, New: Tmp1.getNode());
3441 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3442 return true;
3443 }
3444 break;
3445 case ISD::FP_ROUND: {
3446 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3447 Results.push_back(Elt: Tmp1);
3448 break;
3449 }
3450
3451 [[fallthrough]];
3452 }
3453 case ISD::BITCAST:
3454 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3455 DestVT: Node->getValueType(ResNo: 0), dl)))
3456 Results.push_back(Elt: Tmp1);
3457 break;
3458 case ISD::STRICT_FP_EXTEND:
3459 // When strict mode is enforced we can't do expansion because it
3460 // does not honor the "strict" properties. Only libcall is allowed.
3461 if (TLI.isStrictFPEnabled())
3462 break;
3463 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3464 // since this operation is more efficient than stack operation.
3465 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3466 VT: Node->getValueType(ResNo: 0))
3467 == TargetLowering::Legal)
3468 break;
3469 // We fall back to use stack operation when the FP_EXTEND operation
3470 // isn't available.
3471 if ((Tmp1 = EmitStackConvert(
3472 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3473 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3474 ReplaceNode(Old: Node, New: Tmp1.getNode());
3475 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3476 return true;
3477 }
3478 break;
3479 case ISD::FP_EXTEND: {
3480 SDValue Op = Node->getOperand(Num: 0);
3481 EVT SrcVT = Op.getValueType();
3482 EVT DstVT = Node->getValueType(ResNo: 0);
3483 if (SrcVT.getScalarType() == MVT::bf16) {
3484 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3485 break;
3486 }
3487
3488 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3489 Results.push_back(Elt: Tmp1);
3490 break;
3491 }
3492 case ISD::BF16_TO_FP: {
3493 // Always expand bf16 to f32 casts, they lower to ext + shift.
3494 //
3495 // Note that the operand of this code can be bf16 or an integer type in case
3496 // bf16 is not supported on the target and was softened.
3497 SDValue Op = Node->getOperand(Num: 0);
3498 if (Op.getValueType() == MVT::bf16) {
3499 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3500 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3501 } else {
3502 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3503 }
3504 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3505 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3506 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3507 // Add fp_extend in case the output is bigger than f32.
3508 if (Node->getValueType(ResNo: 0) != MVT::f32)
3509 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3510 Results.push_back(Elt: Op);
3511 break;
3512 }
3513 case ISD::FP_TO_BF16: {
3514 SDValue Op = Node->getOperand(Num: 0);
3515 if (Op.getValueType() != MVT::f32)
3516 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3517 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3518 // Certain SNaNs will turn into infinities if we do a simple shift right.
3519 if (!DAG.isKnownNeverSNaN(Op)) {
3520 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3521 }
3522 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
3523 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3524 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3525 // The result of this node can be bf16 or an integer type in case bf16 is
3526 // not supported on the target and was softened to i16 for storage.
3527 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3528 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3529 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3530 } else {
3531 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3532 }
3533 Results.push_back(Elt: Op);
3534 break;
3535 }
3536 case ISD::CONVERT_FROM_ARBITRARY_FP: {
3537 // Expand conversion from arbitrary FP format stored in an integer to a
3538 // native IEEE float type using integer bit manipulation.
3539 //
3540 // TODO: currently only conversions from FP4, FP6 and FP8 formats from OCP
3541 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3542 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3543 // Float8E8M0FNU.
3544 EVT DstVT = Node->getValueType(ResNo: 0);
3545 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
3546 Results.push_back(Elt: Expanded);
3547 else
3548 Results.push_back(Elt: DAG.getPOISON(VT: DstVT));
3549 break;
3550 }
3551 case ISD::CONVERT_TO_ARBITRARY_FP: {
3552 // Expand conversion from a native IEEE float type to an arbitrary FP
3553 // format, returning the result as an integer using bit manipulation.
3554 //
3555 // TODO: currently only conversions to FP4, FP6 and FP8 formats from OCP
3556 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3557 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3558 // Float8E8M0FNU.
3559 EVT ResVT = Node->getValueType(ResNo: 0);
3560 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
3561 Results.push_back(Elt: Expanded);
3562 else
3563 Results.push_back(Elt: DAG.getPOISON(VT: ResVT));
3564 break;
3565 }
3566 case ISD::FCANONICALIZE: {
3567 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
3568 Results.push_back(Elt: Mul);
3569 break;
3570 }
3571 case ISD::SIGN_EXTEND_INREG: {
3572 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3573 EVT VT = Node->getValueType(ResNo: 0);
3574
3575 // An in-register sign-extend of a boolean is a negation:
3576 // 'true' (1) sign-extended is -1.
3577 // 'false' (0) sign-extended is 0.
3578 // However, we must mask the high bits of the source operand because the
3579 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3580
3581 // TODO: Do this for vectors too?
3582 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3583 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3584 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3585 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3586 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3587 Results.push_back(Elt: Neg);
3588 break;
3589 }
3590
3591 // NOTE: we could fall back on load/store here too for targets without
3592 // SRA. However, it is doubtful that any exist.
3593 unsigned BitsDiff = VT.getScalarSizeInBits() -
3594 ExtraVT.getScalarSizeInBits();
3595 SDValue ShiftCst = DAG.getShiftAmountConstant(Val: BitsDiff, VT, DL: dl);
3596 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: ShiftCst);
3597 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Tmp1, N2: ShiftCst);
3598 Results.push_back(Elt: Tmp1);
3599 break;
3600 }
3601 case ISD::UINT_TO_FP:
3602 case ISD::STRICT_UINT_TO_FP:
3603 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3604 Results.push_back(Elt: Tmp1);
3605 if (Node->isStrictFPOpcode())
3606 Results.push_back(Elt: Tmp2);
3607 break;
3608 }
3609 [[fallthrough]];
3610 case ISD::SINT_TO_FP:
3611 case ISD::STRICT_SINT_TO_FP:
3612 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3613 Results.push_back(Elt: Tmp1);
3614 if (Node->isStrictFPOpcode())
3615 Results.push_back(Elt: Tmp2);
3616 }
3617 break;
3618 case ISD::FP_TO_SINT:
3619 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3620 Results.push_back(Elt: Tmp1);
3621 break;
3622 case ISD::STRICT_FP_TO_SINT:
3623 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3624 ReplaceNode(Old: Node, New: Tmp1.getNode());
3625 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3626 return true;
3627 }
3628 break;
3629 case ISD::FP_TO_UINT:
3630 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3631 Results.push_back(Elt: Tmp1);
3632 break;
3633 case ISD::STRICT_FP_TO_UINT:
3634 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3635 // Relink the chain.
3636 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3637 // Replace the new UINT result.
3638 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3639 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3640 return true;
3641 }
3642 break;
3643 case ISD::FP_TO_SINT_SAT:
3644 case ISD::FP_TO_UINT_SAT:
3645 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3646 break;
3647 case ISD::LROUND:
3648 case ISD::LLROUND: {
3649 SDValue Arg = Node->getOperand(Num: 0);
3650 EVT ArgVT = Arg.getValueType();
3651 EVT ResVT = Node->getValueType(ResNo: 0);
3652 SDLoc dl(Node);
3653 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3654 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3655 break;
3656 }
3657 case ISD::VAARG:
3658 Results.push_back(Elt: DAG.expandVAArg(Node));
3659 Results.push_back(Elt: Results[0].getValue(R: 1));
3660 break;
3661 case ISD::VACOPY:
3662 Results.push_back(Elt: DAG.expandVACopy(Node));
3663 break;
3664 case ISD::EXTRACT_VECTOR_ELT:
3665 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3666 // This must be an access of the only element. Return it.
3667 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3668 Operand: Node->getOperand(Num: 0));
3669 else
3670 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3671 Results.push_back(Elt: Tmp1);
3672 break;
3673 case ISD::EXTRACT_SUBVECTOR:
3674 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3675 break;
3676 case ISD::INSERT_SUBVECTOR:
3677 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3678 break;
3679 case ISD::CONCAT_VECTORS:
3680 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3681 VectorValueType.isScalableVector() ||
3682 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3683 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3684 else
3685 Results.push_back(Elt: ExpandConcatVectors(Node));
3686 break;
3687 case ISD::SCALAR_TO_VECTOR:
3688 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3689 break;
3690 case ISD::INSERT_VECTOR_ELT:
3691 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3692 break;
3693 case ISD::VECTOR_SHUFFLE: {
3694 SmallVector<int, 32> NewMask;
3695 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3696
3697 EVT VT = Node->getValueType(ResNo: 0);
3698 EVT EltVT = VT.getVectorElementType();
3699 SDValue Op0 = Node->getOperand(Num: 0);
3700 SDValue Op1 = Node->getOperand(Num: 1);
3701 if (!TLI.isTypeLegal(VT: EltVT)) {
3702 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3703
3704 // BUILD_VECTOR operands are allowed to be wider than the element type.
3705 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3706 // it.
3707 if (NewEltVT.bitsLT(VT: EltVT)) {
3708 // Convert shuffle node.
3709 // If original node was v4i64 and the new EltVT is i32,
3710 // cast operands to v8i32 and re-build the mask.
3711
3712 // Calculate new VT, the size of the new VT should be equal to original.
3713 EVT NewVT =
3714 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3715 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3716 assert(NewVT.bitsEq(VT));
3717
3718 // cast operands to new VT
3719 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3720 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3721
3722 // Convert the shuffle mask
3723 unsigned int factor =
3724 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3725
3726 // EltVT gets smaller
3727 assert(factor > 0);
3728
3729 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3730 if (Mask[i] < 0) {
3731 for (unsigned fi = 0; fi < factor; ++fi)
3732 NewMask.push_back(Elt: Mask[i]);
3733 }
3734 else {
3735 for (unsigned fi = 0; fi < factor; ++fi)
3736 NewMask.push_back(Elt: Mask[i]*factor+fi);
3737 }
3738 }
3739 Mask = NewMask;
3740 VT = NewVT;
3741 }
3742 EltVT = NewEltVT;
3743 }
3744 unsigned NumElems = VT.getVectorNumElements();
3745 SmallVector<SDValue, 16> Ops;
3746 for (unsigned i = 0; i != NumElems; ++i) {
3747 if (Mask[i] < 0) {
3748 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3749 continue;
3750 }
3751 unsigned Idx = Mask[i];
3752 if (Idx < NumElems)
3753 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3754 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3755 else
3756 Ops.push_back(
3757 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3758 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3759 }
3760
3761 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3762 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3763 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3764 Results.push_back(Elt: Tmp1);
3765 break;
3766 }
3767 case ISD::VECTOR_SPLICE_LEFT:
3768 case ISD::VECTOR_SPLICE_RIGHT: {
3769 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3770 break;
3771 }
3772 case ISD::VECTOR_DEINTERLEAVE: {
3773 unsigned Factor = Node->getNumOperands();
3774 if (Factor <= 2 || Factor % 2 != 0)
3775 break;
3776 SmallVector<SDValue, 8> Ops(Node->ops());
3777 EVT VecVT = Node->getValueType(ResNo: 0);
3778 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3779 // Deinterleave at Factor/2 so each result contains two factors interleaved:
3780 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
3781 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3782 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
3783 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3784 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
3785 Results.resize(N: Factor);
3786 // Deinterleave the 2 factors out:
3787 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
3788 for (unsigned I = 0; I < Factor / 2; I++) {
3789 SDValue Deinterleave =
3790 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3791 Ops: {L.getValue(R: I), R.getValue(R: I)});
3792 Results[I] = Deinterleave.getValue(R: 0);
3793 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
3794 }
3795 break;
3796 }
3797 case ISD::VECTOR_INTERLEAVE: {
3798 unsigned Factor = Node->getNumOperands();
3799 if (Factor <= 2 || Factor % 2 != 0)
3800 break;
3801 EVT VecVT = Node->getValueType(ResNo: 0);
3802 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3803 SmallVector<SDValue, 8> LOps, ROps;
3804 // Interleave so we have 2 factors per result:
3805 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
3806 for (unsigned I = 0; I < Factor / 2; I++) {
3807 SDValue Interleave =
3808 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3809 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
3810 LOps.push_back(Elt: Interleave.getValue(R: 0));
3811 ROps.push_back(Elt: Interleave.getValue(R: 1));
3812 }
3813 // Interleave at Factor/2:
3814 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
3815 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: LOps);
3816 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: ROps);
3817 for (unsigned I = 0; I < Factor / 2; I++)
3818 Results.push_back(Elt: L.getValue(R: I));
3819 for (unsigned I = 0; I < Factor / 2; I++)
3820 Results.push_back(Elt: R.getValue(R: I));
3821 break;
3822 }
3823 case ISD::EXTRACT_ELEMENT: {
3824 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3825 if (Node->getConstantOperandVal(Num: 1)) {
3826 // 1 -> Hi
3827 Tmp1 = DAG.getNode(
3828 Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3829 N2: DAG.getShiftAmountConstant(Val: OpTy.getSizeInBits() / 2, VT: OpTy, DL: dl));
3830 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3831 } else {
3832 // 0 -> Lo
3833 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3834 Operand: Node->getOperand(Num: 0));
3835 }
3836 Results.push_back(Elt: Tmp1);
3837 break;
3838 }
3839 case ISD::STACKADDRESS:
3840 case ISD::STACKSAVE:
3841 // Expand to CopyFromReg if the target set
3842 // StackPointerRegisterToSaveRestore.
3843 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3844 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3845 VT: Node->getValueType(ResNo: 0)));
3846 Results.push_back(Elt: Results[0].getValue(R: 1));
3847 } else {
3848 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3849 Results.push_back(Elt: Node->getOperand(Num: 0));
3850
3851 StringRef IntrinsicName = Node->getOpcode() == ISD::STACKADDRESS
3852 ? "llvm.stackaddress"
3853 : "llvm.stacksave";
3854 DAG.getContext()->diagnose(DI: DiagnosticInfoLegalizationFailure(
3855 Twine(IntrinsicName) + " is not supported on this target.",
3856 DAG.getMachineFunction().getFunction(), dl.getDebugLoc()));
3857 }
3858 break;
3859 case ISD::STACKRESTORE:
3860 // Expand to CopyToReg if the target set
3861 // StackPointerRegisterToSaveRestore.
3862 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3863 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3864 N: Node->getOperand(Num: 1)));
3865 } else {
3866 Results.push_back(Elt: Node->getOperand(Num: 0));
3867 }
3868 break;
3869 case ISD::GET_DYNAMIC_AREA_OFFSET:
3870 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3871 Results.push_back(Elt: Results[0].getValue(R: 0));
3872 break;
3873 case ISD::FCOPYSIGN:
3874 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3875 break;
3876 case ISD::FNEG:
3877 Results.push_back(Elt: ExpandFNEG(Node));
3878 break;
3879 case ISD::FABS:
3880 Results.push_back(Elt: ExpandFABS(Node));
3881 break;
3882 case ISD::IS_FPCLASS: {
3883 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3884 if (SDValue Expanded =
3885 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3886 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3887 Results.push_back(Elt: Expanded);
3888 break;
3889 }
3890 case ISD::SMIN:
3891 case ISD::SMAX:
3892 case ISD::UMIN:
3893 case ISD::UMAX: {
3894 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3895 ISD::CondCode Pred;
3896 switch (Node->getOpcode()) {
3897 default: llvm_unreachable("How did we get here?");
3898 case ISD::SMAX: Pred = ISD::SETGT; break;
3899 case ISD::SMIN: Pred = ISD::SETLT; break;
3900 case ISD::UMAX: Pred = ISD::SETUGT; break;
3901 case ISD::UMIN: Pred = ISD::SETULT; break;
3902 }
3903 Tmp1 = Node->getOperand(Num: 0);
3904 Tmp2 = Node->getOperand(Num: 1);
3905 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3906 Results.push_back(Elt: Tmp1);
3907 break;
3908 }
3909 case ISD::FMINNUM:
3910 case ISD::FMAXNUM: {
3911 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3912 Results.push_back(Elt: Expanded);
3913 break;
3914 }
3915 case ISD::FMINIMUM:
3916 case ISD::FMAXIMUM: {
3917 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
3918 Results.push_back(Elt: Expanded);
3919 break;
3920 }
3921 case ISD::FMINIMUMNUM:
3922 case ISD::FMAXIMUMNUM: {
3923 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
3924 break;
3925 }
3926 case ISD::FSIN:
3927 case ISD::FCOS: {
3928 EVT VT = Node->getValueType(ResNo: 0);
3929 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3930 // fcos which share the same operand and both are used.
3931 if ((TLI.isOperationLegal(Op: ISD::FSINCOS, VT) ||
3932 isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls())) &&
3933 useSinCos(Node)) {
3934 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3935 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3936 if (Node->getOpcode() == ISD::FCOS)
3937 Tmp1 = Tmp1.getValue(R: 1);
3938 Results.push_back(Elt: Tmp1);
3939 }
3940 break;
3941 }
3942 case ISD::FLDEXP:
3943 case ISD::STRICT_FLDEXP: {
3944 EVT VT = Node->getValueType(ResNo: 0);
3945 RTLIB::Libcall LC = RTLIB::getLDEXP(VT);
3946 // Use the LibCall instead, it is very likely faster
3947 // FIXME: Use separate LibCall action.
3948 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3949 break;
3950
3951 if (SDValue Expanded = expandLdexp(Node)) {
3952 Results.push_back(Elt: Expanded);
3953 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3954 Results.push_back(Elt: Expanded.getValue(R: 1));
3955 }
3956
3957 break;
3958 }
3959 case ISD::FFREXP: {
3960 RTLIB::Libcall LC = RTLIB::getFREXP(VT: Node->getValueType(ResNo: 0));
3961 // Use the LibCall instead, it is very likely faster
3962 // FIXME: Use separate LibCall action.
3963 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3964 break;
3965
3966 if (SDValue Expanded = expandFrexp(Node)) {
3967 Results.push_back(Elt: Expanded);
3968 Results.push_back(Elt: Expanded.getValue(R: 1));
3969 }
3970 break;
3971 }
3972 case ISD::FMODF: {
3973 RTLIB::Libcall LC = RTLIB::getMODF(VT: Node->getValueType(ResNo: 0));
3974 // Use the LibCall instead, it is very likely faster
3975 // FIXME: Use separate LibCall action.
3976 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3977 break;
3978
3979 if (SDValue Expanded = expandModf(Node)) {
3980 Results.push_back(Elt: Expanded);
3981 Results.push_back(Elt: Expanded.getValue(R: 1));
3982 }
3983 break;
3984 }
3985 case ISD::FSINCOS: {
3986 if (isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls()))
3987 break;
3988 EVT VT = Node->getValueType(ResNo: 0);
3989 SDValue Op = Node->getOperand(Num: 0);
3990 SDNodeFlags Flags = Node->getFlags();
3991 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
3992 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
3993 Results.append(IL: {Tmp1, Tmp2});
3994 break;
3995 }
3996 case ISD::FMAD:
3997 llvm_unreachable("Illegal fmad should never be formed");
3998
3999 case ISD::FP16_TO_FP:
4000 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4001 // We can extend to types bigger than f32 in two steps without changing
4002 // the result. Since "f16 -> f32" is much more commonly available, give
4003 // CodeGen the option of emitting that before resorting to a libcall.
4004 SDValue Res =
4005 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
4006 Results.push_back(
4007 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
4008 }
4009 break;
4010 case ISD::STRICT_BF16_TO_FP:
4011 case ISD::STRICT_FP16_TO_FP:
4012 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4013 // We can extend to types bigger than f32 in two steps without changing
4014 // the result. Since "f16 -> f32" is much more commonly available, give
4015 // CodeGen the option of emitting that before resorting to a libcall.
4016 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
4017 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
4018 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
4019 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
4020 Ops: {Res.getValue(R: 1), Res});
4021 Results.push_back(Elt: Res);
4022 Results.push_back(Elt: Res.getValue(R: 1));
4023 }
4024 break;
4025 case ISD::FP_TO_FP16:
4026 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
4027 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
4028 SDValue Op = Node->getOperand(Num: 0);
4029 MVT SVT = Op.getSimpleValueType();
4030 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
4031 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
4032 // Under fastmath, we can expand this node into a fround followed by
4033 // a float-half conversion.
4034 SDValue FloatVal =
4035 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
4036 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
4037 Results.push_back(
4038 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
4039 }
4040 }
4041 break;
4042 case ISD::ConstantFP: {
4043 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
4044 // Check to see if this FP immediate is already legal.
4045 // If this is a legal constant, turn it into a TargetConstantFP node.
4046 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
4047 ForCodeSize: DAG.shouldOptForSize()))
4048 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
4049 break;
4050 }
4051 case ISD::Constant: {
4052 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
4053 Results.push_back(Elt: ExpandConstant(CP));
4054 break;
4055 }
4056 case ISD::FSUB: {
4057 EVT VT = Node->getValueType(ResNo: 0);
4058 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
4059 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
4060 const SDNodeFlags Flags = Node->getFlags();
4061 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
4062 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
4063 Results.push_back(Elt: Tmp1);
4064 }
4065 break;
4066 }
4067 case ISD::SUB: {
4068 EVT VT = Node->getValueType(ResNo: 0);
4069 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
4070 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
4071 "Don't know how to expand this subtraction!");
4072 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
4073 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
4074 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
4075 break;
4076 }
4077 case ISD::UREM:
4078 case ISD::SREM:
4079 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
4080 Results.push_back(Elt: Tmp1);
4081 break;
4082 case ISD::UDIV:
4083 case ISD::SDIV: {
4084 bool isSigned = Node->getOpcode() == ISD::SDIV;
4085 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4086 EVT VT = Node->getValueType(ResNo: 0);
4087 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
4088 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4089 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4090 N2: Node->getOperand(Num: 1));
4091 Results.push_back(Elt: Tmp1);
4092 }
4093 break;
4094 }
4095 case ISD::MULHU:
4096 case ISD::MULHS: {
4097 unsigned ExpandOpcode =
4098 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
4099 EVT VT = Node->getValueType(ResNo: 0);
4100 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4101
4102 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4103 N2: Node->getOperand(Num: 1));
4104 Results.push_back(Elt: Tmp1.getValue(R: 1));
4105 break;
4106 }
4107 case ISD::UMUL_LOHI:
4108 case ISD::SMUL_LOHI: {
4109 SDValue LHS = Node->getOperand(Num: 0);
4110 SDValue RHS = Node->getOperand(Num: 1);
4111 EVT VT = LHS.getValueType();
4112 bool IsSigned = Node->getOpcode() == ISD::SMUL_LOHI;
4113 unsigned MULHOpcode = IsSigned ? ISD::MULHS : ISD::MULHU;
4114
4115 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
4116 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
4117 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
4118 break;
4119 }
4120
4121 SmallVector<SDValue, 4> Halves;
4122 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4123 if (TLI.isTypeLegal(VT: HalfType) &&
4124 TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
4125 HiLoVT: HalfType, DAG,
4126 Kind: TargetLowering::MulExpansionKind::Always)) {
4127 for (unsigned i = 0; i < 2; ++i) {
4128 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
4129 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
4130 SDValue Shift =
4131 DAG.getShiftAmountConstant(Val: HalfType.getScalarSizeInBits(), VT, DL: dl);
4132 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4133 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4134 }
4135 break;
4136 }
4137
4138 SDValue Lo, Hi;
4139 TLI.forceExpandWideMUL(DAG, dl, Signed: IsSigned, LHS, RHS, Lo, Hi);
4140 Results.push_back(Elt: Lo);
4141 Results.push_back(Elt: Hi);
4142 break;
4143 }
4144 case ISD::MUL: {
4145 EVT VT = Node->getValueType(ResNo: 0);
4146 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4147 // See if multiply or divide can be lowered using two-result operations.
4148 // We just need the low half of the multiply; try both the signed
4149 // and unsigned forms. If the target supports both SMUL_LOHI and
4150 // UMUL_LOHI, form a preference by checking which forms of plain
4151 // MULH it supports.
4152 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
4153 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
4154 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
4155 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
4156 unsigned OpToUse = 0;
4157 if (HasSMUL_LOHI && !HasMULHS) {
4158 OpToUse = ISD::SMUL_LOHI;
4159 } else if (HasUMUL_LOHI && !HasMULHU) {
4160 OpToUse = ISD::UMUL_LOHI;
4161 } else if (HasSMUL_LOHI) {
4162 OpToUse = ISD::SMUL_LOHI;
4163 } else if (HasUMUL_LOHI) {
4164 OpToUse = ISD::UMUL_LOHI;
4165 }
4166 if (OpToUse) {
4167 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4168 N2: Node->getOperand(Num: 1)));
4169 break;
4170 }
4171
4172 SDValue Lo, Hi;
4173 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4174 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
4175 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
4176 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
4177 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
4178 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
4179 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4180 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
4181 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
4182 SDValue Shift =
4183 DAG.getShiftAmountConstant(Val: HalfType.getSizeInBits(), VT, DL: dl);
4184 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4185 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4186 }
4187 break;
4188 }
4189 case ISD::FSHL:
4190 case ISD::FSHR:
4191 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
4192 Results.push_back(Elt: Expanded);
4193 break;
4194 case ISD::ROTL:
4195 case ISD::ROTR:
4196 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
4197 Results.push_back(Elt: Expanded);
4198 break;
4199 case ISD::CLMUL:
4200 case ISD::CLMULR:
4201 case ISD::CLMULH:
4202 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG))
4203 Results.push_back(Elt: Expanded);
4204 break;
4205 case ISD::PEXT:
4206 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
4207 break;
4208 case ISD::PDEP:
4209 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
4210 break;
4211 case ISD::SADDSAT:
4212 case ISD::UADDSAT:
4213 case ISD::SSUBSAT:
4214 case ISD::USUBSAT:
4215 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
4216 break;
4217 case ISD::SCMP:
4218 case ISD::UCMP:
4219 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
4220 break;
4221 case ISD::SSHLSAT:
4222 case ISD::USHLSAT:
4223 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
4224 break;
4225 case ISD::SMULFIX:
4226 case ISD::SMULFIXSAT:
4227 case ISD::UMULFIX:
4228 case ISD::UMULFIXSAT:
4229 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
4230 break;
4231 case ISD::SDIVFIX:
4232 case ISD::SDIVFIXSAT:
4233 case ISD::UDIVFIX:
4234 case ISD::UDIVFIXSAT:
4235 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
4236 LHS: Node->getOperand(Num: 0),
4237 RHS: Node->getOperand(Num: 1),
4238 Scale: Node->getConstantOperandVal(Num: 2),
4239 DAG)) {
4240 Results.push_back(Elt: V);
4241 break;
4242 }
4243 // FIXME: We might want to retry here with a wider type if we fail, if that
4244 // type is legal.
4245 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4246 // <= 128 (which is the case for all of the default Embedded-C types),
4247 // we will only get here with types and scales that we could always expand
4248 // if we were allowed to generate libcalls to division functions of illegal
4249 // type. But we cannot do that.
4250 llvm_unreachable("Cannot expand DIVFIX!");
4251 case ISD::UADDO_CARRY:
4252 case ISD::USUBO_CARRY: {
4253 SDValue LHS = Node->getOperand(Num: 0);
4254 SDValue RHS = Node->getOperand(Num: 1);
4255 SDValue Carry = Node->getOperand(Num: 2);
4256
4257 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4258
4259 // Initial add of the 2 operands.
4260 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4261 EVT VT = LHS.getValueType();
4262 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4263
4264 // Initial check for overflow.
4265 EVT CarryType = Node->getValueType(ResNo: 1);
4266 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4267 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4268 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4269
4270 // Add of the sum and the carry.
4271 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4272 SDValue CarryExt =
4273 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4274 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4275
4276 // Second check for overflow. If we are adding, we can only overflow if the
4277 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4278 // If we are subtracting, we can only overflow if the initial sum is 0 and
4279 // the carry is set, resulting in a new sum of all 1s.
4280 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4281 SDValue Overflow2 =
4282 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4283 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4284 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4285 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4286
4287 SDValue ResultCarry =
4288 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4289
4290 Results.push_back(Elt: Sum2);
4291 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4292 break;
4293 }
4294 case ISD::SADDO:
4295 case ISD::SSUBO: {
4296 SDValue Result, Overflow;
4297 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4298 Results.push_back(Elt: Result);
4299 Results.push_back(Elt: Overflow);
4300 break;
4301 }
4302 case ISD::UADDO:
4303 case ISD::USUBO: {
4304 SDValue Result, Overflow;
4305 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4306 Results.push_back(Elt: Result);
4307 Results.push_back(Elt: Overflow);
4308 break;
4309 }
4310 case ISD::UMULO:
4311 case ISD::SMULO: {
4312 SDValue Result, Overflow;
4313 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4314 Results.push_back(Elt: Result);
4315 Results.push_back(Elt: Overflow);
4316 }
4317 break;
4318 }
4319 case ISD::BUILD_PAIR: {
4320 EVT PairTy = Node->getValueType(ResNo: 0);
4321 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4322 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4323 Tmp2 = DAG.getNode(
4324 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4325 N2: DAG.getShiftAmountConstant(Val: PairTy.getSizeInBits() / 2, VT: PairTy, DL: dl));
4326 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4327 break;
4328 }
4329 case ISD::SELECT:
4330 Tmp1 = Node->getOperand(Num: 0);
4331 Tmp2 = Node->getOperand(Num: 1);
4332 Tmp3 = Node->getOperand(Num: 2);
4333 if (Tmp1.getOpcode() == ISD::SETCC) {
4334 Tmp1 = DAG.getSelectCC(
4335 DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), True: Tmp2, False: Tmp3,
4336 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get(), Flags: Node->getFlags());
4337 } else {
4338 Tmp1 =
4339 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4340 True: Tmp2, False: Tmp3, Cond: ISD::SETNE, Flags: Node->getFlags());
4341 }
4342 Results.push_back(Elt: Tmp1);
4343 break;
4344 case ISD::BR_JT: {
4345 SDValue Chain = Node->getOperand(Num: 0);
4346 SDValue Table = Node->getOperand(Num: 1);
4347 SDValue Index = Node->getOperand(Num: 2);
4348 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4349
4350 const DataLayout &TD = DAG.getDataLayout();
4351 EVT PTy = TLI.getPointerTy(DL: TD);
4352
4353 unsigned EntrySize =
4354 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4355
4356 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4357 // This transformation needs to be done here since otherwise the MIPS
4358 // backend will end up emitting a three instruction multiply sequence
4359 // instead of a single shift and MSP430 will call a runtime function.
4360 if (llvm::isPowerOf2_32(Value: EntrySize))
4361 Index = DAG.getNode(
4362 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4363 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4364 else
4365 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4366 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4367 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4368
4369 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4370 SDValue LD = DAG.getExtLoad(
4371 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4372 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4373 Addr = LD;
4374 if (TLI.isJumpTableRelative()) {
4375 // For PIC, the sequence is:
4376 // BRIND(RelocBase + load(Jumptable + index))
4377 // RelocBase can be JumpTable, GOT or some sort of global base.
4378 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4379 Offset: Addr, DL: dl);
4380 }
4381
4382 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4383 Results.push_back(Elt: Tmp1);
4384 break;
4385 }
4386 case ISD::BRCOND:
4387 // Expand brcond's setcc into its constituent parts and create a BR_CC
4388 // Node.
4389 Tmp1 = Node->getOperand(Num: 0);
4390 Tmp2 = Node->getOperand(Num: 1);
4391 if (Tmp2.getOpcode() == ISD::SETCC &&
4392 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4393 VT: Tmp2.getOperand(i: 0).getValueType())) {
4394 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4395 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4396 N5: Node->getOperand(Num: 2));
4397 } else {
4398 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4399 if (Tmp2.isUndef() ||
4400 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4401 Tmp3 = Tmp2;
4402 else
4403 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4404 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4405 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4406 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4407 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4408 N5: Node->getOperand(Num: 2));
4409 }
4410 Results.push_back(Elt: Tmp1);
4411 break;
4412 case ISD::SETCC:
4413 case ISD::STRICT_FSETCC:
4414 case ISD::STRICT_FSETCCS: {
4415 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4416 Node->getOpcode() == ISD::STRICT_FSETCCS;
4417 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4418 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4419 unsigned Offset = IsStrict ? 1 : 0;
4420 Tmp1 = Node->getOperand(Num: 0 + Offset);
4421 Tmp2 = Node->getOperand(Num: 1 + Offset);
4422 Tmp3 = Node->getOperand(Num: 2 + Offset);
4423 bool Legalized =
4424 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3,
4425 NeedInvert, dl, Chain, IsSignaling);
4426
4427 if (Legalized) {
4428 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4429 // condition code, create a new SETCC node.
4430 if (Tmp3.getNode()) {
4431 if (IsStrict) {
4432 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4433 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4434 Chain = Tmp1.getValue(R: 1);
4435 } else {
4436 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4437 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4438 }
4439 }
4440
4441 // If we expanded the SETCC by inverting the condition code, then wrap
4442 // the existing SETCC in a NOT to restore the intended condition.
4443 if (NeedInvert) {
4444 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4445 }
4446
4447 Results.push_back(Elt: Tmp1);
4448 if (IsStrict)
4449 Results.push_back(Elt: Chain);
4450
4451 break;
4452 }
4453
4454 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4455 // understand if this code is useful for strict nodes.
4456 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4457
4458 // Otherwise, SETCC for the given comparison type must be completely
4459 // illegal; expand it into a SELECT_CC.
4460 EVT VT = Node->getValueType(ResNo: 0);
4461 EVT Tmp1VT = Tmp1.getValueType();
4462 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4463 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4464 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3,
4465 Flags: Node->getFlags());
4466 Results.push_back(Elt: Tmp1);
4467 break;
4468 }
4469 case ISD::SELECT_CC: {
4470 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4471 Tmp1 = Node->getOperand(Num: 0); // LHS
4472 Tmp2 = Node->getOperand(Num: 1); // RHS
4473 Tmp3 = Node->getOperand(Num: 2); // True
4474 Tmp4 = Node->getOperand(Num: 3); // False
4475 EVT VT = Node->getValueType(ResNo: 0);
4476 SDValue Chain;
4477 SDValue CC = Node->getOperand(Num: 4);
4478 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4479
4480 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4481 // If the condition code is legal, then we need to expand this
4482 // node using SETCC and SELECT.
4483 EVT CmpVT = Tmp1.getValueType();
4484 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4485 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4486 "expanded.");
4487 EVT CCVT = getSetCCResultType(VT: CmpVT);
4488 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4489 Results.push_back(
4490 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4491 break;
4492 }
4493
4494 // SELECT_CC is legal, so the condition code must not be.
4495 bool Legalized = false;
4496 // Try to legalize by inverting the condition. This is for targets that
4497 // might support an ordered version of a condition, but not the unordered
4498 // version (or vice versa).
4499 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4500 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4501 // Use the new condition code and swap true and false
4502 Legalized = true;
4503 Tmp1 =
4504 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC, Flags: Node->getFlags());
4505 } else {
4506 // If The inverse is not legal, then try to swap the arguments using
4507 // the inverse condition code.
4508 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4509 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4510 // The swapped inverse condition is legal, so swap true and false,
4511 // lhs and rhs.
4512 Legalized = true;
4513 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC,
4514 Flags: Node->getFlags());
4515 }
4516 }
4517
4518 if (!Legalized) {
4519 Legalized = TLI.LegalizeSetCCCondCode(
4520 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4521 NeedInvert, dl, Chain);
4522
4523 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4524
4525 // If we expanded the SETCC by inverting the condition code, then swap
4526 // the True/False operands to match.
4527 if (NeedInvert)
4528 std::swap(a&: Tmp3, b&: Tmp4);
4529
4530 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4531 // condition code, create a new SELECT_CC node.
4532 if (CC.getNode()) {
4533 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4534 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4535 } else {
4536 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4537 CC = DAG.getCondCode(Cond: ISD::SETNE);
4538 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4539 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4540 }
4541 }
4542 Results.push_back(Elt: Tmp1);
4543 break;
4544 }
4545 case ISD::BR_CC: {
4546 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4547 SDValue Chain;
4548 Tmp1 = Node->getOperand(Num: 0); // Chain
4549 Tmp2 = Node->getOperand(Num: 2); // LHS
4550 Tmp3 = Node->getOperand(Num: 3); // RHS
4551 Tmp4 = Node->getOperand(Num: 1); // CC
4552
4553 bool Legalized =
4554 TLI.LegalizeSetCCCondCode(DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()),
4555 LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4, NeedInvert, dl, Chain);
4556 (void)Legalized;
4557 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4558
4559 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4560 // node.
4561 if (Tmp4.getNode()) {
4562 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4563
4564 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4565 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4566 } else {
4567 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4568 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4569 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4570 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4571 }
4572 Results.push_back(Elt: Tmp1);
4573 break;
4574 }
4575 case ISD::BUILD_VECTOR:
4576 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4577 break;
4578 case ISD::SPLAT_VECTOR:
4579 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4580 break;
4581 case ISD::SRA:
4582 case ISD::SRL:
4583 case ISD::SHL: {
4584 // Scalarize vector SRA/SRL/SHL.
4585 EVT VT = Node->getValueType(ResNo: 0);
4586 assert(VT.isVector() && "Unable to legalize non-vector shift");
4587 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4588 unsigned NumElem = VT.getVectorNumElements();
4589
4590 SmallVector<SDValue, 8> Scalars;
4591 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4592 SDValue Ex =
4593 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4594 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4595 SDValue Sh =
4596 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4597 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4598 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4599 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4600 }
4601
4602 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4603 Results.push_back(Elt: Result);
4604 break;
4605 }
4606 case ISD::VECREDUCE_FADD:
4607 case ISD::VECREDUCE_FMUL:
4608 case ISD::VECREDUCE_ADD:
4609 case ISD::VECREDUCE_MUL:
4610 case ISD::VECREDUCE_AND:
4611 case ISD::VECREDUCE_OR:
4612 case ISD::VECREDUCE_XOR:
4613 case ISD::VECREDUCE_SMAX:
4614 case ISD::VECREDUCE_SMIN:
4615 case ISD::VECREDUCE_UMAX:
4616 case ISD::VECREDUCE_UMIN:
4617 case ISD::VECREDUCE_FMAX:
4618 case ISD::VECREDUCE_FMIN:
4619 case ISD::VECREDUCE_FMAXIMUM:
4620 case ISD::VECREDUCE_FMINIMUM:
4621 case ISD::VECREDUCE_FMAXIMUMNUM:
4622 case ISD::VECREDUCE_FMINIMUMNUM:
4623 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4624 break;
4625 case ISD::VP_CTTZ_ELTS:
4626 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4627 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4628 break;
4629 case ISD::CLEAR_CACHE:
4630 // The default expansion of llvm.clear_cache is simply a no-op for those
4631 // targets where it is not needed.
4632 Results.push_back(Elt: Node->getOperand(Num: 0));
4633 break;
4634 case ISD::LRINT:
4635 case ISD::LLRINT: {
4636 SDValue Arg = Node->getOperand(Num: 0);
4637 EVT ArgVT = Arg.getValueType();
4638 EVT ResVT = Node->getValueType(ResNo: 0);
4639 SDLoc DL(Node);
4640 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL, VT: ArgVT, Operand: Arg);
4641 SDValue ConvertNode = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: ResVT, Operand: RoundNode);
4642 // Non-deterministic results are equivalent to freeze poison.
4643 Results.push_back(Elt: DAG.getFreeze(V: ConvertNode));
4644 break;
4645 }
4646 case ISD::ADDRSPACECAST:
4647 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4648 break;
4649 case ISD::GLOBAL_OFFSET_TABLE:
4650 case ISD::GlobalAddress:
4651 case ISD::GlobalTLSAddress:
4652 case ISD::ExternalSymbol:
4653 case ISD::ConstantPool:
4654 case ISD::JumpTable:
4655 case ISD::INTRINSIC_W_CHAIN:
4656 case ISD::INTRINSIC_WO_CHAIN:
4657 case ISD::INTRINSIC_VOID:
4658 // FIXME: Custom lowering for these operations shouldn't return null!
4659 // Return true so that we don't call ConvertNodeToLibcall which also won't
4660 // do anything.
4661 return true;
4662 }
4663
4664 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4665 // FIXME: We were asked to expand a strict floating-point operation,
4666 // but there is currently no expansion implemented that would preserve
4667 // the "strict" properties. For now, we just fall back to the non-strict
4668 // version if that is legal on the target. The actual mutation of the
4669 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4670 switch (Node->getOpcode()) {
4671 default:
4672 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4673 VT: Node->getValueType(ResNo: 0))
4674 == TargetLowering::Legal)
4675 return true;
4676 break;
4677 case ISD::STRICT_FSUB: {
4678 if (TLI.getStrictFPOperationAction(
4679 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4680 return true;
4681 if (TLI.getStrictFPOperationAction(
4682 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4683 break;
4684
4685 EVT VT = Node->getValueType(ResNo: 0);
4686 const SDNodeFlags Flags = Node->getFlags();
4687 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4688 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4689 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4690 Flags);
4691
4692 Results.push_back(Elt: Fadd);
4693 Results.push_back(Elt: Fadd.getValue(R: 1));
4694 break;
4695 }
4696 case ISD::STRICT_SINT_TO_FP:
4697 case ISD::STRICT_UINT_TO_FP:
4698 case ISD::STRICT_LRINT:
4699 case ISD::STRICT_LLRINT:
4700 case ISD::STRICT_LROUND:
4701 case ISD::STRICT_LLROUND:
4702 // These are registered by the operand type instead of the value
4703 // type. Reflect that here.
4704 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4705 VT: Node->getOperand(Num: 1).getValueType())
4706 == TargetLowering::Legal)
4707 return true;
4708 break;
4709 }
4710 }
4711
4712 // Replace the original node with the legalized result.
4713 if (Results.empty()) {
4714 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4715 return false;
4716 }
4717
4718 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4719 ReplaceNode(Old: Node, New: Results.data());
4720 return true;
4721}
4722
4723/// Return if we can use the FAST_* variant of a math libcall for the node.
4724/// FIXME: This is just guessing, we probably should have unique specific sets
4725/// flags required per libcall.
4726static bool canUseFastMathLibcall(const SDNode *Node) {
4727 // FIXME: Probably should define fast to respect nan/inf and only be
4728 // approximate functions.
4729
4730 SDNodeFlags Flags = Node->getFlags();
4731 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4732 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4733}
4734
4735void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4736 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4737 SmallVector<SDValue, 8> Results;
4738 SDLoc dl(Node);
4739 TargetLowering::MakeLibCallOptions CallOptions;
4740 CallOptions.IsPostTypeLegalization = true;
4741 // FIXME: Check flags on the node to see if we can use a finite call.
4742 unsigned Opc = Node->getOpcode();
4743 switch (Opc) {
4744 case ISD::ATOMIC_FENCE: {
4745 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4746 // FIXME: handle "fence singlethread" more efficiently.
4747 TargetLowering::ArgListTy Args;
4748
4749 TargetLowering::CallLoweringInfo CLI(DAG);
4750 CLI.setDebugLoc(dl)
4751 .setChain(Node->getOperand(Num: 0))
4752 .setLibCallee(
4753 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4754 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4755 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4756 ArgsList: std::move(Args));
4757
4758 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4759
4760 Results.push_back(Elt: CallResult.second);
4761 break;
4762 }
4763 // By default, atomic intrinsics are marked Legal and lowered. Targets
4764 // which don't support them directly, however, may want libcalls, in which
4765 // case they mark them Expand, and we get here.
4766 case ISD::ATOMIC_SWAP:
4767 case ISD::ATOMIC_LOAD_ADD:
4768 case ISD::ATOMIC_LOAD_SUB:
4769 case ISD::ATOMIC_LOAD_AND:
4770 case ISD::ATOMIC_LOAD_CLR:
4771 case ISD::ATOMIC_LOAD_OR:
4772 case ISD::ATOMIC_LOAD_XOR:
4773 case ISD::ATOMIC_LOAD_NAND:
4774 case ISD::ATOMIC_LOAD_MIN:
4775 case ISD::ATOMIC_LOAD_MAX:
4776 case ISD::ATOMIC_LOAD_UMIN:
4777 case ISD::ATOMIC_LOAD_UMAX:
4778 case ISD::ATOMIC_CMP_SWAP: {
4779 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4780 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4781 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4782 EVT RetVT = Node->getValueType(ResNo: 0);
4783 SmallVector<SDValue, 4> Ops;
4784 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported) {
4785 // If outline atomic available, prepare its arguments and expand.
4786 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4787 Ops.push_back(Elt: Node->getOperand(Num: 1));
4788
4789 } else {
4790 LC = RTLIB::getSYNC(Opc, VT);
4791 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4792 "Unexpected atomic op or value type!");
4793 // Arguments for expansion to sync libcall
4794 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4795 }
4796 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4797 Ops, CallOptions,
4798 dl: SDLoc(Node),
4799 Chain: Node->getOperand(Num: 0));
4800 Results.push_back(Elt: Tmp.first);
4801 Results.push_back(Elt: Tmp.second);
4802 break;
4803 }
4804 case ISD::TRAP: {
4805 // If this operation is not supported, lower it to 'abort()' call
4806 TargetLowering::ArgListTy Args;
4807 TargetLowering::CallLoweringInfo CLI(DAG);
4808 CLI.setDebugLoc(dl)
4809 .setChain(Node->getOperand(Num: 0))
4810 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4811 Target: DAG.getExternalSymbol(
4812 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4813 ArgsList: std::move(Args));
4814 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4815
4816 Results.push_back(Elt: CallResult.second);
4817 break;
4818 }
4819 case ISD::CLEAR_CACHE: {
4820 SDValue InputChain = Node->getOperand(Num: 0);
4821 SDValue StartVal = Node->getOperand(Num: 1);
4822 SDValue EndVal = Node->getOperand(Num: 2);
4823 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4824 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
4825 dl: SDLoc(Node), Chain: InputChain);
4826 Results.push_back(Elt: Tmp.second);
4827 break;
4828 }
4829 case ISD::FMINNUM:
4830 case ISD::STRICT_FMINNUM:
4831 ExpandFPLibCall(Node, LC: RTLIB::getFMIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4832 break;
4833 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4834 // libcall legalization for these nodes, but there is no default expasion for
4835 // these nodes either (see PR63267 for example).
4836 case ISD::FMAXNUM:
4837 case ISD::STRICT_FMAXNUM:
4838 ExpandFPLibCall(Node, LC: RTLIB::getFMAX(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4839 break;
4840 case ISD::FMINIMUMNUM:
4841 ExpandFPLibCall(Node, LC: RTLIB::getFMINIMUM_NUM(VT: Node->getSimpleValueType(ResNo: 0)),
4842 Results);
4843 break;
4844 case ISD::FMAXIMUMNUM:
4845 ExpandFPLibCall(Node, LC: RTLIB::getFMAXIMUM_NUM(VT: Node->getSimpleValueType(ResNo: 0)),
4846 Results);
4847 break;
4848 case ISD::FSQRT:
4849 case ISD::STRICT_FSQRT: {
4850 // FIXME: Probably should define fast to respect nan/inf and only be
4851 // approximate functions.
4852 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
4853 Call_F32: {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
4854 Call_F64: {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
4855 Call_F80: {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
4856 Call_F128: {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
4857 Call_PPCF128: {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
4858 Results);
4859 break;
4860 }
4861 case ISD::FCBRT:
4862 ExpandFPLibCall(Node, LC: RTLIB::getCBRT(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4863 break;
4864 case ISD::FSIN:
4865 case ISD::STRICT_FSIN:
4866 ExpandFPLibCall(Node, LC: RTLIB::getSIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4867 break;
4868 case ISD::FCOS:
4869 case ISD::STRICT_FCOS:
4870 ExpandFPLibCall(Node, LC: RTLIB::getCOS(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4871 break;
4872 case ISD::FTAN:
4873 case ISD::STRICT_FTAN:
4874 ExpandFPLibCall(Node, LC: RTLIB::getTAN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4875 break;
4876 case ISD::FASIN:
4877 case ISD::STRICT_FASIN:
4878 ExpandFPLibCall(Node, LC: RTLIB::getASIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4879 break;
4880 case ISD::FACOS:
4881 case ISD::STRICT_FACOS:
4882 ExpandFPLibCall(Node, LC: RTLIB::getACOS(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4883 break;
4884 case ISD::FATAN:
4885 case ISD::STRICT_FATAN:
4886 ExpandFPLibCall(Node, LC: RTLIB::getATAN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4887 break;
4888 case ISD::FATAN2:
4889 case ISD::STRICT_FATAN2:
4890 ExpandFPLibCall(Node, LC: RTLIB::getATAN2(VT: Node->getSimpleValueType(ResNo: 0)),
4891 Results);
4892 break;
4893 case ISD::FSINH:
4894 case ISD::STRICT_FSINH:
4895 ExpandFPLibCall(Node, LC: RTLIB::getSINH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4896 break;
4897 case ISD::FCOSH:
4898 case ISD::STRICT_FCOSH:
4899 ExpandFPLibCall(Node, LC: RTLIB::getCOSH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4900 break;
4901 case ISD::FTANH:
4902 case ISD::STRICT_FTANH:
4903 ExpandFPLibCall(Node, LC: RTLIB::getTANH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4904 break;
4905 case ISD::FSINCOS:
4906 case ISD::FSINCOSPI: {
4907 EVT VT = Node->getValueType(ResNo: 0);
4908
4909 if (Node->getOpcode() == ISD::FSINCOS) {
4910 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(VT);
4911 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
4912 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
4913 Results.push_back(Elt: Expanded);
4914 Results.push_back(Elt: Expanded.getValue(R: 1));
4915 break;
4916 }
4917 }
4918 }
4919
4920 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4921 ? RTLIB::getSINCOS(VT)
4922 : RTLIB::getSINCOSPI(VT);
4923 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);
4924 if (!Expanded) {
4925 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
4926 Node->getOperationName(G: &DAG));
4927 SDValue Poison = DAG.getPOISON(VT);
4928 Results.push_back(Elt: Poison);
4929 Results.push_back(Elt: Poison);
4930 }
4931
4932 break;
4933 }
4934 case ISD::FLOG:
4935 case ISD::STRICT_FLOG:
4936 ExpandFPLibCall(Node, LC: RTLIB::getLOG(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4937 break;
4938 case ISD::FLOG2:
4939 case ISD::STRICT_FLOG2:
4940 ExpandFPLibCall(Node, LC: RTLIB::getLOG2(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4941 break;
4942 case ISD::FLOG10:
4943 case ISD::STRICT_FLOG10:
4944 ExpandFPLibCall(Node, LC: RTLIB::getLOG10(VT: Node->getSimpleValueType(ResNo: 0)),
4945 Results);
4946 break;
4947 case ISD::FEXP:
4948 case ISD::STRICT_FEXP:
4949 ExpandFPLibCall(Node, LC: RTLIB::getEXP(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4950 break;
4951 case ISD::FEXP2:
4952 case ISD::STRICT_FEXP2:
4953 ExpandFPLibCall(Node, LC: RTLIB::getEXP2(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4954 break;
4955 case ISD::FEXP10:
4956 ExpandFPLibCall(Node, LC: RTLIB::getEXP10(VT: Node->getSimpleValueType(ResNo: 0)),
4957 Results);
4958 break;
4959 case ISD::FTRUNC:
4960 case ISD::STRICT_FTRUNC:
4961 ExpandFPLibCall(Node, LC: RTLIB::getTRUNC(VT: Node->getSimpleValueType(ResNo: 0)),
4962 Results);
4963 break;
4964 case ISD::FFLOOR:
4965 case ISD::STRICT_FFLOOR:
4966 ExpandFPLibCall(Node, LC: RTLIB::getFLOOR(VT: Node->getSimpleValueType(ResNo: 0)),
4967 Results);
4968 break;
4969 case ISD::FCEIL:
4970 case ISD::STRICT_FCEIL:
4971 ExpandFPLibCall(Node, LC: RTLIB::getCEIL(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4972 break;
4973 case ISD::FRINT:
4974 case ISD::STRICT_FRINT:
4975 ExpandFPLibCall(Node, LC: RTLIB::getRINT(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4976 break;
4977 case ISD::FNEARBYINT:
4978 case ISD::STRICT_FNEARBYINT:
4979 ExpandFPLibCall(Node, LC: RTLIB::getNEARBYINT(VT: Node->getSimpleValueType(ResNo: 0)),
4980 Results);
4981 break;
4982 case ISD::FROUND:
4983 case ISD::STRICT_FROUND:
4984 ExpandFPLibCall(Node, LC: RTLIB::getROUND(VT: Node->getSimpleValueType(ResNo: 0)),
4985 Results);
4986 break;
4987 case ISD::FROUNDEVEN:
4988 case ISD::STRICT_FROUNDEVEN:
4989 ExpandFPLibCall(Node, LC: RTLIB::getROUNDEVEN(VT: Node->getSimpleValueType(ResNo: 0)),
4990 Results);
4991 break;
4992 case ISD::FLDEXP:
4993 case ISD::STRICT_FLDEXP:
4994 ExpandFPLibCall(Node, LC: RTLIB::getLDEXP(VT: Node->getSimpleValueType(ResNo: 0)),
4995 Results);
4996 break;
4997 case ISD::FMODF:
4998 case ISD::FFREXP: {
4999 EVT VT = Node->getValueType(ResNo: 0);
5000 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
5001 : RTLIB::getFREXP(VT);
5002 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
5003 /*CallRetResNo=*/0);
5004 if (!Expanded) {
5005 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
5006 Node->getOperationName(G: &DAG));
5007 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
5008 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: I)));
5009 }
5010 break;
5011 }
5012 case ISD::FPOWI:
5013 case ISD::STRICT_FPOWI: {
5014 RTLIB::Libcall LC = RTLIB::getPOWI(VT: Node->getSimpleValueType(ResNo: 0));
5015 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
5016 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
5017 // Some targets don't have a powi libcall; use pow instead.
5018 if (Node->isStrictFPOpcode()) {
5019 SDValue Exponent =
5020 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
5021 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5022 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5023 SDValue FPOW =
5024 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
5025 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5026 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
5027 Results.push_back(Elt: FPOW);
5028 Results.push_back(Elt: FPOW.getValue(R: 1));
5029 } else {
5030 SDValue Exponent =
5031 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
5032 Operand: Node->getOperand(Num: 1));
5033 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
5034 VT: Node->getValueType(ResNo: 0),
5035 N1: Node->getOperand(Num: 0), N2: Exponent));
5036 }
5037 break;
5038 }
5039 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
5040 bool ExponentHasSizeOfInt =
5041 DAG.getLibInfo().getIntSize() ==
5042 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
5043 if (!ExponentHasSizeOfInt) {
5044 // If the exponent does not match with sizeof(int) a libcall to
5045 // RTLIB::POWI would use the wrong type for the argument.
5046 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
5047 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
5048 break;
5049 }
5050 ExpandFPLibCall(Node, LC, Results);
5051 break;
5052 }
5053 case ISD::FPOW:
5054 case ISD::STRICT_FPOW:
5055 ExpandFPLibCall(Node, LC: RTLIB::getPOW(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5056 break;
5057 case ISD::LROUND:
5058 case ISD::STRICT_LROUND:
5059 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
5060 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
5061 Call_F128: RTLIB::LROUND_F128,
5062 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
5063 break;
5064 case ISD::LLROUND:
5065 case ISD::STRICT_LLROUND:
5066 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
5067 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
5068 Call_F128: RTLIB::LLROUND_F128,
5069 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
5070 break;
5071 case ISD::LRINT:
5072 case ISD::STRICT_LRINT:
5073 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
5074 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
5075 Call_F128: RTLIB::LRINT_F128,
5076 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
5077 break;
5078 case ISD::LLRINT:
5079 case ISD::STRICT_LLRINT:
5080 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
5081 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
5082 Call_F128: RTLIB::LLRINT_F128,
5083 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
5084 break;
5085 case ISD::FDIV:
5086 case ISD::STRICT_FDIV: {
5087 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5088 Call_F32: {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5089 Call_F64: {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5090 Call_F80: {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5091 Call_F128: {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5092 Call_PPCF128: {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5093 break;
5094 }
5095 case ISD::FREM:
5096 case ISD::STRICT_FREM:
5097 ExpandFPLibCall(Node, LC: RTLIB::getREM(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5098 break;
5099 case ISD::FMA:
5100 case ISD::STRICT_FMA:
5101 ExpandFPLibCall(Node, LC: RTLIB::getFMA(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5102 break;
5103 case ISD::FADD:
5104 case ISD::STRICT_FADD: {
5105 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5106 Call_F32: {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5107 Call_F64: {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5108 Call_F80: {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5109 Call_F128: {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5110 Call_PPCF128: {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5111 break;
5112 }
5113 case ISD::FMUL:
5114 case ISD::STRICT_FMUL: {
5115 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5116 Call_F32: {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5117 Call_F64: {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5118 Call_F80: {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5119 Call_F128: {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5120 Call_PPCF128: {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5121 break;
5122 }
5123 case ISD::FP16_TO_FP:
5124 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5125 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
5126 }
5127 break;
5128 case ISD::STRICT_BF16_TO_FP:
5129 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5130 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5131 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
5132 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5133 Results.push_back(Elt: Tmp.first);
5134 Results.push_back(Elt: Tmp.second);
5135 }
5136 break;
5137 case ISD::STRICT_FP16_TO_FP: {
5138 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5139 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5140 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
5141 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5142 Results.push_back(Elt: Tmp.first);
5143 Results.push_back(Elt: Tmp.second);
5144 }
5145 break;
5146 }
5147 case ISD::FP_TO_FP16: {
5148 RTLIB::Libcall LC =
5149 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
5150 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5151 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5152 break;
5153 }
5154 case ISD::FP_TO_BF16: {
5155 RTLIB::Libcall LC =
5156 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
5157 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5158 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5159 break;
5160 }
5161 case ISD::STRICT_SINT_TO_FP:
5162 case ISD::STRICT_UINT_TO_FP:
5163 case ISD::SINT_TO_FP:
5164 case ISD::UINT_TO_FP: {
5165 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
5166 bool IsStrict = Node->isStrictFPOpcode();
5167 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
5168 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
5169 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
5170 EVT RVT = Node->getValueType(ResNo: 0);
5171 EVT NVT = EVT();
5172 SDLoc dl(Node);
5173
5174 // Even if the input is legal, no libcall may exactly match, eg. we don't
5175 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
5176 // eg: i13 -> fp. Then, look for an appropriate libcall.
5177 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5178 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5179 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5180 ++t) {
5181 NVT = (MVT::SimpleValueType)t;
5182 // The source needs to big enough to hold the operand.
5183 if (NVT.bitsGE(VT: SVT))
5184 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
5185 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
5186 }
5187 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5188
5189 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5190 // Sign/zero extend the argument if the libcall takes a larger type.
5191 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
5192 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
5193 CallOptions.setIsSigned(Signed);
5194 std::pair<SDValue, SDValue> Tmp =
5195 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
5196 Results.push_back(Elt: Tmp.first);
5197 if (IsStrict)
5198 Results.push_back(Elt: Tmp.second);
5199 break;
5200 }
5201 case ISD::FP_TO_SINT:
5202 case ISD::FP_TO_UINT:
5203 case ISD::STRICT_FP_TO_SINT:
5204 case ISD::STRICT_FP_TO_UINT: {
5205 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
5206 bool IsStrict = Node->isStrictFPOpcode();
5207 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
5208 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
5209
5210 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5211 EVT SVT = Op.getValueType();
5212 EVT RVT = Node->getValueType(ResNo: 0);
5213 EVT NVT = EVT();
5214 SDLoc dl(Node);
5215
5216 // Even if the result is legal, no libcall may exactly match, eg. we don't
5217 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
5218 // eg: fp -> i32. Then, look for an appropriate libcall.
5219 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5220 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5221 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5222 ++IntVT) {
5223 NVT = (MVT::SimpleValueType)IntVT;
5224 // The type needs to big enough to hold the result.
5225 if (NVT.bitsGE(VT: RVT))
5226 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
5227 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
5228 }
5229 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5230
5231 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5232 std::pair<SDValue, SDValue> Tmp =
5233 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
5234
5235 // Truncate the result if the libcall returns a larger type.
5236 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
5237 if (IsStrict)
5238 Results.push_back(Elt: Tmp.second);
5239 break;
5240 }
5241
5242 case ISD::FP_ROUND:
5243 case ISD::STRICT_FP_ROUND: {
5244 // X = FP_ROUND(Y, TRUNC)
5245 // TRUNC is a flag, which is always an integer that is zero or one.
5246 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5247 // is known to not change the value of Y.
5248 // We can only expand it into libcall if the TRUNC is 0.
5249 bool IsStrict = Node->isStrictFPOpcode();
5250 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5251 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5252 EVT VT = Node->getValueType(ResNo: 0);
5253 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5254 "Unable to expand as libcall if it is not normal rounding");
5255
5256 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5257 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5258
5259 std::pair<SDValue, SDValue> Tmp =
5260 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5261 Results.push_back(Elt: Tmp.first);
5262 if (IsStrict)
5263 Results.push_back(Elt: Tmp.second);
5264 break;
5265 }
5266 case ISD::FP_EXTEND: {
5267 Results.push_back(
5268 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5269 RetVT: Node->getValueType(ResNo: 0)),
5270 Node, isSigned: false).first);
5271 break;
5272 }
5273 case ISD::STRICT_FP_EXTEND:
5274 case ISD::STRICT_FP_TO_FP16:
5275 case ISD::STRICT_FP_TO_BF16: {
5276 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5277 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5278 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5279 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5280 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5281 else
5282 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5283 RetVT: Node->getValueType(ResNo: 0));
5284
5285 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5286
5287 std::pair<SDValue, SDValue> Tmp =
5288 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5289 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5290 Results.push_back(Elt: Tmp.first);
5291 Results.push_back(Elt: Tmp.second);
5292 break;
5293 }
5294 case ISD::FSUB:
5295 case ISD::STRICT_FSUB: {
5296 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5297 Call_F32: {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5298 Call_F64: {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5299 Call_F80: {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5300 Call_F128: {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5301 Call_PPCF128: {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5302 break;
5303 }
5304 case ISD::SREM:
5305 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5306 Call_I8: RTLIB::SREM_I8,
5307 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5308 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5309 break;
5310 case ISD::UREM:
5311 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5312 Call_I8: RTLIB::UREM_I8,
5313 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5314 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5315 break;
5316 case ISD::SDIV:
5317 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5318 Call_I8: RTLIB::SDIV_I8,
5319 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5320 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5321 break;
5322 case ISD::UDIV:
5323 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5324 Call_I8: RTLIB::UDIV_I8,
5325 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5326 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5327 break;
5328 case ISD::SDIVREM:
5329 case ISD::UDIVREM:
5330 // Expand into divrem libcall
5331 ExpandDivRemLibCall(Node, Results);
5332 break;
5333 case ISD::MUL:
5334 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5335 Call_I8: RTLIB::MUL_I8,
5336 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5337 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5338 break;
5339 case ISD::CTLZ_ZERO_POISON:
5340 Results.push_back(Elt: ExpandBitCountingLibCall(
5341 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5342 break;
5343 case ISD::CTPOP:
5344 Results.push_back(Elt: ExpandBitCountingLibCall(
5345 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5346 break;
5347 case ISD::RESET_FPENV: {
5348 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5349 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5350 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5351 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5352 SDValue Chain = Node->getOperand(Num: 0);
5353 Results.push_back(
5354 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
5355 break;
5356 }
5357 case ISD::GET_FPENV_MEM: {
5358 SDValue Chain = Node->getOperand(Num: 0);
5359 SDValue EnvPtr = Node->getOperand(Num: 1);
5360 Results.push_back(
5361 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5362 break;
5363 }
5364 case ISD::SET_FPENV_MEM: {
5365 SDValue Chain = Node->getOperand(Num: 0);
5366 SDValue EnvPtr = Node->getOperand(Num: 1);
5367 Results.push_back(
5368 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5369 break;
5370 }
5371 case ISD::GET_FPMODE: {
5372 // Call fegetmode, which saves control modes into a stack slot. Then load
5373 // the value to return from the stack.
5374 EVT ModeVT = Node->getValueType(ResNo: 0);
5375 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5376 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5377 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5378 InChain: Node->getOperand(Num: 0), DLoc: dl);
5379 SDValue LdInst = DAG.getLoad(
5380 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5381 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5382 Results.push_back(Elt: LdInst);
5383 Results.push_back(Elt: LdInst.getValue(R: 1));
5384 break;
5385 }
5386 case ISD::SET_FPMODE: {
5387 // Move control modes to stack slot and then call fesetmode with the pointer
5388 // to the slot as argument.
5389 SDValue Mode = Node->getOperand(Num: 1);
5390 EVT ModeVT = Mode.getValueType();
5391 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5392 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5393 SDValue StInst = DAG.getStore(
5394 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5395 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5396 Results.push_back(
5397 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
5398 break;
5399 }
5400 case ISD::RESET_FPMODE: {
5401 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5402 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5403 // target must provide custom lowering.
5404 const DataLayout &DL = DAG.getDataLayout();
5405 EVT PtrTy = TLI.getPointerTy(DL);
5406 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5407 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5408 InChain: Node->getOperand(Num: 0), DLoc: dl));
5409 break;
5410 }
5411 }
5412
5413 // Replace the original node with the legalized result.
5414 if (!Results.empty()) {
5415 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5416 ReplaceNode(Old: Node, New: Results.data());
5417 } else
5418 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5419}
5420
5421// Determine the vector type to use in place of an original scalar element when
5422// promoting equally sized vectors.
5423static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5424 MVT EltVT, MVT NewEltVT) {
5425 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5426 MVT MidVT = OldEltsPerNewElt == 1
5427 ? NewEltVT
5428 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5429 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5430 return MidVT;
5431}
5432
5433void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5434 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5435 SmallVector<SDValue, 8> Results;
5436 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5437 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5438 Node->getOpcode() == ISD::SINT_TO_FP || Node->getOpcode() == ISD::SETCC ||
5439 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5440 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5441 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5442 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5443 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5444 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM ||
5445 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUMNUM ||
5446 Node->getOpcode() == ISD::VECREDUCE_FMINIMUMNUM) {
5447 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5448 }
5449 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5450 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5451 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5452 Node->getOpcode() == ISD::STRICT_FSETCC ||
5453 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5454 Node->getOpcode() == ISD::STRICT_LRINT ||
5455 Node->getOpcode() == ISD::STRICT_LLRINT ||
5456 Node->getOpcode() == ISD::STRICT_LROUND ||
5457 Node->getOpcode() == ISD::STRICT_LLROUND ||
5458 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5459 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5460 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5461 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5462 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5463 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5464 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5465 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5466 if (Node->getOpcode() == ISD::BR_CC ||
5467 Node->getOpcode() == ISD::SELECT_CC)
5468 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5469 // Preserve fast math flags
5470 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5471 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5472 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5473 SDLoc dl(Node);
5474 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5475 switch (Node->getOpcode()) {
5476 case ISD::CTTZ:
5477 case ISD::CTTZ_ZERO_POISON:
5478 case ISD::CTLZ:
5479 case ISD::CTPOP: {
5480 // Zero extend the argument unless its cttz, then use any_extend.
5481 if (Node->getOpcode() == ISD::CTTZ ||
5482 Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
5483 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5484 else
5485 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5486
5487 unsigned NewOpc = Node->getOpcode();
5488 if (NewOpc == ISD::CTTZ) {
5489 // The count is the same in the promoted type except if the original
5490 // value was zero. This can be handled by setting the bit just off
5491 // the top of the original type.
5492 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5493 BitNo: OVT.getSizeInBits());
5494 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5495 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5496 NewOpc = ISD::CTTZ_ZERO_POISON;
5497 }
5498 // Perform the larger operation. For CTPOP and CTTZ_ZERO_POISON, this is
5499 // already the correct result.
5500 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5501 if (NewOpc == ISD::CTLZ) {
5502 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5503 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5504 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5505 OVT.getSizeInBits(), DL: dl, VT: NVT));
5506 }
5507 Results.push_back(
5508 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5509 break;
5510 }
5511 case ISD::CTLZ_ZERO_POISON: {
5512 // We know that the argument is unlikely to be zero, hence we can take a
5513 // different approach as compared to ISD::CTLZ
5514
5515 // Any Extend the argument
5516 auto AnyExtendedNode =
5517 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5518
5519 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5520 auto ShiftConstant = DAG.getShiftAmountConstant(
5521 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5522 auto LeftShiftResult =
5523 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5524
5525 // Perform the larger operation
5526 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5527 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5528 break;
5529 }
5530 case ISD::PEXT: {
5531 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5532 Tmp2 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5533 Tmp1 = DAG.getNode(Opcode: ISD::PEXT, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5534 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5535 break;
5536 }
5537 case ISD::PDEP: {
5538 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5539 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5540 Tmp1 = DAG.getNode(Opcode: ISD::PDEP, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5541 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5542 break;
5543 }
5544 case ISD::BITREVERSE:
5545 case ISD::BSWAP: {
5546 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5547 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5548 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5549 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5550 N2: DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl));
5551
5552 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5553 break;
5554 }
5555 case ISD::FP_TO_UINT:
5556 case ISD::STRICT_FP_TO_UINT:
5557 case ISD::FP_TO_SINT:
5558 case ISD::STRICT_FP_TO_SINT:
5559 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5560 break;
5561 case ISD::FP_TO_UINT_SAT:
5562 case ISD::FP_TO_SINT_SAT:
5563 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5564 break;
5565 case ISD::UINT_TO_FP:
5566 case ISD::STRICT_UINT_TO_FP:
5567 case ISD::SINT_TO_FP:
5568 case ISD::STRICT_SINT_TO_FP:
5569 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5570 break;
5571 case ISD::VAARG: {
5572 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5573 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5574
5575 unsigned TruncOp;
5576 if (OVT.isVector()) {
5577 TruncOp = ISD::BITCAST;
5578 } else {
5579 assert(OVT.isInteger()
5580 && "VAARG promotion is supported only for vectors or integer types");
5581 TruncOp = ISD::TRUNCATE;
5582 }
5583
5584 // Perform the larger operation, then convert back
5585 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5586 Align: Node->getConstantOperandVal(Num: 3));
5587 Chain = Tmp1.getValue(R: 1);
5588
5589 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5590
5591 // Modified the chain result - switch anything that used the old chain to
5592 // use the new one.
5593 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5594 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5595 if (UpdatedNodes) {
5596 UpdatedNodes->insert(X: Tmp2.getNode());
5597 UpdatedNodes->insert(X: Chain.getNode());
5598 }
5599 ReplacedNode(N: Node);
5600 break;
5601 }
5602 case ISD::MUL:
5603 case ISD::SDIV:
5604 case ISD::SREM:
5605 case ISD::UDIV:
5606 case ISD::UREM:
5607 case ISD::SMIN:
5608 case ISD::SMAX:
5609 case ISD::UMIN:
5610 case ISD::UMAX:
5611 case ISD::AND:
5612 case ISD::OR:
5613 case ISD::XOR: {
5614 unsigned ExtOp, TruncOp;
5615 if (OVT.isVector()) {
5616 ExtOp = ISD::BITCAST;
5617 TruncOp = ISD::BITCAST;
5618 } else {
5619 assert(OVT.isInteger() && "Cannot promote logic operation");
5620
5621 switch (Node->getOpcode()) {
5622 default:
5623 ExtOp = ISD::ANY_EXTEND;
5624 break;
5625 case ISD::SDIV:
5626 case ISD::SREM:
5627 case ISD::SMIN:
5628 case ISD::SMAX:
5629 ExtOp = ISD::SIGN_EXTEND;
5630 break;
5631 case ISD::UDIV:
5632 case ISD::UREM:
5633 ExtOp = ISD::ZERO_EXTEND;
5634 break;
5635 case ISD::UMIN:
5636 case ISD::UMAX:
5637 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5638 ExtOp = ISD::SIGN_EXTEND;
5639 else
5640 ExtOp = ISD::ZERO_EXTEND;
5641 break;
5642 }
5643 TruncOp = ISD::TRUNCATE;
5644 }
5645 // Promote each of the values to the new type.
5646 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5647 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5648 // Perform the larger operation, then convert back
5649 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5650 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5651 break;
5652 }
5653 case ISD::UMUL_LOHI:
5654 case ISD::SMUL_LOHI: {
5655 // Promote to a multiply in a wider integer type.
5656 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5657 : ISD::SIGN_EXTEND;
5658 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5659 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5660 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5661
5662 unsigned OriginalSize = OVT.getScalarSizeInBits();
5663 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5664 N2: DAG.getShiftAmountConstant(Val: OriginalSize, VT: NVT, DL: dl));
5665 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5666 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5667 break;
5668 }
5669 case ISD::SELECT: {
5670 unsigned ExtOp, TruncOp;
5671 if (Node->getValueType(ResNo: 0).isVector() ||
5672 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5673 ExtOp = ISD::BITCAST;
5674 TruncOp = ISD::BITCAST;
5675 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5676 ExtOp = ISD::ANY_EXTEND;
5677 TruncOp = ISD::TRUNCATE;
5678 } else {
5679 ExtOp = ISD::FP_EXTEND;
5680 TruncOp = ISD::FP_ROUND;
5681 }
5682 Tmp1 = Node->getOperand(Num: 0);
5683 // Promote each of the values to the new type.
5684 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5685 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5686 // Perform the larger operation, then round down.
5687 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5688 if (TruncOp != ISD::FP_ROUND)
5689 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5690 else
5691 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5692 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5693 Results.push_back(Elt: Tmp1);
5694 break;
5695 }
5696 case ISD::VECTOR_SHUFFLE: {
5697 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5698
5699 // Cast the two input vectors.
5700 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5701 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5702
5703 // Convert the shuffle mask to the right # elements.
5704 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5705 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5706 Results.push_back(Elt: Tmp1);
5707 break;
5708 }
5709 case ISD::VECTOR_SPLICE_LEFT:
5710 case ISD::VECTOR_SPLICE_RIGHT: {
5711 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5712 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5713 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5714 N3: Node->getOperand(Num: 2));
5715 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5716 break;
5717 }
5718 case ISD::SELECT_CC: {
5719 SDValue Cond = Node->getOperand(Num: 4);
5720 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5721 // Type of the comparison operands.
5722 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5723 assert(CVT == OVT && "not handled");
5724
5725 unsigned ExtOp = ISD::FP_EXTEND;
5726 if (NVT.isInteger()) {
5727 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5728 }
5729
5730 // Promote the comparison operands, if needed.
5731 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5732 Tmp1 = Node->getOperand(Num: 0);
5733 Tmp2 = Node->getOperand(Num: 1);
5734 } else {
5735 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5736 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5737 }
5738 // Cast the true/false operands.
5739 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5740 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5741
5742 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5743 Flags: Node->getFlags());
5744
5745 // Cast the result back to the original type.
5746 if (ExtOp != ISD::FP_EXTEND)
5747 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5748 else
5749 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5750 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5751
5752 Results.push_back(Elt: Tmp1);
5753 break;
5754 }
5755 case ISD::SETCC:
5756 case ISD::STRICT_FSETCC:
5757 case ISD::STRICT_FSETCCS: {
5758 unsigned ExtOp = ISD::FP_EXTEND;
5759 if (NVT.isInteger()) {
5760 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5761 if (isSignedIntSetCC(Code: CCCode) ||
5762 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5763 ExtOp = ISD::SIGN_EXTEND;
5764 else
5765 ExtOp = ISD::ZERO_EXTEND;
5766 }
5767 if (Node->isStrictFPOpcode()) {
5768 SDValue InChain = Node->getOperand(Num: 0);
5769 std::tie(args&: Tmp1, args: std::ignore) =
5770 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5771 std::tie(args&: Tmp2, args: std::ignore) =
5772 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5773 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5774 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5775 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
5776 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5777 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5778 Flags: Node->getFlags()));
5779 Results.push_back(Elt: Results.back().getValue(R: 1));
5780 break;
5781 }
5782 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5783 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5784 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5785 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5786 break;
5787 }
5788 case ISD::BR_CC: {
5789 unsigned ExtOp = ISD::FP_EXTEND;
5790 if (NVT.isInteger()) {
5791 ISD::CondCode CCCode =
5792 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5793 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5794 }
5795 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5796 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5797 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5798 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5799 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5800 break;
5801 }
5802 case ISD::FADD:
5803 case ISD::FSUB:
5804 case ISD::FMUL:
5805 case ISD::FDIV:
5806 case ISD::FREM:
5807 case ISD::FMINNUM:
5808 case ISD::FMAXNUM:
5809 case ISD::FMINIMUM:
5810 case ISD::FMAXIMUM:
5811 case ISD::FMINIMUMNUM:
5812 case ISD::FMAXIMUMNUM:
5813 case ISD::FPOW:
5814 case ISD::FATAN2:
5815 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5816 if (!OVT.isVector() && NVT.isVector() &&
5817 NVT.getVectorElementType() == OVT) {
5818 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5819 Tmp2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5820 Tmp3 =
5821 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, Flags: Node->getFlags());
5822 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Tmp3,
5823 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5824 break;
5825 }
5826 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5827 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5828 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5829 Results.push_back(
5830 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5831 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5832 break;
5833
5834 case ISD::STRICT_FMINIMUM:
5835 case ISD::STRICT_FMAXIMUM: {
5836 SDValue InChain = Node->getOperand(Num: 0);
5837 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
5838 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5839 N2: Node->getOperand(Num: 1));
5840 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5841 N2: Node->getOperand(Num: 2));
5842 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
5843 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
5844 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
5845 N1: InChain, N2: Tmp3,
5846 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5847 Results.push_back(Elt: Tmp4);
5848 Results.push_back(Elt: Tmp4.getValue(R: 1));
5849 break;
5850 }
5851
5852 case ISD::STRICT_FADD:
5853 case ISD::STRICT_FSUB:
5854 case ISD::STRICT_FMUL:
5855 case ISD::STRICT_FDIV:
5856 case ISD::STRICT_FMINNUM:
5857 case ISD::STRICT_FMAXNUM:
5858 case ISD::STRICT_FREM:
5859 case ISD::STRICT_FPOW:
5860 case ISD::STRICT_FATAN2:
5861 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5862 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5863 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5864 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5865 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5866 N2: Tmp2.getValue(R: 1));
5867 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5868 Ops: {Tmp3, Tmp1, Tmp2});
5869 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5870 Ops: {Tmp1.getValue(R: 1), Tmp1,
5871 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5872 Results.push_back(Elt: Tmp1);
5873 Results.push_back(Elt: Tmp1.getValue(R: 1));
5874 break;
5875 case ISD::FMA:
5876 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5877 if (!OVT.isVector() && NVT.isVector() &&
5878 NVT.getVectorElementType() == OVT) {
5879 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5880 Tmp2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5881 Tmp3 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5882 SDValue Result = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3,
5883 Flags: Node->getFlags());
5884 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Result,
5885 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5886 break;
5887 }
5888 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5889 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5890 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5891 Results.push_back(
5892 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5893 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5894 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5895 break;
5896 case ISD::STRICT_FMA:
5897 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5898 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5899 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5900 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5901 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5902 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
5903 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5904 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
5905 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5906 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
5907 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5908 Ops: {Tmp4.getValue(R: 1), Tmp4,
5909 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5910 Results.push_back(Elt: Tmp4);
5911 Results.push_back(Elt: Tmp4.getValue(R: 1));
5912 break;
5913 case ISD::FCOPYSIGN:
5914 case ISD::FLDEXP:
5915 case ISD::FPOWI: {
5916 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5917 Tmp2 = Node->getOperand(Num: 1);
5918 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5919
5920 // fcopysign doesn't change anything but the sign bit, so
5921 // (fp_round (fcopysign (fpext a), b))
5922 // is as precise as
5923 // (fp_round (fpext a))
5924 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5925 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5926 Results.push_back(
5927 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5928 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5929 break;
5930 }
5931 case ISD::STRICT_FLDEXP: {
5932 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5933 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5934 Tmp2 = Node->getOperand(Num: 2);
5935 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
5936 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
5937 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5938 Ops: {Tmp3.getValue(R: 1), Tmp3,
5939 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5940 Results.push_back(Elt: Tmp4);
5941 Results.push_back(Elt: Tmp4.getValue(R: 1));
5942 break;
5943 }
5944 case ISD::STRICT_FPOWI:
5945 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5946 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5947 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5948 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
5949 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5950 Ops: {Tmp2.getValue(R: 1), Tmp2,
5951 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5952 Results.push_back(Elt: Tmp3);
5953 Results.push_back(Elt: Tmp3.getValue(R: 1));
5954 break;
5955 case ISD::FFREXP: {
5956 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5957 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5958
5959 Results.push_back(
5960 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5961 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5962
5963 Results.push_back(Elt: Tmp2.getValue(R: 1));
5964 break;
5965 }
5966 case ISD::FMODF:
5967 case ISD::FSINCOS:
5968 case ISD::FSINCOSPI: {
5969 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5970 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
5971 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
5972 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5973 Results.push_back(
5974 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
5975 break;
5976 }
5977 case ISD::FFLOOR:
5978 case ISD::FCEIL:
5979 case ISD::FRINT:
5980 case ISD::FNEARBYINT:
5981 case ISD::FROUND:
5982 case ISD::FROUNDEVEN:
5983 case ISD::FTRUNC:
5984 case ISD::FNEG:
5985 case ISD::FSQRT:
5986 case ISD::FSIN:
5987 case ISD::FCOS:
5988 case ISD::FTAN:
5989 case ISD::FASIN:
5990 case ISD::FACOS:
5991 case ISD::FATAN:
5992 case ISD::FSINH:
5993 case ISD::FCOSH:
5994 case ISD::FTANH:
5995 case ISD::FLOG:
5996 case ISD::FLOG2:
5997 case ISD::FLOG10:
5998 case ISD::FABS:
5999 case ISD::FEXP:
6000 case ISD::FEXP2:
6001 case ISD::FEXP10:
6002 case ISD::FCANONICALIZE:
6003 // Promote scalar operations to vector using SCALAR_TO_VECTOR
6004 if (!OVT.isVector() && NVT.isVector() &&
6005 NVT.getVectorElementType() == OVT) {
6006 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6007 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1, Flags: Node->getFlags());
6008 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Tmp2,
6009 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
6010 break;
6011 }
6012 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6013 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6014 Results.push_back(
6015 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6016 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6017 break;
6018 case ISD::STRICT_FFLOOR:
6019 case ISD::STRICT_FCEIL:
6020 case ISD::STRICT_FRINT:
6021 case ISD::STRICT_FNEARBYINT:
6022 case ISD::STRICT_FROUND:
6023 case ISD::STRICT_FROUNDEVEN:
6024 case ISD::STRICT_FTRUNC:
6025 case ISD::STRICT_FSQRT:
6026 case ISD::STRICT_FSIN:
6027 case ISD::STRICT_FCOS:
6028 case ISD::STRICT_FTAN:
6029 case ISD::STRICT_FASIN:
6030 case ISD::STRICT_FACOS:
6031 case ISD::STRICT_FATAN:
6032 case ISD::STRICT_FSINH:
6033 case ISD::STRICT_FCOSH:
6034 case ISD::STRICT_FTANH:
6035 case ISD::STRICT_FLOG:
6036 case ISD::STRICT_FLOG2:
6037 case ISD::STRICT_FLOG10:
6038 case ISD::STRICT_FEXP:
6039 case ISD::STRICT_FEXP2:
6040 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6041 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6042 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6043 Ops: {Tmp1.getValue(R: 1), Tmp1});
6044 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6045 Ops: {Tmp2.getValue(R: 1), Tmp2,
6046 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6047 Results.push_back(Elt: Tmp3);
6048 Results.push_back(Elt: Tmp3.getValue(R: 1));
6049 break;
6050 case ISD::LLROUND:
6051 case ISD::LROUND:
6052 case ISD::LRINT:
6053 case ISD::LLRINT:
6054 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6055 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
6056 Results.push_back(Elt: Tmp2);
6057 break;
6058 case ISD::STRICT_LLROUND:
6059 case ISD::STRICT_LROUND:
6060 case ISD::STRICT_LRINT:
6061 case ISD::STRICT_LLRINT:
6062 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6063 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6064 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6065 Ops: {Tmp1.getValue(R: 1), Tmp1});
6066 Results.push_back(Elt: Tmp2);
6067 Results.push_back(Elt: Tmp2.getValue(R: 1));
6068 break;
6069 case ISD::BUILD_VECTOR: {
6070 MVT EltVT = OVT.getVectorElementType();
6071 MVT NewEltVT = NVT.getVectorElementType();
6072
6073 // Handle bitcasts to a different vector type with the same total bit size
6074 //
6075 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
6076 // =>
6077 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
6078
6079 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6080 "Invalid promote type for build_vector");
6081 assert(NewEltVT.bitsLE(EltVT) && "not handled");
6082
6083 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6084
6085 SmallVector<SDValue, 8> NewOps;
6086 for (const SDValue &Op : Node->op_values())
6087 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
6088
6089 SDLoc SL(Node);
6090 SDValue Concat =
6091 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
6092 DL: SL, VT: NVT, Ops: NewOps);
6093 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6094 Results.push_back(Elt: CvtVec);
6095 break;
6096 }
6097 case ISD::EXTRACT_VECTOR_ELT: {
6098 MVT EltVT = OVT.getVectorElementType();
6099 MVT NewEltVT = NVT.getVectorElementType();
6100
6101 // Handle bitcasts to a different vector type with the same total bit size.
6102 //
6103 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
6104 // =>
6105 // v4i32:castx = bitcast x:v2i64
6106 //
6107 // i64 = bitcast
6108 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
6109 // (i32 (extract_vector_elt castx, (2 * y + 1)))
6110 //
6111
6112 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6113 "Invalid promote type for extract_vector_elt");
6114 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6115
6116 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6117 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6118
6119 SDValue Idx = Node->getOperand(Num: 1);
6120 EVT IdxVT = Idx.getValueType();
6121 SDLoc SL(Node);
6122 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
6123 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6124
6125 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6126
6127 SmallVector<SDValue, 8> NewOps;
6128 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6129 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6130 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6131
6132 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6133 N1: CastVec, N2: TmpIdx);
6134 NewOps.push_back(Elt);
6135 }
6136
6137 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
6138 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
6139 break;
6140 }
6141 case ISD::INSERT_VECTOR_ELT: {
6142 MVT EltVT = OVT.getVectorElementType();
6143 MVT NewEltVT = NVT.getVectorElementType();
6144
6145 // Handle bitcasts to a different vector type with the same total bit size
6146 //
6147 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
6148 // =>
6149 // v4i32:castx = bitcast x:v2i64
6150 // v2i32:casty = bitcast y:i64
6151 //
6152 // v2i64 = bitcast
6153 // (v4i32 insert_vector_elt
6154 // (v4i32 insert_vector_elt v4i32:castx,
6155 // (extract_vector_elt casty, 0), 2 * z),
6156 // (extract_vector_elt casty, 1), (2 * z + 1))
6157
6158 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6159 "Invalid promote type for insert_vector_elt");
6160 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6161
6162 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6163 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6164
6165 SDValue Val = Node->getOperand(Num: 1);
6166 SDValue Idx = Node->getOperand(Num: 2);
6167 EVT IdxVT = Idx.getValueType();
6168 SDLoc SL(Node);
6169
6170 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
6171 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6172
6173 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6174 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6175
6176 SDValue NewVec = CastVec;
6177 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6178 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6179 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6180
6181 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6182 N1: CastVal, N2: IdxOffset);
6183
6184 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
6185 N1: NewVec, N2: Elt, N3: InEltIdx);
6186 }
6187
6188 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
6189 break;
6190 }
6191 case ISD::SCALAR_TO_VECTOR: {
6192 MVT EltVT = OVT.getVectorElementType();
6193 MVT NewEltVT = NVT.getVectorElementType();
6194
6195 // Handle bitcasts to different vector type with the same total bit size.
6196 //
6197 // e.g. v2i64 = scalar_to_vector x:i64
6198 // =>
6199 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
6200 //
6201
6202 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6203 SDValue Val = Node->getOperand(Num: 0);
6204 SDLoc SL(Node);
6205
6206 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6207 SDValue Undef = DAG.getUNDEF(VT: MidVT);
6208
6209 SmallVector<SDValue, 8> NewElts;
6210 NewElts.push_back(Elt: CastVal);
6211 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
6212 NewElts.push_back(Elt: Undef);
6213
6214 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
6215 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6216 Results.push_back(Elt: CvtVec);
6217 break;
6218 }
6219 case ISD::ATOMIC_SWAP:
6220 case ISD::ATOMIC_STORE: {
6221 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6222 SDLoc SL(Node);
6223 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
6224 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6225 "unexpected promotion type");
6226 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6227 "unexpected atomic_swap with illegal type");
6228
6229 SDValue Op0 = AM->getBasePtr();
6230 SDValue Op1 = CastVal;
6231
6232 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
6233 // but really it should merge with ISD::STORE.
6234 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6235 std::swap(a&: Op0, b&: Op1);
6236
6237 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
6238 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
6239
6240 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6241 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6242 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6243 } else
6244 Results.push_back(Elt: NewAtomic);
6245 break;
6246 }
6247 case ISD::ATOMIC_LOAD: {
6248 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6249 SDLoc SL(Node);
6250 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6251 "unexpected promotion type");
6252 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6253 "unexpected atomic_load with illegal type");
6254
6255 SDValue NewAtomic =
6256 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
6257 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
6258 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6259 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6260 break;
6261 }
6262 case ISD::SPLAT_VECTOR: {
6263 SDValue Scalar = Node->getOperand(Num: 0);
6264 MVT ScalarType = Scalar.getSimpleValueType();
6265 MVT NewScalarType = NVT.getVectorElementType();
6266 if (ScalarType.isInteger()) {
6267 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6268 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6269 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
6270 break;
6271 }
6272 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6273 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6274 Results.push_back(
6275 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6276 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6277 break;
6278 }
6279 case ISD::VECREDUCE_FMAX:
6280 case ISD::VECREDUCE_FMIN:
6281 case ISD::VECREDUCE_FMAXIMUM:
6282 case ISD::VECREDUCE_FMINIMUM:
6283 case ISD::VECREDUCE_FMAXIMUMNUM:
6284 case ISD::VECREDUCE_FMINIMUMNUM:
6285 case ISD::VP_REDUCE_FMAX:
6286 case ISD::VP_REDUCE_FMIN:
6287 case ISD::VP_REDUCE_FMAXIMUM:
6288 case ISD::VP_REDUCE_FMINIMUM:
6289 Results.push_back(Elt: PromoteReduction(Node));
6290 break;
6291 }
6292
6293 // Replace the original node with the legalized result.
6294 if (!Results.empty()) {
6295 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6296 ReplaceNode(Old: Node, New: Results.data());
6297 } else
6298 LLVM_DEBUG(dbgs() << "Could not promote node\n");
6299}
6300
6301/// This is the entry point for the file.
6302void SelectionDAG::Legalize() {
6303 AssignTopologicalOrder();
6304
6305 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6306 // Use a delete listener to remove nodes which were deleted during
6307 // legalization from LegalizeNodes. This is needed to handle the situation
6308 // where a new node is allocated by the object pool to the same address of a
6309 // previously deleted node.
6310 DAGNodeDeletedListener DeleteListener(
6311 *this,
6312 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
6313
6314 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6315
6316 // Visit all the nodes. We start in topological order, so that we see
6317 // nodes with their original operands intact. Legalization can produce
6318 // new nodes which may themselves need to be legalized. Iterate until all
6319 // nodes have been legalized.
6320 while (true) {
6321 bool AnyLegalized = false;
6322 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6323 --NI;
6324
6325 SDNode *N = &*NI;
6326 if (N->use_empty() && N != getRoot().getNode()) {
6327 ++NI;
6328 DeleteNode(N);
6329 continue;
6330 }
6331
6332 if (LegalizedNodes.insert(Ptr: N).second) {
6333 AnyLegalized = true;
6334 Legalizer.LegalizeOp(Node: N);
6335
6336 if (N->use_empty() && N != getRoot().getNode()) {
6337 ++NI;
6338 DeleteNode(N);
6339 }
6340 }
6341 }
6342 if (!AnyLegalized)
6343 break;
6344
6345 }
6346
6347 // Remove dead nodes now.
6348 RemoveDeadNodes();
6349}
6350
6351bool SelectionDAG::LegalizeOp(SDNode *N,
6352 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6353 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6354 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6355
6356 // Directly insert the node in question, and legalize it. This will recurse
6357 // as needed through operands.
6358 LegalizedNodes.insert(Ptr: N);
6359 Legalizer.LegalizeOp(Node: N);
6360
6361 return LegalizedNodes.count(Ptr: N);
6362}
6363