1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SelectionDAGNodes.h"
32#include "llvm/CodeGen/TargetFrameLowering.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/CodeGenTypes/MachineValueType.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Compiler.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/MathExtras.h"
49#include "llvm/Support/raw_ostream.h"
50#include "llvm/Target/TargetMachine.h"
51#include "llvm/Target/TargetOptions.h"
52#include <cassert>
53#include <cstdint>
54#include <tuple>
55#include <utility>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "legalizedag"
60
61namespace {
62
63/// Keeps track of state when getting the sign of a floating-point value as an
64/// integer.
65struct FloatSignAsInt {
66 EVT FloatVT;
67 SDValue Chain;
68 SDValue FloatPtr;
69 SDValue IntPtr;
70 MachinePointerInfo IntPointerInfo;
71 MachinePointerInfo FloatPointerInfo;
72 SDValue IntValue;
73 APInt SignMask;
74 uint8_t SignBit;
75};
76
77//===----------------------------------------------------------------------===//
78/// This takes an arbitrary SelectionDAG as input and
79/// hacks on it until the target machine can handle it. This involves
80/// eliminating value sizes the machine cannot handle (promoting small sizes to
81/// large sizes or splitting up large values into small values) as well as
82/// eliminating operations the machine cannot handle.
83///
84/// This code also does a small amount of optimization and recognition of idioms
85/// as part of its processing. For example, if a target does not support a
86/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
87/// will attempt merge setcc and brc instructions into brcc's.
88class SelectionDAGLegalize {
89 const TargetMachine &TM;
90 const TargetLowering &TLI;
91 SelectionDAG &DAG;
92
93 /// The set of nodes which have already been legalized. We hold a
94 /// reference to it in order to update as necessary on node deletion.
95 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
96
97 /// A set of all the nodes updated during legalization.
98 SmallSetVector<SDNode *, 16> *UpdatedNodes;
99
100 EVT getSetCCResultType(EVT VT) const {
101 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
102 }
103
104 // Libcall insertion helpers.
105
106public:
107 SelectionDAGLegalize(SelectionDAG &DAG,
108 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
109 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
110 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
112
113 /// Legalizes the given operation.
114 void LegalizeOp(SDNode *Node);
115
116private:
117 SDValue OptimizeFloatStore(StoreSDNode *ST);
118
119 void LegalizeLoadOps(SDNode *Node);
120 void LegalizeStoreOps(SDNode *Node);
121
122 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
123
124 /// Return a vector shuffle operation which
125 /// performs the same shuffe in terms of order or result bytes, but on a type
126 /// whose vector element type is narrower than the original shuffle type.
127 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
128 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
129 SDValue N1, SDValue N2,
130 ArrayRef<int> Mask) const;
131
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 TargetLowering::ArgListTy &&Args,
134 bool IsSigned, EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
136
137 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 SmallVectorImpl<SDValue> &Results);
139 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
140 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
141 RTLIB::Libcall Call_F128,
142 RTLIB::Libcall Call_PPCF128,
143 SmallVectorImpl<SDValue> &Results);
144
145 void
146 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
147 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
148 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
149 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
150 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
151 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
152 SmallVectorImpl<SDValue> &Results);
153
154 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
155 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
156 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
157 void ExpandArgFPLibCall(SDNode *Node,
158 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
159 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
160 RTLIB::Libcall Call_PPCF128,
161 SmallVectorImpl<SDValue> &Results);
162 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
163 RTLIB::Libcall CallI64,
164 RTLIB::Libcall CallI128);
165 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
166
167 SDValue ExpandSincosStretLibCall(SDNode *Node) const;
168
169 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
170 const SDLoc &dl);
171 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
172 const SDLoc &dl, SDValue ChainIn);
173 SDValue ExpandBUILD_VECTOR(SDNode *Node);
174 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
175 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
176 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
177 SmallVectorImpl<SDValue> &Results);
178 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
179 SDValue Value) const;
180 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
181 SDValue NewIntValue) const;
182 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
183 SDValue ExpandFABS(SDNode *Node) const;
184 SDValue ExpandFNEG(SDNode *Node) const;
185 SDValue expandLdexp(SDNode *Node) const;
186 SDValue expandFrexp(SDNode *Node) const;
187 SDValue expandModf(SDNode *Node) const;
188
189 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
190 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
191 SmallVectorImpl<SDValue> &Results);
192 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
193 SmallVectorImpl<SDValue> &Results);
194 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
195
196 /// Implements vector reduce operation promotion.
197 ///
198 /// All vector operands are promoted to a vector type with larger element
199 /// type, and the start value is promoted to a larger scalar type. Then the
200 /// result is truncated back to the original scalar type.
201 SDValue PromoteReduction(SDNode *Node);
202
203 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
204
205 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
206 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
207 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
208 SDValue ExpandConcatVectors(SDNode *Node);
209
210 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
211 SDValue ExpandConstant(ConstantSDNode *CP);
212
213 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
214 bool ExpandNode(SDNode *Node);
215 void ConvertNodeToLibcall(SDNode *Node);
216 void PromoteNode(SDNode *Node);
217
218public:
219 // Node replacement helpers
220
221 void ReplacedNode(SDNode *N) {
222 LegalizedNodes.erase(Ptr: N);
223 if (UpdatedNodes)
224 UpdatedNodes->insert(X: N);
225 }
226
227 void ReplaceNode(SDNode *Old, SDNode *New) {
228 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
229 dbgs() << " with: "; New->dump(&DAG));
230
231 assert(Old->getNumValues() == New->getNumValues() &&
232 "Replacing one node with another that produces a different number "
233 "of values!");
234 DAG.ReplaceAllUsesWith(From: Old, To: New);
235 if (UpdatedNodes)
236 UpdatedNodes->insert(X: New);
237 ReplacedNode(N: Old);
238 }
239
240 void ReplaceNode(SDValue Old, SDValue New) {
241 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
242 dbgs() << " with: "; New->dump(&DAG));
243
244 DAG.ReplaceAllUsesWith(From: Old, To: New);
245 if (UpdatedNodes)
246 UpdatedNodes->insert(X: New.getNode());
247 ReplacedNode(N: Old.getNode());
248 }
249
250 void ReplaceNode(SDNode *Old, const SDValue *New) {
251 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
252
253 DAG.ReplaceAllUsesWith(From: Old, To: New);
254 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
255 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
256 New[i]->dump(&DAG));
257 if (UpdatedNodes)
258 UpdatedNodes->insert(X: New[i].getNode());
259 }
260 ReplacedNode(N: Old);
261 }
262
263 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
264 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
265 dbgs() << " with: "; New->dump(&DAG));
266
267 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
268 if (UpdatedNodes)
269 UpdatedNodes->insert(X: New.getNode());
270 ReplacedNode(N: Old.getNode());
271 }
272};
273
274} // end anonymous namespace
275
276// Helper function that generates an MMO that considers the alignment of the
277// stack, and the size of the stack object
278static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
279 MachineFunction &MF,
280 bool isObjectScalable) {
281 auto &MFI = MF.getFrameInfo();
282 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
283 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
284 LocationSize ObjectSize = isObjectScalable
285 ? LocationSize::beforeOrAfterPointer()
286 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
287 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
288 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
289}
290
291/// Return a vector shuffle operation which
292/// performs the same shuffle in terms of order or result bytes, but on a type
293/// whose vector element type is narrower than the original shuffle type.
294/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
295SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
296 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
297 ArrayRef<int> Mask) const {
298 unsigned NumMaskElts = VT.getVectorNumElements();
299 unsigned NumDestElts = NVT.getVectorNumElements();
300 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
301
302 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
303
304 if (NumEltsGrowth == 1)
305 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
306
307 SmallVector<int, 8> NewMask;
308 for (unsigned i = 0; i != NumMaskElts; ++i) {
309 int Idx = Mask[i];
310 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
311 if (Idx < 0)
312 NewMask.push_back(Elt: -1);
313 else
314 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
315 }
316 }
317 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
318 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
319 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
320}
321
322/// Expands the ConstantFP node to an integer constant or
323/// a load from the constant pool.
324SDValue
325SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
326 bool Extend = false;
327 SDLoc dl(CFP);
328
329 // If a FP immediate is precise when represented as a float and if the
330 // target can do an extending load from float to double, we put it into
331 // the constant pool as a float, even if it's is statically typed as a
332 // double. This shrinks FP constants and canonicalizes them for targets where
333 // an FP extending load is the same cost as a normal load (such as on the x87
334 // fp stack or PPC FP unit).
335 EVT VT = CFP->getValueType(ResNo: 0);
336 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
337 if (!UseCP) {
338 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
339 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
340 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
341 }
342
343 APFloat APF = CFP->getValueAPF();
344 EVT OrigVT = VT;
345 EVT SVT = VT;
346
347 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
348 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
349 if (!APF.isSignaling()) {
350 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
351 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
352 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
353 // Only do this if the target has a native EXTLOAD instruction from
354 // smaller type.
355 TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: OrigVT, MemVT: SVT) &&
356 TLI.ShouldShrinkFPConstant(OrigVT)) {
357 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
358 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
359 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
360 VT = SVT;
361 Extend = true;
362 }
363 }
364 }
365
366 SDValue CPIdx =
367 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
368 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
369 if (Extend) {
370 SDValue Result = DAG.getExtLoad(
371 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
372 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
373 Alignment);
374 return Result;
375 }
376 SDValue Result = DAG.getLoad(
377 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
378 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
379 return Result;
380}
381
382/// Expands the Constant node to a load from the constant pool.
383SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
384 SDLoc dl(CP);
385 EVT VT = CP->getValueType(ResNo: 0);
386 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
387 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
388 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
389 SDValue Result = DAG.getLoad(
390 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
391 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
392 return Result;
393}
394
395SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
396 SDValue Vec = Op.getOperand(i: 0);
397 SDValue Val = Op.getOperand(i: 1);
398 SDValue Idx = Op.getOperand(i: 2);
399 SDLoc dl(Op);
400
401 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
402 // SCALAR_TO_VECTOR requires that the type of the value being inserted
403 // match the element type of the vector being created, except for
404 // integers in which case the inserted value can be over width.
405 EVT EltVT = Vec.getValueType().getVectorElementType();
406 if (Val.getValueType() == EltVT ||
407 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
408 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
409 VT: Vec.getValueType(), Operand: Val);
410
411 unsigned NumElts = Vec.getValueType().getVectorNumElements();
412 // We generate a shuffle of InVec and ScVec, so the shuffle mask
413 // should be 0,1,2,3,4,5... with the appropriate element replaced with
414 // elt 0 of the RHS.
415 SmallVector<int, 8> ShufOps;
416 for (unsigned i = 0; i != NumElts; ++i)
417 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
418
419 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
420 }
421 }
422 return ExpandInsertToVectorThroughStack(Op);
423}
424
425SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
426 if (!ISD::isNormalStore(N: ST))
427 return SDValue();
428
429 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
430 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
431 // FIXME: move this to the DAG Combiner! Note that we can't regress due
432 // to phase ordering between legalized code and the dag combiner. This
433 // probably means that we need to integrate dag combiner and legalizer
434 // together.
435 // We generally can't do this one for long doubles.
436 SDValue Chain = ST->getChain();
437 SDValue Ptr = ST->getBasePtr();
438 SDValue Value = ST->getValue();
439 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
440 AAMDNodes AAInfo = ST->getAAInfo();
441 SDLoc dl(ST);
442
443 // Don't optimise TargetConstantFP
444 if (Value.getOpcode() == ISD::TargetConstantFP)
445 return SDValue();
446
447 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
448 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
449 TLI.isTypeLegal(VT: MVT::i32)) {
450 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
451 bitcastToAPInt().zextOrTrunc(width: 32),
452 DL: SDLoc(CFP), VT: MVT::i32);
453 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
454 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
455 }
456
457 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
458 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
459 // If this target supports 64-bit registers, do a single 64-bit store.
460 if (TLI.isTypeLegal(VT: MVT::i64)) {
461 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
462 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
463 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
464 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
465 }
466
467 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
468 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
469 // stores. If the target supports neither 32- nor 64-bits, this
470 // xform is certainly not worth it.
471 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
472 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
473 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
474 if (DAG.getDataLayout().isBigEndian())
475 std::swap(a&: Lo, b&: Hi);
476
477 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
478 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
479 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
480 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
481 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
482 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
483
484 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
485 }
486 }
487 }
488 return SDValue();
489}
490
491void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
492 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
493 SDValue Chain = ST->getChain();
494 SDValue Ptr = ST->getBasePtr();
495 SDLoc dl(Node);
496
497 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
498 AAMDNodes AAInfo = ST->getAAInfo();
499
500 if (!ST->isTruncatingStore()) {
501 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
502 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
503 ReplaceNode(Old: ST, New: OptStore);
504 return;
505 }
506
507 SDValue Value = ST->getValue();
508 MVT VT = Value.getSimpleValueType();
509 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
510 default: llvm_unreachable("This action is not supported yet!");
511 case TargetLowering::Legal: {
512 // If this is an unaligned store and the target doesn't support it,
513 // expand it.
514 EVT MemVT = ST->getMemoryVT();
515 const DataLayout &DL = DAG.getDataLayout();
516 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
517 MMO: *ST->getMemOperand())) {
518 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
519 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
520 ReplaceNode(Old: SDValue(ST, 0), New: Result);
521 } else
522 LLVM_DEBUG(dbgs() << "Legal store\n");
523 break;
524 }
525 case TargetLowering::Custom: {
526 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
527 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
528 if (Res && Res != SDValue(Node, 0))
529 ReplaceNode(Old: SDValue(Node, 0), New: Res);
530 return;
531 }
532 case TargetLowering::Promote: {
533 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
534 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
535 "Can only promote stores to same size type");
536 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
537 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
538 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
539 ReplaceNode(Old: SDValue(Node, 0), New: Result);
540 break;
541 }
542 }
543 return;
544 }
545
546 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
547 SDValue Value = ST->getValue();
548 EVT StVT = ST->getMemoryVT();
549 TypeSize StWidth = StVT.getSizeInBits();
550 TypeSize StSize = StVT.getStoreSizeInBits();
551 auto &DL = DAG.getDataLayout();
552
553 if (StWidth != StSize) {
554 // Promote to a byte-sized store with upper bits zero if not
555 // storing an integral number of bytes. For example, promote
556 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
557 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
558 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
559 SDValue Result =
560 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
561 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
562 ReplaceNode(Old: SDValue(Node, 0), New: Result);
563 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
564 // If not storing a power-of-2 number of bits, expand as two stores.
565 assert(!StVT.isVector() && "Unsupported truncstore!");
566 unsigned StWidthBits = StWidth.getFixedValue();
567 unsigned LogStWidth = Log2_32(Value: StWidthBits);
568 assert(LogStWidth < 32);
569 unsigned RoundWidth = 1 << LogStWidth;
570 assert(RoundWidth < StWidthBits);
571 unsigned ExtraWidth = StWidthBits - RoundWidth;
572 assert(ExtraWidth < RoundWidth);
573 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
574 "Store size not an integral number of bytes!");
575 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
576 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
577 SDValue Lo, Hi;
578 unsigned IncrementSize;
579
580 if (DL.isLittleEndian()) {
581 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
582 // Store the bottom RoundWidth bits.
583 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
584 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
585
586 // Store the remaining ExtraWidth bits.
587 IncrementSize = RoundWidth / 8;
588 Ptr =
589 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
590 Hi = DAG.getNode(
591 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
592 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Value.getValueType(), DL: dl));
593 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
594 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
595 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
596 } else {
597 // Big endian - avoid unaligned stores.
598 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
599 // Store the top RoundWidth bits.
600 Hi = DAG.getNode(
601 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
602 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Value.getValueType(), DL: dl));
603 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
604 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
605
606 // Store the remaining ExtraWidth bits.
607 IncrementSize = RoundWidth / 8;
608 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
609 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
610 VT: Ptr.getValueType()));
611 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
612 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
613 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
614 }
615
616 // The order of the stores doesn't matter.
617 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
618 ReplaceNode(Old: SDValue(Node, 0), New: Result);
619 } else {
620 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT)) {
621 default: llvm_unreachable("This action is not supported yet!");
622 case TargetLowering::Legal: {
623 EVT MemVT = ST->getMemoryVT();
624 // If this is an unaligned store and the target doesn't support it,
625 // expand it.
626 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
627 MMO: *ST->getMemOperand())) {
628 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
629 ReplaceNode(Old: SDValue(ST, 0), New: Result);
630 }
631 break;
632 }
633 case TargetLowering::Custom: {
634 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
635 if (Res && Res != SDValue(Node, 0))
636 ReplaceNode(Old: SDValue(Node, 0), New: Res);
637 return;
638 }
639 case TargetLowering::Expand:
640 assert(!StVT.isVector() &&
641 "Vector Stores are handled in LegalizeVectorOps");
642
643 SDValue Result;
644
645 // TRUNCSTORE:i16 i32 -> STORE i16
646 if (TLI.isTypeLegal(VT: StVT)) {
647 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
648 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
649 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
650 } else {
651 // The in-memory type isn't legal. Truncate to the type it would promote
652 // to, and then do a truncstore.
653 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
654 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
655 Operand: Value);
656 Result = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
657 SVT: StVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
658 }
659
660 ReplaceNode(Old: SDValue(Node, 0), New: Result);
661 break;
662 }
663 }
664}
665
666void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
667 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
668 SDValue Chain = LD->getChain(); // The chain.
669 SDValue Ptr = LD->getBasePtr(); // The base pointer.
670 SDValue Value; // The value returned by the load op.
671 SDLoc dl(Node);
672
673 ISD::LoadExtType ExtType = LD->getExtensionType();
674 if (ExtType == ISD::NON_EXTLOAD) {
675 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
676 MVT VT = Node->getSimpleValueType(ResNo: 0);
677 SDValue RVal = SDValue(Node, 0);
678 SDValue RChain = SDValue(Node, 1);
679
680 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
681 default: llvm_unreachable("This action is not supported yet!");
682 case TargetLowering::Legal: {
683 EVT MemVT = LD->getMemoryVT();
684 const DataLayout &DL = DAG.getDataLayout();
685 // If this is an unaligned load and the target doesn't support it,
686 // expand it.
687 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
688 MMO: *LD->getMemOperand())) {
689 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
690 }
691 break;
692 }
693 case TargetLowering::Custom:
694 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
695 RVal = Res;
696 RChain = Res.getValue(R: 1);
697 }
698 break;
699
700 case TargetLowering::Promote: {
701 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
702 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
703 "Can only promote loads to same size type");
704
705 // If the range metadata type does not match the legalized memory
706 // operation type, remove the range metadata.
707 if (const MDNode *MD = LD->getRanges()) {
708 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
709 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
710 !NVT.isInteger())
711 LD->getMemOperand()->clearRanges();
712 }
713 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
714 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
715 RChain = Res.getValue(R: 1);
716 break;
717 }
718 }
719 if (RChain.getNode() != Node) {
720 assert(RVal.getNode() != Node && "Load must be completely replaced");
721 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
722 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
723 if (UpdatedNodes) {
724 UpdatedNodes->insert(X: RVal.getNode());
725 UpdatedNodes->insert(X: RChain.getNode());
726 }
727 ReplacedNode(N: Node);
728 }
729 return;
730 }
731
732 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
733 EVT SrcVT = LD->getMemoryVT();
734 TypeSize SrcWidth = SrcVT.getSizeInBits();
735 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
736 AAMDNodes AAInfo = LD->getAAInfo();
737
738 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
739 // Some targets pretend to have an i1 loading operation, and actually
740 // load an i8. This trick is correct for ZEXTLOAD because the top 7
741 // bits are guaranteed to be zero; it helps the optimizers understand
742 // that these bits are zero. It is also useful for EXTLOAD, since it
743 // tells the optimizers that those bits are undefined. It would be
744 // nice to have an effective generic way of getting these benefits...
745 // Until such a way is found, don't insist on promoting i1 here.
746 (SrcVT != MVT::i1 ||
747 TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1) ==
748 TargetLowering::Promote)) {
749 // Promote to a byte-sized load if not loading an integral number of
750 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
751 unsigned NewWidth = SrcVT.getStoreSizeInBits();
752 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
753 SDValue Ch;
754
755 // The extra bits are guaranteed to be zero, since we stored them that
756 // way. A zext load from NVT thus automatically gives zext from SrcVT.
757
758 ISD::LoadExtType NewExtType =
759 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
760
761 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
762 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
763 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
764
765 Ch = Result.getValue(R: 1); // The chain.
766
767 if (ExtType == ISD::SEXTLOAD)
768 // Having the top bits zero doesn't help when sign extending.
769 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
770 VT: Result.getValueType(),
771 N1: Result, N2: DAG.getValueType(SrcVT));
772 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
773 // All the top bits are guaranteed to be zero - inform the optimizers.
774 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
775 VT: Result.getValueType(), N1: Result,
776 N2: DAG.getValueType(SrcVT));
777
778 Value = Result;
779 Chain = Ch;
780 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
781 // If not loading a power-of-2 number of bits, expand as two loads.
782 assert(!SrcVT.isVector() && "Unsupported extload!");
783 unsigned SrcWidthBits = SrcWidth.getFixedValue();
784 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
785 assert(LogSrcWidth < 32);
786 unsigned RoundWidth = 1 << LogSrcWidth;
787 assert(RoundWidth < SrcWidthBits);
788 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
789 assert(ExtraWidth < RoundWidth);
790 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
791 "Load size not an integral number of bytes!");
792 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
793 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
794 SDValue Lo, Hi, Ch;
795 unsigned IncrementSize;
796 auto &DL = DAG.getDataLayout();
797
798 if (DL.isLittleEndian()) {
799 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
800 // Load the bottom RoundWidth bits.
801 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
802 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
803 MMOFlags, AAInfo);
804
805 // Load the remaining ExtraWidth bits.
806 IncrementSize = RoundWidth / 8;
807 Ptr =
808 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
809 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
810 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
811 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
812
813 // Build a factor node to remember that this load is independent of
814 // the other one.
815 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
816 N2: Hi.getValue(R: 1));
817
818 // Move the top bits to the right place.
819 Hi = DAG.getNode(
820 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
821 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Hi.getValueType(), DL: dl));
822
823 // Join the hi and lo parts.
824 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
825 } else {
826 // Big endian - avoid unaligned loads.
827 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
828 // Load the top RoundWidth bits.
829 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
830 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
831 MMOFlags, AAInfo);
832
833 // Load the remaining ExtraWidth bits.
834 IncrementSize = RoundWidth / 8;
835 Ptr =
836 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
837 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
838 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
839 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
840
841 // Build a factor node to remember that this load is independent of
842 // the other one.
843 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
844 N2: Hi.getValue(R: 1));
845
846 // Move the top bits to the right place.
847 Hi = DAG.getNode(
848 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
849 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Hi.getValueType(), DL: dl));
850
851 // Join the hi and lo parts.
852 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
853 }
854
855 Chain = Ch;
856 } else {
857 bool isCustom = false;
858 switch (TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0),
859 MemVT: SrcVT.getSimpleVT())) {
860 default: llvm_unreachable("This action is not supported yet!");
861 case TargetLowering::Custom:
862 isCustom = true;
863 [[fallthrough]];
864 case TargetLowering::Legal:
865 Value = SDValue(Node, 0);
866 Chain = SDValue(Node, 1);
867
868 if (isCustom) {
869 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
870 Value = Res;
871 Chain = Res.getValue(R: 1);
872 }
873 } else {
874 // If this is an unaligned load and the target doesn't support it,
875 // expand it.
876 EVT MemVT = LD->getMemoryVT();
877 const DataLayout &DL = DAG.getDataLayout();
878 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
879 MMO: *LD->getMemOperand())) {
880 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
881 }
882 }
883 break;
884
885 case TargetLowering::Expand: {
886 EVT DestVT = Node->getValueType(ResNo: 0);
887 if (!TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SrcVT)) {
888 // If the source type is not legal, see if there is a legal extload to
889 // an intermediate type that we can then extend further.
890 EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT());
891 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
892 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
893 TLI.isLoadExtLegal(ExtType, ValVT: LoadVT, MemVT: SrcVT))) {
894 // If we are loading a legal type, this is a non-extload followed by a
895 // full extend.
896 ISD::LoadExtType MidExtType =
897 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
898
899 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
900 MemVT: SrcVT, MMO: LD->getMemOperand());
901 unsigned ExtendOp =
902 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
903 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
904 Chain = Load.getValue(R: 1);
905 break;
906 }
907
908 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
909 // normal undefined upper bits behavior to allow using an in-reg extend
910 // with the illegal FP type, so load as an integer and do the
911 // from-integer conversion.
912 EVT SVT = SrcVT.getScalarType();
913 if (SVT == MVT::f16 || SVT == MVT::bf16) {
914 EVT ISrcVT = SrcVT.changeTypeToInteger();
915 EVT IDestVT = DestVT.changeTypeToInteger();
916 EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT());
917
918 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
919 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
920 Value =
921 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
922 DL: dl, VT: DestVT, Operand: Result);
923 Chain = Result.getValue(R: 1);
924 break;
925 }
926 }
927
928 assert(!SrcVT.isVector() &&
929 "Vector Loads are handled in LegalizeVectorOps");
930
931 // FIXME: This does not work for vectors on most targets. Sign-
932 // and zero-extend operations are currently folded into extending
933 // loads, whether they are legal or not, and then we end up here
934 // without any support for legalizing them.
935 assert(ExtType != ISD::EXTLOAD &&
936 "EXTLOAD should always be supported!");
937 // Turn the unsupported load into an EXTLOAD followed by an
938 // explicit zero/sign extend inreg.
939 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
940 VT: Node->getValueType(ResNo: 0),
941 Chain, Ptr, MemVT: SrcVT,
942 MMO: LD->getMemOperand());
943 SDValue ValRes;
944 if (ExtType == ISD::SEXTLOAD)
945 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
946 VT: Result.getValueType(),
947 N1: Result, N2: DAG.getValueType(SrcVT));
948 else
949 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
950 Value = ValRes;
951 Chain = Result.getValue(R: 1);
952 break;
953 }
954 }
955 }
956
957 // Since loads produce two values, make sure to remember that we legalized
958 // both of them.
959 if (Chain.getNode() != Node) {
960 assert(Value.getNode() != Node && "Load must be completely replaced");
961 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
962 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
963 if (UpdatedNodes) {
964 UpdatedNodes->insert(X: Value.getNode());
965 UpdatedNodes->insert(X: Chain.getNode());
966 }
967 ReplacedNode(N: Node);
968 }
969}
970
971/// Return a legal replacement for the given operation, with all legal operands.
972void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
973 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
974
975 // Allow illegal target nodes and illegal registers.
976 if (Node->getOpcode() == ISD::TargetConstant ||
977 Node->getOpcode() == ISD::Register)
978 return;
979
980#ifndef NDEBUG
981 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
982 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
983 TargetLowering::TypeLegal &&
984 "Unexpected illegal type!");
985
986 for (const SDValue &Op : Node->op_values())
987 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
988 TargetLowering::TypeLegal ||
989 Op.getOpcode() == ISD::TargetConstant ||
990 Op.getOpcode() == ISD::Register) &&
991 "Unexpected illegal type!");
992#endif
993
994 // Figure out the correct action; the way to query this varies by opcode
995 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
996 bool SimpleFinishLegalizing = true;
997 switch (Node->getOpcode()) {
998 case ISD::POISON: {
999 // TODO: Currently, POISON is being lowered to UNDEF here. However, there is
1000 // an open concern that this transformation may not be ideal, as targets
1001 // should ideally handle POISON directly. Changing this behavior would
1002 // require adding support for POISON in TableGen, which is a large change.
1003 // Additionally, many existing test cases rely on the current behavior
1004 // (e.g., llvm/test/CodeGen/PowerPC/vec_shuffle.ll). A broader discussion
1005 // and incremental changes might be needed to properly support POISON
1006 // without breaking existing targets and tests.
1007 SDValue UndefNode = DAG.getUNDEF(VT: Node->getValueType(ResNo: 0));
1008 ReplaceNode(Old: Node, New: UndefNode.getNode());
1009 return;
1010 }
1011 case ISD::INTRINSIC_W_CHAIN:
1012 case ISD::INTRINSIC_WO_CHAIN:
1013 case ISD::INTRINSIC_VOID:
1014 case ISD::STACKSAVE:
1015 case ISD::STACKADDRESS:
1016 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1017 break;
1018 case ISD::GET_DYNAMIC_AREA_OFFSET:
1019 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1020 VT: Node->getValueType(ResNo: 0));
1021 break;
1022 case ISD::VAARG:
1023 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1024 VT: Node->getValueType(ResNo: 0));
1025 if (Action != TargetLowering::Promote)
1026 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1027 break;
1028 case ISD::SET_FPENV:
1029 case ISD::SET_FPMODE:
1030 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1031 VT: Node->getOperand(Num: 1).getValueType());
1032 break;
1033 case ISD::FP_TO_FP16:
1034 case ISD::FP_TO_BF16:
1035 case ISD::SINT_TO_FP:
1036 case ISD::UINT_TO_FP:
1037 case ISD::EXTRACT_VECTOR_ELT:
1038 case ISD::LROUND:
1039 case ISD::LLROUND:
1040 case ISD::LRINT:
1041 case ISD::LLRINT:
1042 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1043 VT: Node->getOperand(Num: 0).getValueType());
1044 break;
1045 case ISD::STRICT_FP_TO_FP16:
1046 case ISD::STRICT_FP_TO_BF16:
1047 case ISD::STRICT_SINT_TO_FP:
1048 case ISD::STRICT_UINT_TO_FP:
1049 case ISD::STRICT_LRINT:
1050 case ISD::STRICT_LLRINT:
1051 case ISD::STRICT_LROUND:
1052 case ISD::STRICT_LLROUND:
1053 // These pseudo-ops are the same as the other STRICT_ ops except
1054 // they are registered with setOperationAction() using the input type
1055 // instead of the output type.
1056 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1057 VT: Node->getOperand(Num: 1).getValueType());
1058 break;
1059 case ISD::SIGN_EXTEND_INREG: {
1060 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1061 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1062 break;
1063 }
1064 case ISD::ATOMIC_STORE:
1065 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1066 VT: Node->getOperand(Num: 1).getValueType());
1067 break;
1068 case ISD::SELECT_CC:
1069 case ISD::STRICT_FSETCC:
1070 case ISD::STRICT_FSETCCS:
1071 case ISD::SETCC:
1072 case ISD::SETCCCARRY:
1073 case ISD::VP_SETCC:
1074 case ISD::BR_CC: {
1075 unsigned Opc = Node->getOpcode();
1076 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1077 : Opc == ISD::STRICT_FSETCC ? 3
1078 : Opc == ISD::STRICT_FSETCCS ? 3
1079 : Opc == ISD::SETCCCARRY ? 3
1080 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1081 : 1;
1082 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1083 : Opc == ISD::STRICT_FSETCC ? 1
1084 : Opc == ISD::STRICT_FSETCCS ? 1
1085 : 0;
1086 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1087 ISD::CondCode CCCode =
1088 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1089 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1090 if (Action == TargetLowering::Legal) {
1091 if (Node->getOpcode() == ISD::SELECT_CC)
1092 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1093 VT: Node->getValueType(ResNo: 0));
1094 else
1095 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1096 }
1097 break;
1098 }
1099 case ISD::LOAD:
1100 case ISD::STORE:
1101 // FIXME: Model these properly. LOAD and STORE are complicated, and
1102 // STORE expects the unlegalized operand in some cases.
1103 SimpleFinishLegalizing = false;
1104 break;
1105 case ISD::CALLSEQ_START:
1106 case ISD::CALLSEQ_END:
1107 // FIXME: This shouldn't be necessary. These nodes have special properties
1108 // dealing with the recursive nature of legalization. Removing this
1109 // special case should be done as part of making LegalizeDAG non-recursive.
1110 SimpleFinishLegalizing = false;
1111 break;
1112 case ISD::EXTRACT_ELEMENT:
1113 case ISD::GET_ROUNDING:
1114 case ISD::MERGE_VALUES:
1115 case ISD::EH_RETURN:
1116 case ISD::FRAME_TO_ARGS_OFFSET:
1117 case ISD::EH_DWARF_CFA:
1118 case ISD::EH_SJLJ_SETJMP:
1119 case ISD::EH_SJLJ_LONGJMP:
1120 case ISD::EH_SJLJ_SETUP_DISPATCH:
1121 // These operations lie about being legal: when they claim to be legal,
1122 // they should actually be expanded.
1123 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1124 if (Action == TargetLowering::Legal)
1125 Action = TargetLowering::Expand;
1126 break;
1127 case ISD::INIT_TRAMPOLINE:
1128 case ISD::ADJUST_TRAMPOLINE:
1129 case ISD::FRAMEADDR:
1130 case ISD::RETURNADDR:
1131 case ISD::ADDROFRETURNADDR:
1132 case ISD::SPONENTRY:
1133 // These operations lie about being legal: when they claim to be legal,
1134 // they should actually be custom-lowered.
1135 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1136 if (Action == TargetLowering::Legal)
1137 Action = TargetLowering::Custom;
1138 break;
1139 case ISD::CLEAR_CACHE:
1140 // This operation is typically going to be LibCall unless the target wants
1141 // something differrent.
1142 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1143 break;
1144 case ISD::READCYCLECOUNTER:
1145 case ISD::READSTEADYCOUNTER:
1146 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1147 // legalization might have expanded that to several smaller types.
1148 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1149 break;
1150 case ISD::READ_REGISTER:
1151 case ISD::WRITE_REGISTER:
1152 // Named register is legal in the DAG, but blocked by register name
1153 // selection if not implemented by target (to chose the correct register)
1154 // They'll be converted to Copy(To/From)Reg.
1155 Action = TargetLowering::Legal;
1156 break;
1157 case ISD::UBSANTRAP:
1158 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1159 if (Action == TargetLowering::Expand) {
1160 // replace ISD::UBSANTRAP with ISD::TRAP
1161 SDValue NewVal;
1162 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1163 N: Node->getOperand(Num: 0));
1164 ReplaceNode(Old: Node, New: NewVal.getNode());
1165 LegalizeOp(Node: NewVal.getNode());
1166 return;
1167 }
1168 break;
1169 case ISD::DEBUGTRAP:
1170 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1171 if (Action == TargetLowering::Expand) {
1172 // replace ISD::DEBUGTRAP with ISD::TRAP
1173 SDValue NewVal;
1174 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1175 N: Node->getOperand(Num: 0));
1176 ReplaceNode(Old: Node, New: NewVal.getNode());
1177 LegalizeOp(Node: NewVal.getNode());
1178 return;
1179 }
1180 break;
1181 case ISD::SADDSAT:
1182 case ISD::UADDSAT:
1183 case ISD::SSUBSAT:
1184 case ISD::USUBSAT:
1185 case ISD::SSHLSAT:
1186 case ISD::USHLSAT:
1187 case ISD::SCMP:
1188 case ISD::UCMP:
1189 case ISD::FP_TO_SINT_SAT:
1190 case ISD::FP_TO_UINT_SAT:
1191 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1192 break;
1193 case ISD::SMULFIX:
1194 case ISD::SMULFIXSAT:
1195 case ISD::UMULFIX:
1196 case ISD::UMULFIXSAT:
1197 case ISD::SDIVFIX:
1198 case ISD::SDIVFIXSAT:
1199 case ISD::UDIVFIX:
1200 case ISD::UDIVFIXSAT: {
1201 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1202 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1203 VT: Node->getValueType(ResNo: 0), Scale);
1204 break;
1205 }
1206 case ISD::MSCATTER:
1207 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1208 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1209 break;
1210 case ISD::MSTORE:
1211 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1212 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1213 break;
1214 case ISD::VP_SCATTER:
1215 Action = TLI.getOperationAction(
1216 Op: Node->getOpcode(),
1217 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1218 break;
1219 case ISD::VP_STORE:
1220 Action = TLI.getOperationAction(
1221 Op: Node->getOpcode(),
1222 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1223 break;
1224 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1225 Action = TLI.getOperationAction(
1226 Op: Node->getOpcode(),
1227 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1228 break;
1229 case ISD::VECREDUCE_FADD:
1230 case ISD::VECREDUCE_FMUL:
1231 case ISD::VECREDUCE_ADD:
1232 case ISD::VECREDUCE_MUL:
1233 case ISD::VECREDUCE_AND:
1234 case ISD::VECREDUCE_OR:
1235 case ISD::VECREDUCE_XOR:
1236 case ISD::VECREDUCE_SMAX:
1237 case ISD::VECREDUCE_SMIN:
1238 case ISD::VECREDUCE_UMAX:
1239 case ISD::VECREDUCE_UMIN:
1240 case ISD::VECREDUCE_FMAX:
1241 case ISD::VECREDUCE_FMIN:
1242 case ISD::VECREDUCE_FMAXIMUM:
1243 case ISD::VECREDUCE_FMINIMUM:
1244 case ISD::IS_FPCLASS:
1245 Action = TLI.getOperationAction(
1246 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1247 break;
1248 case ISD::VECREDUCE_SEQ_FADD:
1249 case ISD::VECREDUCE_SEQ_FMUL:
1250 case ISD::VP_REDUCE_FADD:
1251 case ISD::VP_REDUCE_FMUL:
1252 case ISD::VP_REDUCE_ADD:
1253 case ISD::VP_REDUCE_MUL:
1254 case ISD::VP_REDUCE_AND:
1255 case ISD::VP_REDUCE_OR:
1256 case ISD::VP_REDUCE_XOR:
1257 case ISD::VP_REDUCE_SMAX:
1258 case ISD::VP_REDUCE_SMIN:
1259 case ISD::VP_REDUCE_UMAX:
1260 case ISD::VP_REDUCE_UMIN:
1261 case ISD::VP_REDUCE_FMAX:
1262 case ISD::VP_REDUCE_FMIN:
1263 case ISD::VP_REDUCE_FMAXIMUM:
1264 case ISD::VP_REDUCE_FMINIMUM:
1265 case ISD::VP_REDUCE_SEQ_FADD:
1266 case ISD::VP_REDUCE_SEQ_FMUL:
1267 Action = TLI.getOperationAction(
1268 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1269 break;
1270 case ISD::CTTZ_ELTS:
1271 case ISD::CTTZ_ELTS_ZERO_POISON:
1272 case ISD::VP_CTTZ_ELTS:
1273 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1274 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1275 VT: Node->getOperand(Num: 0).getValueType());
1276 break;
1277 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1278 Action = TLI.getOperationAction(
1279 Op: Node->getOpcode(),
1280 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1281 break;
1282 default:
1283 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1284 Action = TLI.getCustomOperationAction(Op&: *Node);
1285 } else {
1286 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1287 }
1288 break;
1289 }
1290
1291 if (SimpleFinishLegalizing) {
1292 SDNode *NewNode = Node;
1293 switch (Node->getOpcode()) {
1294 default: break;
1295 case ISD::SHL:
1296 case ISD::SRL:
1297 case ISD::SRA:
1298 case ISD::ROTL:
1299 case ISD::ROTR:
1300 case ISD::SSHLSAT:
1301 case ISD::USHLSAT: {
1302 // Legalizing shifts/rotates requires adjusting the shift amount
1303 // to the appropriate width.
1304 SDValue Op0 = Node->getOperand(Num: 0);
1305 SDValue Op1 = Node->getOperand(Num: 1);
1306 if (!Op1.getValueType().isVector()) {
1307 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1308 // The getShiftAmountOperand() may create a new operand node or
1309 // return the existing one. If new operand is created we need
1310 // to update the parent node.
1311 // Do not try to legalize SAO here! It will be automatically legalized
1312 // in the next round.
1313 if (SAO != Op1)
1314 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1315 }
1316 break;
1317 }
1318 case ISD::FSHL:
1319 case ISD::FSHR:
1320 case ISD::SRL_PARTS:
1321 case ISD::SRA_PARTS:
1322 case ISD::SHL_PARTS: {
1323 // Legalizing shifts/rotates requires adjusting the shift amount
1324 // to the appropriate width.
1325 SDValue Op0 = Node->getOperand(Num: 0);
1326 SDValue Op1 = Node->getOperand(Num: 1);
1327 SDValue Op2 = Node->getOperand(Num: 2);
1328 if (!Op2.getValueType().isVector()) {
1329 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1330 // The getShiftAmountOperand() may create a new operand node or
1331 // return the existing one. If new operand is created we need
1332 // to update the parent node.
1333 if (SAO != Op2)
1334 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1335 }
1336 break;
1337 }
1338 }
1339
1340 if (NewNode != Node) {
1341 ReplaceNode(Old: Node, New: NewNode);
1342 Node = NewNode;
1343 }
1344 switch (Action) {
1345 case TargetLowering::Legal:
1346 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1347 return;
1348 case TargetLowering::Custom:
1349 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1350 // FIXME: The handling for custom lowering with multiple results is
1351 // a complete mess.
1352 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1353 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1354 return;
1355
1356 if (Node->getNumValues() == 1) {
1357 // Verify the new types match the original. Glue is waived because
1358 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1359 assert((Res.getValueType() == Node->getValueType(0) ||
1360 Node->getValueType(0) == MVT::Glue) &&
1361 "Type mismatch for custom legalized operation");
1362 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1363 // We can just directly replace this node with the lowered value.
1364 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1365 return;
1366 }
1367
1368 SmallVector<SDValue, 8> ResultVals;
1369 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1370 // Verify the new types match the original. Glue is waived because
1371 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1372 assert((Res->getValueType(i) == Node->getValueType(i) ||
1373 Node->getValueType(i) == MVT::Glue) &&
1374 "Type mismatch for custom legalized operation");
1375 ResultVals.push_back(Elt: Res.getValue(R: i));
1376 }
1377 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1378 ReplaceNode(Old: Node, New: ResultVals.data());
1379 return;
1380 }
1381 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1382 [[fallthrough]];
1383 case TargetLowering::Expand:
1384 if (ExpandNode(Node))
1385 return;
1386 [[fallthrough]];
1387 case TargetLowering::LibCall:
1388 ConvertNodeToLibcall(Node);
1389 return;
1390 case TargetLowering::Promote:
1391 PromoteNode(Node);
1392 return;
1393 }
1394 }
1395
1396 switch (Node->getOpcode()) {
1397 default:
1398#ifndef NDEBUG
1399 dbgs() << "NODE: ";
1400 Node->dump( &DAG);
1401 dbgs() << "\n";
1402#endif
1403 llvm_unreachable("Do not know how to legalize this operator!");
1404
1405 case ISD::CALLSEQ_START:
1406 case ISD::CALLSEQ_END:
1407 break;
1408 case ISD::LOAD:
1409 return LegalizeLoadOps(Node);
1410 case ISD::STORE:
1411 return LegalizeStoreOps(Node);
1412 }
1413}
1414
1415SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1416 SDValue Vec = Op.getOperand(i: 0);
1417 SDValue Idx = Op.getOperand(i: 1);
1418 SDLoc dl(Op);
1419
1420 // Before we generate a new store to a temporary stack slot, see if there is
1421 // already one that we can use. There often is because when we scalarize
1422 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1423 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1424 // the vector. If all are expanded here, we don't want one store per vector
1425 // element.
1426
1427 // Caches for hasPredecessorHelper
1428 SmallPtrSet<const SDNode *, 32> Visited;
1429 SmallVector<const SDNode *, 16> Worklist;
1430 Visited.insert(Ptr: Op.getNode());
1431 Worklist.push_back(Elt: Idx.getNode());
1432 SDValue StackPtr, Ch;
1433 for (SDNode *User : Vec.getNode()->users()) {
1434 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1435 if (ST->isIndexed() || ST->isTruncatingStore() ||
1436 ST->getValue() != Vec)
1437 continue;
1438
1439 // Make sure that nothing else could have stored into the destination of
1440 // this store.
1441 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1442 continue;
1443
1444 // If the index is dependent on the store we will introduce a cycle when
1445 // creating the load (the load uses the index, and by replacing the chain
1446 // we will make the index dependent on the load). Also, the store might be
1447 // dependent on the extractelement and introduce a cycle when creating
1448 // the load.
1449 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1450 ST->hasPredecessor(N: Op.getNode()))
1451 continue;
1452
1453 StackPtr = ST->getBasePtr();
1454 Ch = SDValue(ST, 0);
1455 break;
1456 }
1457 }
1458
1459 EVT VecVT = Vec.getValueType();
1460
1461 if (!Ch.getNode()) {
1462 // Store the value to a temporary stack slot, then LOAD the returned part.
1463 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1464 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1465 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1466 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1467 }
1468
1469 SDValue NewLoad;
1470 Align ElementAlignment =
1471 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1472 b: DAG.getDataLayout().getPrefTypeAlign(
1473 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1474
1475 if (Op.getValueType().isVector()) {
1476 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1477 SubVecVT: Op.getValueType(), Index: Idx);
1478 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1479 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1480 } else {
1481 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1482 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1483 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1484 Alignment: ElementAlignment);
1485 }
1486
1487 // Replace the chain going out of the store, by the one out of the load.
1488 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1489
1490 // We introduced a cycle though, so update the loads operands, making sure
1491 // to use the original store's chain as an incoming chain.
1492 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1493 NewLoadOperands[0] = Ch;
1494 NewLoad =
1495 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1496 return NewLoad;
1497}
1498
1499SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1500 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1501
1502 SDValue Vec = Op.getOperand(i: 0);
1503 SDValue Part = Op.getOperand(i: 1);
1504 SDValue Idx = Op.getOperand(i: 2);
1505 SDLoc dl(Op);
1506
1507 // Store the value to a temporary stack slot, then LOAD the returned part.
1508 EVT VecVT = Vec.getValueType();
1509 EVT PartVT = Part.getValueType();
1510 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1511 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1512 MachinePointerInfo PtrInfo =
1513 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1514
1515 // First store the whole vector.
1516 Align BaseVecAlignment =
1517 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1518 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1519 Alignment: BaseVecAlignment);
1520
1521 // Freeze the index so we don't poison the clamping code we're about to emit.
1522 Idx = DAG.getFreeze(V: Idx);
1523
1524 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1525 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1526
1527 // Then store the inserted part.
1528 if (PartVT.isVector()) {
1529 SDValue SubStackPtr =
1530 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1531
1532 // Store the subvector.
1533 Ch = DAG.getStore(
1534 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1535 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1536 Alignment: PartAlignment);
1537 } else {
1538 SDValue SubStackPtr =
1539 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1540
1541 // Store the scalar value.
1542 Ch = DAG.getTruncStore(
1543 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1544 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1545 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1546 }
1547
1548 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1549 "ElementAlignment does not match!");
1550
1551 // Finally, load the updated vector.
1552 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1553 Alignment: BaseVecAlignment);
1554}
1555
1556SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1557 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1558 SDLoc DL(Node);
1559 SmallVector<SDValue, 16> Ops;
1560 unsigned NumOperands = Node->getNumOperands();
1561 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1562 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1563 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1564 EVT ElementValueType = TLI.getTypeToTransformTo(
1565 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1566 for (unsigned I = 0; I < NumOperands; ++I) {
1567 SDValue SubOp = Node->getOperand(Num: I);
1568 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1569 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1570 N1: SubOp,
1571 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1572 }
1573 }
1574 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1575}
1576
1577SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1578 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1579 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1580 "Unexpected opcode!");
1581
1582 // We can't handle this case efficiently. Allocate a sufficiently
1583 // aligned object on the stack, store each operand into it, then load
1584 // the result as a vector.
1585 // Create the stack frame object.
1586 EVT VT = Node->getValueType(ResNo: 0);
1587 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1588 : Node->getOperand(Num: 0).getValueType();
1589 SDLoc dl(Node);
1590 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1591 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1592 MachinePointerInfo PtrInfo =
1593 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1594
1595 // Emit a store of each element to the stack slot.
1596 SmallVector<SDValue, 8> Stores;
1597 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1598 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1599
1600 // If the destination vector element type of a BUILD_VECTOR is narrower than
1601 // the source element type, only store the bits necessary.
1602 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1603 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1604
1605 // Store (in the right endianness) the elements to memory.
1606 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1607 // Ignore undef elements.
1608 if (Node->getOperand(Num: i).isUndef()) continue;
1609
1610 unsigned Offset = TypeByteSize*i;
1611
1612 SDValue Idx =
1613 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1614
1615 if (Truncate)
1616 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1617 Val: Node->getOperand(Num: i), Ptr: Idx,
1618 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1619 else
1620 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1621 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1622 }
1623
1624 SDValue StoreChain;
1625 if (!Stores.empty()) // Not all undef elements?
1626 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1627 else
1628 StoreChain = DAG.getEntryNode();
1629
1630 // Result is a load from the stack slot.
1631 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1632}
1633
1634/// Bitcast a floating-point value to an integer value. Only bitcast the part
1635/// containing the sign bit if the target has no integer value capable of
1636/// holding all bits of the floating-point value.
1637void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1638 const SDLoc &DL,
1639 SDValue Value) const {
1640 EVT FloatVT = Value.getValueType();
1641 unsigned NumBits = FloatVT.getScalarSizeInBits();
1642 State.FloatVT = FloatVT;
1643 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1644 // Convert to an integer of the same size.
1645 if (TLI.isTypeLegal(VT: IVT)) {
1646 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1647 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1648 State.SignBit = NumBits - 1;
1649 return;
1650 }
1651
1652 auto &DataLayout = DAG.getDataLayout();
1653 // Store the float to memory, then load the sign part out as an integer.
1654 MVT LoadTy = TLI.getRegisterType(VT: MVT::i8);
1655 // First create a temporary that is aligned for both the load and store.
1656 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1657 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1658 // Then store the float to it.
1659 State.FloatPtr = StackPtr;
1660 MachineFunction &MF = DAG.getMachineFunction();
1661 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1662 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1663 PtrInfo: State.FloatPointerInfo);
1664
1665 SDValue IntPtr;
1666 if (DataLayout.isBigEndian()) {
1667 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1668 // Load out a legal integer with the same sign bit as the float.
1669 IntPtr = StackPtr;
1670 State.IntPointerInfo = State.FloatPointerInfo;
1671 } else {
1672 // Advance the pointer so that the loaded byte will contain the sign bit.
1673 unsigned ByteOffset = (NumBits / 8) - 1;
1674 IntPtr =
1675 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1676 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1677 Offset: ByteOffset);
1678 }
1679
1680 State.IntPtr = IntPtr;
1681 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1682 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1683 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1684 State.SignBit = 7;
1685}
1686
1687/// Replace the integer value produced by getSignAsIntValue() with a new value
1688/// and cast the result back to a floating-point type.
1689SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1690 const SDLoc &DL,
1691 SDValue NewIntValue) const {
1692 if (!State.Chain)
1693 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1694
1695 // Override the part containing the sign bit in the value stored on the stack.
1696 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1697 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1698 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1699 PtrInfo: State.FloatPointerInfo);
1700}
1701
1702SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1703 SDLoc DL(Node);
1704 SDValue Mag = Node->getOperand(Num: 0);
1705 SDValue Sign = Node->getOperand(Num: 1);
1706
1707 // Get sign bit into an integer value.
1708 FloatSignAsInt SignAsInt;
1709 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1710
1711 EVT IntVT = SignAsInt.IntValue.getValueType();
1712 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1713 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1714 N2: SignMask);
1715
1716 // If FABS is legal transform
1717 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1718 EVT FloatVT = Mag.getValueType();
1719 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1720 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1721 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1722 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1723 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1724 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1725 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1726 }
1727
1728 // Transform Mag value to integer, and clear the sign bit.
1729 FloatSignAsInt MagAsInt;
1730 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1731 EVT MagVT = MagAsInt.IntValue.getValueType();
1732 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1733 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1734 N2: ClearSignMask);
1735
1736 // Get the signbit at the right position for MagAsInt.
1737 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1738 EVT ShiftVT = IntVT;
1739 if (SignBit.getScalarValueSizeInBits() <
1740 ClearedSign.getScalarValueSizeInBits()) {
1741 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1742 ShiftVT = MagVT;
1743 }
1744 if (ShiftAmount > 0) {
1745 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1746 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1747 } else if (ShiftAmount < 0) {
1748 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1749 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1750 }
1751 if (SignBit.getScalarValueSizeInBits() >
1752 ClearedSign.getScalarValueSizeInBits()) {
1753 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1754 }
1755
1756 // Store the part with the modified sign and convert back to float.
1757 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1758 Flags: SDNodeFlags::Disjoint);
1759
1760 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1761}
1762
1763SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1764 // Get the sign bit as an integer.
1765 SDLoc DL(Node);
1766 FloatSignAsInt SignAsInt;
1767 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1768 EVT IntVT = SignAsInt.IntValue.getValueType();
1769
1770 // Flip the sign.
1771 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1772 SDValue SignFlip =
1773 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1774
1775 // Convert back to float.
1776 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1777}
1778
1779SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1780 SDLoc DL(Node);
1781 SDValue Value = Node->getOperand(Num: 0);
1782
1783 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1784 EVT FloatVT = Value.getValueType();
1785 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1786 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1787 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1788 }
1789
1790 // Transform value to integer, clear the sign bit and transform back.
1791 FloatSignAsInt ValueAsInt;
1792 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1793 EVT IntVT = ValueAsInt.IntValue.getValueType();
1794 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1795 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1796 N2: ClearSignMask);
1797 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1798}
1799
1800void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1801 SmallVectorImpl<SDValue> &Results) {
1802 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1803 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1804 " not tell us which reg is the stack pointer!");
1805 SDLoc dl(Node);
1806 EVT VT = Node->getValueType(ResNo: 0);
1807 SDValue Tmp1 = SDValue(Node, 0);
1808 SDValue Tmp2 = SDValue(Node, 1);
1809 SDValue Tmp3 = Node->getOperand(Num: 2);
1810 SDValue Chain = Tmp1.getOperand(i: 0);
1811
1812 // Chain the dynamic stack allocation so that it doesn't modify the stack
1813 // pointer when other instructions are using the stack.
1814 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1815
1816 SDValue Size = Tmp2.getOperand(i: 1);
1817 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1818 Chain = SP.getValue(R: 1);
1819 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1820 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1821 unsigned Opc =
1822 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1823 ISD::ADD : ISD::SUB;
1824
1825 Align StackAlign = TFL->getStackAlign();
1826 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1827 if (Alignment > StackAlign)
1828 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1829 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1830 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1831
1832 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1833
1834 Results.push_back(Elt: Tmp1);
1835 Results.push_back(Elt: Tmp2);
1836}
1837
1838/// Emit a store/load combination to the stack. This stores
1839/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1840/// a load from the stack slot to DestVT, extending it if needed.
1841/// The resultant code need not be legal.
1842SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1843 EVT DestVT, const SDLoc &dl) {
1844 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1845}
1846
1847SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1848 EVT DestVT, const SDLoc &dl,
1849 SDValue Chain) {
1850 EVT SrcVT = SrcOp.getValueType();
1851 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1852 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1853
1854 // Don't convert with stack if the load/store is expensive.
1855 if ((SrcVT.bitsGT(VT: SlotVT) &&
1856 !TLI.isTruncStoreLegalOrCustom(ValVT: SrcOp.getValueType(), MemVT: SlotVT)) ||
1857 (SlotVT.bitsLT(VT: DestVT) &&
1858 !TLI.isLoadExtLegalOrCustom(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SlotVT)))
1859 return SDValue();
1860
1861 // Create the stack frame object.
1862 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1863 Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1864 SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1865
1866 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1867 int SPFI = StackPtrFI->getIndex();
1868 MachinePointerInfo PtrInfo =
1869 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1870
1871 // Emit a store to the stack slot. Use a truncstore if the input value is
1872 // later than DestVT.
1873 SDValue Store;
1874
1875 if (SrcVT.bitsGT(VT: SlotVT))
1876 Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1877 SVT: SlotVT, Alignment: SrcAlign);
1878 else {
1879 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1880 Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1881 }
1882
1883 // Result is a load from the stack slot.
1884 if (SlotVT.bitsEq(VT: DestVT))
1885 return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1886
1887 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1888 return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1889 Alignment: DestAlign);
1890}
1891
1892SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1893 SDLoc dl(Node);
1894 // Create a vector sized/aligned stack slot, store the value to element #0,
1895 // then load the whole vector back out.
1896 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1897
1898 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1899 int SPFI = StackPtrFI->getIndex();
1900
1901 SDValue Ch = DAG.getTruncStore(
1902 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1903 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1904 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1905 return DAG.getLoad(
1906 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1907 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1908}
1909
1910static bool
1911ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1912 const TargetLowering &TLI, SDValue &Res) {
1913 unsigned NumElems = Node->getNumOperands();
1914 SDLoc dl(Node);
1915 EVT VT = Node->getValueType(ResNo: 0);
1916
1917 // Try to group the scalars into pairs, shuffle the pairs together, then
1918 // shuffle the pairs of pairs together, etc. until the vector has
1919 // been built. This will work only if all of the necessary shuffle masks
1920 // are legal.
1921
1922 // We do this in two phases; first to check the legality of the shuffles,
1923 // and next, assuming that all shuffles are legal, to create the new nodes.
1924 for (int Phase = 0; Phase < 2; ++Phase) {
1925 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1926 NewIntermedVals;
1927 for (unsigned i = 0; i < NumElems; ++i) {
1928 SDValue V = Node->getOperand(Num: i);
1929 if (V.isUndef())
1930 continue;
1931
1932 SDValue Vec;
1933 if (Phase)
1934 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1935 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1936 }
1937
1938 while (IntermedVals.size() > 2) {
1939 NewIntermedVals.clear();
1940 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1941 // This vector and the next vector are shuffled together (simply to
1942 // append the one to the other).
1943 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1944
1945 SmallVector<int, 16> FinalIndices;
1946 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1947 IntermedVals[i+1].second.size());
1948
1949 int k = 0;
1950 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1951 ++j, ++k) {
1952 ShuffleVec[k] = j;
1953 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1954 }
1955 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1956 ++j, ++k) {
1957 ShuffleVec[k] = NumElems + j;
1958 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1959 }
1960
1961 SDValue Shuffle;
1962 if (Phase)
1963 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1964 N2: IntermedVals[i+1].first,
1965 Mask: ShuffleVec);
1966 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1967 return false;
1968 NewIntermedVals.push_back(
1969 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1970 }
1971
1972 // If we had an odd number of defined values, then append the last
1973 // element to the array of new vectors.
1974 if ((IntermedVals.size() & 1) != 0)
1975 NewIntermedVals.push_back(Elt: IntermedVals.back());
1976
1977 IntermedVals.swap(RHS&: NewIntermedVals);
1978 }
1979
1980 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1981 "Invalid number of intermediate vectors");
1982 SDValue Vec1 = IntermedVals[0].first;
1983 SDValue Vec2;
1984 if (IntermedVals.size() > 1)
1985 Vec2 = IntermedVals[1].first;
1986 else if (Phase)
1987 Vec2 = DAG.getPOISON(VT);
1988
1989 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1990 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1991 ShuffleVec[IntermedVals[0].second[i]] = i;
1992 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1993 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1994
1995 if (Phase)
1996 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1997 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1998 return false;
1999 }
2000
2001 return true;
2002}
2003
2004/// Expand a BUILD_VECTOR node on targets that don't
2005/// support the operation, but do support the resultant vector type.
2006SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2007 unsigned NumElems = Node->getNumOperands();
2008 SDValue Value1, Value2;
2009 SDLoc dl(Node);
2010 EVT VT = Node->getValueType(ResNo: 0);
2011 EVT OpVT = Node->getOperand(Num: 0).getValueType();
2012 EVT EltVT = VT.getVectorElementType();
2013
2014 // If the only non-undef value is the low element, turn this into a
2015 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
2016 bool isOnlyLowElement = true;
2017 bool MoreThanTwoValues = false;
2018 bool isConstant = true;
2019 for (unsigned i = 0; i < NumElems; ++i) {
2020 SDValue V = Node->getOperand(Num: i);
2021 if (V.isUndef())
2022 continue;
2023 if (i > 0)
2024 isOnlyLowElement = false;
2025 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2026 isConstant = false;
2027
2028 if (!Value1.getNode()) {
2029 Value1 = V;
2030 } else if (!Value2.getNode()) {
2031 if (V != Value1)
2032 Value2 = V;
2033 } else if (V != Value1 && V != Value2) {
2034 MoreThanTwoValues = true;
2035 }
2036 }
2037
2038 if (!Value1.getNode())
2039 return DAG.getUNDEF(VT);
2040
2041 if (isOnlyLowElement)
2042 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2043
2044 // If all elements are constants, create a load from the constant pool.
2045 if (isConstant) {
2046 SmallVector<Constant*, 16> CV;
2047 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2048 if (ConstantFPSDNode *V =
2049 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2050 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2051 } else if (ConstantSDNode *V =
2052 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2053 if (OpVT==EltVT)
2054 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2055 else {
2056 // If OpVT and EltVT don't match, EltVT is not legal and the
2057 // element values have been promoted/truncated earlier. Undo this;
2058 // we don't want a v16i8 to become a v16i32 for example.
2059 const ConstantInt *CI = V->getConstantIntValue();
2060 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2061 V: CI->getZExtValue(), /*IsSigned=*/false,
2062 /*ImplicitTrunc=*/true));
2063 }
2064 } else {
2065 assert(Node->getOperand(i).isUndef());
2066 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2067 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2068 }
2069 }
2070 Constant *CP = ConstantVector::get(V: CV);
2071 SDValue CPIdx =
2072 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2073 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2074 return DAG.getLoad(
2075 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2076 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2077 Alignment);
2078 }
2079
2080 SmallSet<SDValue, 16> DefinedValues;
2081 for (unsigned i = 0; i < NumElems; ++i) {
2082 if (Node->getOperand(Num: i).isUndef())
2083 continue;
2084 DefinedValues.insert(V: Node->getOperand(Num: i));
2085 }
2086
2087 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2088 if (!MoreThanTwoValues) {
2089 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2090 for (unsigned i = 0; i < NumElems; ++i) {
2091 SDValue V = Node->getOperand(Num: i);
2092 if (V.isUndef())
2093 continue;
2094 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2095 }
2096 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2097 // Get the splatted value into the low element of a vector register.
2098 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2099 SDValue Vec2;
2100 if (Value2.getNode())
2101 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2102 else
2103 Vec2 = DAG.getPOISON(VT);
2104
2105 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2106 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2107 }
2108 } else {
2109 SDValue Res;
2110 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2111 return Res;
2112 }
2113 }
2114
2115 // Otherwise, we can't handle this case efficiently.
2116 return ExpandVectorBuildThroughStack(Node);
2117}
2118
2119SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2120 SDLoc DL(Node);
2121 EVT VT = Node->getValueType(ResNo: 0);
2122 SDValue SplatVal = Node->getOperand(Num: 0);
2123
2124 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2125}
2126
2127// Expand a node into a call to a libcall, returning the value as the first
2128// result and the chain as the second. If the result value does not fit into a
2129// register, return the lo part and set the hi part to the by-reg argument in
2130// the first. If it does fit into a single register, return the result and
2131// leave the Hi part unset.
2132std::pair<SDValue, SDValue>
2133SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2134 TargetLowering::ArgListTy &&Args,
2135 bool IsSigned, EVT RetVT) {
2136 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2137 SDValue Callee;
2138 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2139 if (LCImpl != RTLIB::Unsupported)
2140 Callee = DAG.getExternalSymbol(LCImpl, VT: CodePtrTy);
2141 else {
2142 Callee = DAG.getPOISON(VT: CodePtrTy);
2143 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2144 Node->getOperationName(G: &DAG));
2145 }
2146
2147 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2148
2149 // By default, the input chain to this libcall is the entry node of the
2150 // function. If the libcall is going to be emitted as a tail call then
2151 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2152 // node which is being folded has a non-entry input chain.
2153 SDValue InChain = DAG.getEntryNode();
2154
2155 // isTailCall may be true since the callee does not reference caller stack
2156 // frame. Check if it's in the right position and that the return types match.
2157 SDValue TCChain = InChain;
2158 const Function &F = DAG.getMachineFunction().getFunction();
2159 bool isTailCall =
2160 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2161 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2162 if (isTailCall)
2163 InChain = TCChain;
2164
2165 TargetLowering::CallLoweringInfo CLI(DAG);
2166 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2167 CLI.setDebugLoc(SDLoc(Node))
2168 .setChain(InChain)
2169 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
2170 Target: Callee, ArgsList: std::move(Args))
2171 .setTailCall(isTailCall)
2172 .setSExtResult(signExtend)
2173 .setZExtResult(!signExtend)
2174 .setIsPostTypeLegalization(true);
2175
2176 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2177
2178 if (!CallInfo.second.getNode()) {
2179 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2180 // It's a tailcall, return the chain (which is the DAG root).
2181 return {DAG.getRoot(), DAG.getRoot()};
2182 }
2183
2184 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2185 return CallInfo;
2186}
2187
2188std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2189 bool isSigned) {
2190 TargetLowering::ArgListTy Args;
2191 for (const SDValue &Op : Node->op_values()) {
2192 EVT ArgVT = Op.getValueType();
2193 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2194 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2195 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2196 Entry.IsZExt = !Entry.IsSExt;
2197 Args.push_back(x: Entry);
2198 }
2199
2200 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2201 RetVT: Node->getValueType(ResNo: 0));
2202}
2203
2204void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2205 RTLIB::Libcall LC,
2206 SmallVectorImpl<SDValue> &Results) {
2207 if (LC == RTLIB::UNKNOWN_LIBCALL)
2208 llvm_unreachable("Can't create an unknown libcall!");
2209
2210 if (Node->isStrictFPOpcode()) {
2211 EVT RetVT = Node->getValueType(ResNo: 0);
2212 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2213 TargetLowering::MakeLibCallOptions CallOptions;
2214 CallOptions.IsPostTypeLegalization = true;
2215 // FIXME: This doesn't support tail calls.
2216 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2217 Ops, CallOptions,
2218 dl: SDLoc(Node),
2219 Chain: Node->getOperand(Num: 0));
2220 Results.push_back(Elt: Tmp.first);
2221 Results.push_back(Elt: Tmp.second);
2222 } else {
2223 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2224 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2225 Results.push_back(Elt: Tmp);
2226 }
2227}
2228
2229/// Expand the node to a libcall based on the result type.
2230void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2231 RTLIB::Libcall Call_F32,
2232 RTLIB::Libcall Call_F64,
2233 RTLIB::Libcall Call_F80,
2234 RTLIB::Libcall Call_F128,
2235 RTLIB::Libcall Call_PPCF128,
2236 SmallVectorImpl<SDValue> &Results) {
2237 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0),
2238 Call_F32, Call_F64, Call_F80,
2239 Call_F128, Call_PPCF128);
2240 ExpandFPLibCall(Node, LC, Results);
2241}
2242
2243void SelectionDAGLegalize::ExpandFastFPLibCall(
2244 SDNode *Node, bool IsFast,
2245 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2246 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2247 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2248 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2249 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2250 SmallVectorImpl<SDValue> &Results) {
2251
2252 EVT VT = Node->getSimpleValueType(ResNo: 0);
2253
2254 RTLIB::Libcall LC;
2255
2256 // FIXME: Probably should define fast to respect nan/inf and only be
2257 // approximate functions.
2258
2259 if (IsFast) {
2260 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.first, Call_F64: Call_F64.first, Call_F80: Call_F80.first,
2261 Call_F128: Call_F128.first, Call_PPCF128: Call_PPCF128.first);
2262 }
2263
2264 if (!IsFast || DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
2265 // Fall back if we don't have a fast implementation.
2266 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.second, Call_F64: Call_F64.second,
2267 Call_F80: Call_F80.second, Call_F128: Call_F128.second,
2268 Call_PPCF128: Call_PPCF128.second);
2269 }
2270
2271 ExpandFPLibCall(Node, LC, Results);
2272}
2273
2274SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2275 RTLIB::Libcall Call_I8,
2276 RTLIB::Libcall Call_I16,
2277 RTLIB::Libcall Call_I32,
2278 RTLIB::Libcall Call_I64,
2279 RTLIB::Libcall Call_I128) {
2280 RTLIB::Libcall LC;
2281 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2282 default: llvm_unreachable("Unexpected request for libcall!");
2283 case MVT::i8: LC = Call_I8; break;
2284 case MVT::i16: LC = Call_I16; break;
2285 case MVT::i32: LC = Call_I32; break;
2286 case MVT::i64: LC = Call_I64; break;
2287 case MVT::i128: LC = Call_I128; break;
2288 }
2289 return ExpandLibCall(LC, Node, isSigned).first;
2290}
2291
2292/// Expand the node to a libcall based on first argument type (for instance
2293/// lround and its variant).
2294void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2295 RTLIB::Libcall Call_F32,
2296 RTLIB::Libcall Call_F64,
2297 RTLIB::Libcall Call_F80,
2298 RTLIB::Libcall Call_F128,
2299 RTLIB::Libcall Call_PPCF128,
2300 SmallVectorImpl<SDValue> &Results) {
2301 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2302 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2303 Call_F32, Call_F64, Call_F80,
2304 Call_F128, Call_PPCF128);
2305 ExpandFPLibCall(Node, LC, Results);
2306}
2307
2308SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2309 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2310 RTLIB::Libcall CallI128) {
2311 RTLIB::Libcall LC;
2312 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2313 default:
2314 llvm_unreachable("Unexpected request for libcall!");
2315 case MVT::i32:
2316 LC = CallI32;
2317 break;
2318 case MVT::i64:
2319 LC = CallI64;
2320 break;
2321 case MVT::i128:
2322 LC = CallI128;
2323 break;
2324 }
2325
2326 // Bit-counting libcalls have one unsigned argument and return `int`.
2327 // Note that `int` may be illegal on this target; ExpandLibCall will
2328 // take care of promoting it to a legal type.
2329 SDValue Op = Node->getOperand(Num: 0);
2330 EVT IntVT =
2331 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2332
2333 EVT ArgVT = Op.getValueType();
2334 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2335 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2336 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2337 Arg.IsZExt = !Arg.IsSExt;
2338
2339 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2340 /*IsSigned=*/true, RetVT: IntVT)
2341 .first;
2342
2343 // If ExpandLibCall created a tail call, the result was already
2344 // of the correct type. Otherwise, we need to sign extend it.
2345 if (Res.getValueType() != MVT::Other)
2346 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2347 return Res;
2348}
2349
2350/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2351void
2352SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2353 SmallVectorImpl<SDValue> &Results) {
2354 unsigned Opcode = Node->getOpcode();
2355 bool isSigned = Opcode == ISD::SDIVREM;
2356
2357 RTLIB::Libcall LC;
2358 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2359 default: llvm_unreachable("Unexpected request for libcall!");
2360 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2361 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2362 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2363 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2364 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2365 }
2366
2367 // The input chain to this libcall is the entry node of the function.
2368 // Legalizing the call will automatically add the previous call to the
2369 // dependence.
2370 SDValue InChain = DAG.getEntryNode();
2371
2372 EVT RetVT = Node->getValueType(ResNo: 0);
2373 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2374
2375 TargetLowering::ArgListTy Args;
2376 for (const SDValue &Op : Node->op_values()) {
2377 EVT ArgVT = Op.getValueType();
2378 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2379 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2380 Entry.IsSExt = isSigned;
2381 Entry.IsZExt = !isSigned;
2382 Args.push_back(x: Entry);
2383 }
2384
2385 // Also pass the return address of the remainder.
2386 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2387 TargetLowering::ArgListEntry Entry(
2388 FIPtr, PointerType::getUnqual(C&: RetTy->getContext()));
2389 Entry.IsSExt = isSigned;
2390 Entry.IsZExt = !isSigned;
2391 Args.push_back(x: Entry);
2392
2393 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2394 if (LibcallImpl == RTLIB::Unsupported) {
2395 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2396 Node->getOperationName(G: &DAG));
2397 SDValue Poison = DAG.getPOISON(VT: RetVT);
2398 Results.push_back(Elt: Poison);
2399 Results.push_back(Elt: Poison);
2400 return;
2401 }
2402
2403 SDValue Callee =
2404 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2405
2406 SDLoc dl(Node);
2407 TargetLowering::CallLoweringInfo CLI(DAG);
2408 CLI.setDebugLoc(dl)
2409 .setChain(InChain)
2410 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
2411 ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2412 .setSExtResult(isSigned)
2413 .setZExtResult(!isSigned);
2414
2415 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2416
2417 // Remainder is loaded back from the stack frame.
2418 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
2419 MachinePointerInfo PtrInfo =
2420 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2421
2422 SDValue Rem = DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo);
2423 Results.push_back(Elt: CallInfo.first);
2424 Results.push_back(Elt: Rem);
2425}
2426
2427/// Return true if sincos or __sincos_stret libcall is available.
2428static bool isSinCosLibcallAvailable(SDNode *Node,
2429 const LibcallLoweringInfo &Libcalls) {
2430 MVT::SimpleValueType VT = Node->getSimpleValueType(ResNo: 0).SimpleTy;
2431 return Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS(RetVT: VT)) != RTLIB::Unsupported ||
2432 Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS_STRET(RetVT: VT)) !=
2433 RTLIB::Unsupported;
2434}
2435
2436/// Only issue sincos libcall if both sin and cos are needed.
2437static bool useSinCos(SDNode *Node) {
2438 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2439 ? ISD::FCOS : ISD::FSIN;
2440
2441 SDValue Op0 = Node->getOperand(Num: 0);
2442 for (const SDNode *User : Op0.getNode()->users()) {
2443 if (User == Node)
2444 continue;
2445 // The other user might have been turned into sincos already.
2446 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2447 return true;
2448 }
2449 return false;
2450}
2451
2452SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2453 // For iOS, we want to call an alternative entry point: __sincos_stret,
2454 // which returns the values in two S / D registers.
2455 SDLoc dl(Node);
2456 SDValue Arg = Node->getOperand(Num: 0);
2457 EVT ArgVT = Arg.getValueType();
2458 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(RetVT: ArgVT);
2459 RTLIB::LibcallImpl SincosStret = DAG.getLibcalls().getLibcallImpl(Call: LC);
2460 if (SincosStret == RTLIB::Unsupported)
2461 return SDValue();
2462
2463 /// There are 3 different ABI cases to handle:
2464 /// - Direct return of separate fields in registers
2465 /// - Single return as vector elements
2466 /// - sret struct
2467
2468 const RTLIB::RuntimeLibcallsInfo &CallsInfo = TLI.getRuntimeLibcallsInfo();
2469
2470 const DataLayout &DL = DAG.getDataLayout();
2471
2472 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2473 Ctx&: *DAG.getContext(), TT: TM.getTargetTriple(), DL, LibcallImpl: SincosStret);
2474
2475 Type *SincosStretRetTy = FuncTy->getReturnType();
2476 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(Call: SincosStret);
2477
2478 SDValue Callee =
2479 DAG.getExternalSymbol(LCImpl: SincosStret, VT: TLI.getProgramPointerTy(DL));
2480
2481 TargetLowering::ArgListTy Args;
2482 SDValue SRet;
2483
2484 int FrameIdx;
2485 if (FuncTy->getParamType(i: 0)->isPointerTy()) {
2486 // Uses sret
2487 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2488
2489 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(ArgNo: 0);
2490 Type *StructTy = PtrAttrs.getStructRetType();
2491 const uint64_t ByteSize = DL.getTypeAllocSize(Ty: StructTy);
2492 const Align StackAlign = DL.getPrefTypeAlign(Ty: StructTy);
2493
2494 FrameIdx = MFI.CreateStackObject(Size: ByteSize, Alignment: StackAlign, isSpillSlot: false);
2495 SRet = DAG.getFrameIndex(FI: FrameIdx, VT: TLI.getFrameIndexTy(DL));
2496
2497 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(i: 0));
2498 Entry.IsSRet = true;
2499 Entry.IndirectType = StructTy;
2500 Entry.Alignment = StackAlign;
2501
2502 Args.push_back(x: Entry);
2503 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 1));
2504 } else {
2505 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 0));
2506 }
2507
2508 TargetLowering::CallLoweringInfo CLI(DAG);
2509 CLI.setDebugLoc(dl)
2510 .setChain(DAG.getEntryNode())
2511 .setLibCallee(CC: CallConv, ResultType: SincosStretRetTy, Target: Callee, ArgsList: std::move(Args))
2512 .setIsPostTypeLegalization();
2513
2514 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2515
2516 if (SRet) {
2517 MachinePointerInfo PtrInfo =
2518 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2519 SDValue LoadSin = DAG.getLoad(VT: ArgVT, dl, Chain: CallResult.second, Ptr: SRet, PtrInfo);
2520
2521 TypeSize StoreSize = ArgVT.getStoreSize();
2522
2523 // Address of cos field.
2524 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr: SRet, Offset: StoreSize);
2525 SDValue LoadCos = DAG.getLoad(VT: ArgVT, dl, Chain: LoadSin.getValue(R: 1), Ptr: Add,
2526 PtrInfo: PtrInfo.getWithOffset(O: StoreSize));
2527
2528 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2529 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: LoadSin.getValue(R: 0),
2530 N2: LoadCos.getValue(R: 0));
2531 }
2532
2533 if (!CallResult.first.getValueType().isVector())
2534 return CallResult.first;
2535
2536 SDValue SinVal =
2537 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2538 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2539 SDValue CosVal =
2540 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2541 N2: DAG.getVectorIdxConstant(Val: 1, DL: dl));
2542 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2543 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: SinVal, N2: CosVal);
2544}
2545
2546SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2547 SDLoc dl(Node);
2548 EVT VT = Node->getValueType(ResNo: 0);
2549 SDValue X = Node->getOperand(Num: 0);
2550 SDValue N = Node->getOperand(Num: 1);
2551 EVT ExpVT = N.getValueType();
2552 EVT AsIntVT = VT.changeTypeToInteger();
2553 if (AsIntVT == EVT()) // TODO: How to handle f80?
2554 return SDValue();
2555
2556 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2557 return SDValue();
2558
2559 SDNodeFlags NSW;
2560 NSW.setNoSignedWrap(true);
2561 SDNodeFlags NUW_NSW;
2562 NUW_NSW.setNoUnsignedWrap(true);
2563 NUW_NSW.setNoSignedWrap(true);
2564
2565 EVT SetCCVT =
2566 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2567 const fltSemantics &FltSem = VT.getFltSemantics();
2568
2569 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2570 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2571 const int Precision = APFloat::semanticsPrecision(FltSem);
2572
2573 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2574 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2575
2576 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2577
2578 const APFloat One(FltSem, "1.0");
2579 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2580
2581 // Offset by precision to avoid denormal range.
2582 APFloat ScaleDownK =
2583 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2584
2585 // TODO: Should really introduce control flow and use a block for the >
2586 // MaxExp, < MinExp cases
2587
2588 // First, handle exponents Exp > MaxExp and scale down.
2589 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2590
2591 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2592 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2593 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2594 SDValue DecN1 =
2595 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2596
2597 SDValue ScaleUpTwice =
2598 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2599
2600 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2601 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2602 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2603
2604 SDValue SelectN_Big =
2605 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2606 SDValue SelectX_Big =
2607 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2608
2609 // Now handle exponents Exp < MinExp
2610 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2611
2612 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2613 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2614
2615 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2616
2617 SDValue ClampMinVal =
2618 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2619 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2620 SDValue IncN1 =
2621 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2622
2623 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2624 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2625 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2626
2627 SDValue ScaleDownTwice = DAG.getSetCC(
2628 DL: dl, VT: SetCCVT, LHS: N,
2629 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2630
2631 SDValue SelectN_Small =
2632 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2633 SDValue SelectX_Small =
2634 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2635
2636 // Now combine the two out of range exponent handling cases with the base
2637 // case.
2638 SDValue NewX = DAG.getNode(
2639 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2640 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2641
2642 SDValue NewN = DAG.getNode(
2643 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2644 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2645
2646 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2647
2648 SDValue ExponentShiftAmt =
2649 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2650 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2651
2652 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2653 N2: ExponentShiftAmt, Flags: NUW_NSW);
2654 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2655 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2656}
2657
2658SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2659 SDLoc dl(Node);
2660 SDValue Val = Node->getOperand(Num: 0);
2661 EVT VT = Val.getValueType();
2662 EVT ExpVT = Node->getValueType(ResNo: 1);
2663 EVT AsIntVT = VT.changeTypeToInteger();
2664 if (AsIntVT == EVT()) // TODO: How to handle f80?
2665 return SDValue();
2666
2667 const fltSemantics &FltSem = VT.getFltSemantics();
2668 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2669 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2670 const unsigned BitSize = VT.getScalarSizeInBits();
2671
2672 // TODO: Could introduce control flow and skip over the denormal handling.
2673
2674 // scale_up = fmul value, scalbn(1.0, precision + 1)
2675 // extracted_exp = (bitcast value to uint) >> precision - 1
2676 // biased_exp = extracted_exp + min_exp
2677 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2678 //
2679 // is_denormal = val < smallest_normalized
2680 // computed_fract = is_denormal ? scale_up : extracted_fract
2681 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2682 //
2683 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2684 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2685
2686 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2687 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2688 VT: AsIntVT);
2689
2690 SDValue SmallestNormalizedInt = DAG.getConstant(
2691 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2692 VT: AsIntVT);
2693
2694 // Masks out the exponent bits.
2695 SDValue ExpMask =
2696 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2697
2698 // Mask out the exponent part of the value.
2699 //
2700 // e.g, for f32 FractSignMaskVal = 0x807fffff
2701 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2702 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2703
2704 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2705 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2706
2707 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2708
2709 const APFloat One(FltSem, "1.0");
2710 // Scale a possible denormal input.
2711 // e.g., for f64, 0x1p+54
2712 APFloat ScaleUpKVal =
2713 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2714
2715 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2716 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2717
2718 EVT SetCCVT =
2719 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2720
2721 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2722
2723 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2724
2725 SDValue AddNegSmallestNormal =
2726 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2727 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2728 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2729
2730 SDValue IsDenormal =
2731 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2732
2733 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2734 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2735
2736 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2737 SDValue ScaledSelect =
2738 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2739
2740 SDValue ExpMaskScaled =
2741 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2742
2743 SDValue ScaledValue =
2744 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2745
2746 // Extract the exponent bits.
2747 SDValue ExponentShiftAmt =
2748 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2749 SDValue ShiftedExp =
2750 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2751 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2752
2753 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2754 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2755 SDValue DenormalExpBias =
2756 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2757
2758 SDValue MaskedFractAsInt =
2759 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2760 const APFloat Half(FltSem, "0.5");
2761 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2762 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2763 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2764
2765 SDValue ComputedExp =
2766 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2767
2768 SDValue Result0 =
2769 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2770
2771 SDValue Result1 =
2772 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2773
2774 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2775}
2776
2777SDValue SelectionDAGLegalize::expandModf(SDNode *Node) const {
2778 SDLoc dl(Node);
2779 SDValue Val = Node->getOperand(Num: 0);
2780 EVT VT = Val.getValueType();
2781 SDNodeFlags Flags = Node->getFlags();
2782
2783 SDValue IntPart = DAG.getNode(Opcode: ISD::FTRUNC, DL: dl, VT, Operand: Val, Flags);
2784 SDValue FracPart = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT, N1: Val, N2: IntPart, Flags);
2785
2786 SDValue FracToUse;
2787 if (Flags.hasNoInfs()) {
2788 FracToUse = FracPart;
2789 } else {
2790 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val, Flags);
2791 SDValue Inf =
2792 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
2793 EVT SetCCVT =
2794 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2795 SDValue IsInf = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: Inf, Cond: ISD::SETOEQ);
2796 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: dl, VT);
2797 FracToUse = DAG.getSelect(DL: dl, VT, Cond: IsInf, LHS: Zero, RHS: FracPart);
2798 }
2799
2800 SDValue ResultFrac =
2801 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT, N1: FracToUse, N2: Val, Flags);
2802 return DAG.getMergeValues(Ops: {ResultFrac, IntPart}, dl);
2803}
2804
2805/// This function is responsible for legalizing a
2806/// INT_TO_FP operation of the specified operand when the target requests that
2807/// we expand it. At this point, we know that the result and operand types are
2808/// legal for the target.
2809SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2810 SDValue &Chain) {
2811 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2812 Node->getOpcode() == ISD::SINT_TO_FP);
2813 EVT DestVT = Node->getValueType(ResNo: 0);
2814 SDLoc dl(Node);
2815 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2816 SDValue Op0 = Node->getOperand(Num: OpNo);
2817 EVT SrcVT = Op0.getValueType();
2818
2819 // TODO: Should any fast-math-flags be set for the created nodes?
2820 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2821 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2822 (DestVT.bitsLE(VT: MVT::f64) ||
2823 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2824 : ISD::FP_EXTEND,
2825 VT: DestVT))) {
2826 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2827 "expansion\n");
2828
2829 // Get the stack frame index of a 8 byte buffer.
2830 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2831
2832 SDValue Lo = Op0;
2833 // if signed map to unsigned space
2834 if (isSigned) {
2835 // Invert sign bit (signed to unsigned mapping).
2836 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2837 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2838 }
2839 // Initial hi portion of constructed double.
2840 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2841
2842 // If this a big endian target, swap the lo and high data.
2843 if (DAG.getDataLayout().isBigEndian())
2844 std::swap(a&: Lo, b&: Hi);
2845
2846 SDValue MemChain = DAG.getEntryNode();
2847
2848 // Store the lo of the constructed double.
2849 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2850 PtrInfo: MachinePointerInfo());
2851 // Store the hi of the constructed double.
2852 SDValue HiPtr =
2853 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2854 SDValue Store2 =
2855 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2856 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2857
2858 // load the constructed double
2859 SDValue Load =
2860 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2861 // FP constant to bias correct the final result
2862 SDValue Bias = DAG.getConstantFP(
2863 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2864 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2865 DL: dl, VT: MVT::f64);
2866 // Subtract the bias and get the final result.
2867 SDValue Sub;
2868 SDValue Result;
2869 if (Node->isStrictFPOpcode()) {
2870 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2871 Ops: {Node->getOperand(Num: 0), Load, Bias});
2872 Chain = Sub.getValue(R: 1);
2873 if (DestVT != Sub.getValueType()) {
2874 std::pair<SDValue, SDValue> ResultPair;
2875 ResultPair =
2876 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2877 Result = ResultPair.first;
2878 Chain = ResultPair.second;
2879 }
2880 else
2881 Result = Sub;
2882 } else {
2883 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2884 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2885 }
2886 return Result;
2887 }
2888
2889 if (isSigned)
2890 return SDValue();
2891
2892 // TODO: Generalize this for use with other types.
2893 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2894 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2895 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2896 // For unsigned conversions, convert them to signed conversions using the
2897 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2898 // should be valid for i32->f32 as well.
2899
2900 // More generally this transform should be valid if there are 3 more bits
2901 // in the integer type than the significand. Rounding uses the first bit
2902 // after the width of the significand and the OR of all bits after that. So
2903 // we need to be able to OR the shifted out bit into one of the bits that
2904 // participate in the OR.
2905
2906 // TODO: This really should be implemented using a branch rather than a
2907 // select. We happen to get lucky and machinesink does the right
2908 // thing most of the time. This would be a good candidate for a
2909 // pseudo-op, or, even better, for whole-function isel.
2910 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2911
2912 SDValue SignBitTest = DAG.getSetCC(
2913 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2914
2915 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL: dl);
2916 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2917 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2918 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2919 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2920
2921 SDValue Slow, Fast;
2922 if (Node->isStrictFPOpcode()) {
2923 // In strict mode, we must avoid spurious exceptions, and therefore
2924 // must make sure to only emit a single STRICT_SINT_TO_FP.
2925 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2926 // The STRICT_SINT_TO_FP inherits the exception mode from the
2927 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2928 // never raise any exception.
2929 SDNodeFlags Flags;
2930 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2931 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: {DestVT, MVT::Other},
2932 Ops: {Node->getOperand(Num: 0), InCvt}, Flags);
2933 Flags.setNoFPExcept(true);
2934 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: {DestVT, MVT::Other},
2935 Ops: {Fast.getValue(R: 1), Fast, Fast}, Flags);
2936 Chain = Slow.getValue(R: 1);
2937 } else {
2938 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2939 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2940 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2941 }
2942
2943 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2944 }
2945
2946 // Don't expand it if there isn't cheap fadd.
2947 if (!TLI.isOperationLegalOrCustom(
2948 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2949 return SDValue();
2950
2951 // The following optimization is valid only if every value in SrcVT (when
2952 // treated as signed) is representable in DestVT. Check that the mantissa
2953 // size of DestVT is >= than the number of bits in SrcVT -1.
2954 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2955 SrcVT.getSizeInBits() - 1 &&
2956 "Cannot perform lossless SINT_TO_FP!");
2957
2958 SDValue Tmp1;
2959 if (Node->isStrictFPOpcode()) {
2960 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2961 Ops: { Node->getOperand(Num: 0), Op0 });
2962 } else
2963 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2964
2965 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2966 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2967 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2968 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2969 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2970 Cond: SignSet, LHS: Four, RHS: Zero);
2971
2972 // If the sign bit of the integer is set, the large number will be treated
2973 // as a negative number. To counteract this, the dynamic code adds an
2974 // offset depending on the data type.
2975 uint64_t FF;
2976 switch (SrcVT.getSimpleVT().SimpleTy) {
2977 default:
2978 return SDValue();
2979 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2980 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2981 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2982 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2983 }
2984 if (DAG.getDataLayout().isLittleEndian())
2985 FF <<= 32;
2986 Constant *FudgeFactor = ConstantInt::get(
2987 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2988
2989 SDValue CPIdx =
2990 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2991 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2992 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2993 Alignment = commonAlignment(A: Alignment, Offset: 4);
2994 SDValue FudgeInReg;
2995 if (DestVT == MVT::f32)
2996 FudgeInReg = DAG.getLoad(
2997 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2998 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2999 Alignment);
3000 else {
3001 SDValue Load = DAG.getExtLoad(
3002 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
3003 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
3004 Alignment);
3005 HandleSDNode Handle(Load);
3006 LegalizeOp(Node: Load.getNode());
3007 FudgeInReg = Handle.getValue();
3008 }
3009
3010 if (Node->isStrictFPOpcode()) {
3011 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
3012 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
3013 Chain = Result.getValue(R: 1);
3014 return Result;
3015 }
3016
3017 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
3018}
3019
3020/// This function is responsible for legalizing a
3021/// *INT_TO_FP operation of the specified operand when the target requests that
3022/// we promote it. At this point, we know that the result and operand types are
3023/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
3024/// operation that takes a larger input.
3025void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
3026 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
3027 bool IsStrict = N->isStrictFPOpcode();
3028 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
3029 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
3030 EVT DestVT = N->getValueType(ResNo: 0);
3031 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3032 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
3033 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
3034
3035 // First step, figure out the appropriate *INT_TO_FP operation to use.
3036 EVT NewInTy = LegalOp.getValueType();
3037
3038 unsigned OpToUse = 0;
3039
3040 // Scan for the appropriate larger type to use.
3041 while (true) {
3042 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
3043 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3044
3045 // If the target supports SINT_TO_FP of this type, use it.
3046 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
3047 OpToUse = SIntOp;
3048 break;
3049 }
3050 if (IsSigned)
3051 continue;
3052
3053 // If the target supports UINT_TO_FP of this type, use it.
3054 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
3055 OpToUse = UIntOp;
3056 break;
3057 }
3058
3059 // Otherwise, try a larger type.
3060 }
3061
3062 // Okay, we found the operation and type to use. Zero extend our input to the
3063 // desired type then run the operation on it.
3064 if (IsStrict) {
3065 SDValue Res =
3066 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
3067 Ops: {N->getOperand(Num: 0),
3068 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3069 DL: dl, VT: NewInTy, Operand: LegalOp)});
3070 Results.push_back(Elt: Res);
3071 Results.push_back(Elt: Res.getValue(R: 1));
3072 return;
3073 }
3074
3075 Results.push_back(
3076 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
3077 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3078 DL: dl, VT: NewInTy, Operand: LegalOp)));
3079}
3080
3081/// This function is responsible for legalizing a
3082/// FP_TO_*INT operation of the specified operand when the target requests that
3083/// we promote it. At this point, we know that the result and operand types are
3084/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
3085/// operation that returns a larger result.
3086void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3087 SmallVectorImpl<SDValue> &Results) {
3088 bool IsStrict = N->isStrictFPOpcode();
3089 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3090 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3091 EVT DestVT = N->getValueType(ResNo: 0);
3092 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3093 // First step, figure out the appropriate FP_TO*INT operation to use.
3094 EVT NewOutTy = DestVT;
3095
3096 unsigned OpToUse = 0;
3097
3098 // Scan for the appropriate larger type to use.
3099 while (true) {
3100 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
3101 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3102
3103 // A larger signed type can hold all unsigned values of the requested type,
3104 // so using FP_TO_SINT is valid
3105 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
3106 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3107 break;
3108
3109 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
3110 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
3111 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3112 break;
3113
3114 // Otherwise, try a larger type.
3115 }
3116
3117 // Okay, we found the operation and type to use.
3118 SDValue Operation;
3119 if (IsStrict) {
3120 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
3121 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
3122 } else
3123 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
3124
3125 // Truncate the result of the extended FP_TO_*INT operation to the desired
3126 // size.
3127 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
3128 Results.push_back(Elt: Trunc);
3129 if (IsStrict)
3130 Results.push_back(Elt: Operation.getValue(R: 1));
3131}
3132
3133/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
3134/// the result and operand types are legal and there must be a legal
3135/// FP_TO_*INT_SAT operation for a larger result type.
3136SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3137 const SDLoc &dl) {
3138 unsigned Opcode = Node->getOpcode();
3139
3140 // Scan for the appropriate larger type to use.
3141 EVT NewOutTy = Node->getValueType(ResNo: 0);
3142 while (true) {
3143 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
3144 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3145
3146 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
3147 break;
3148 }
3149
3150 // Saturation width is determined by second operand, so we don't have to
3151 // perform any fixup and can directly truncate the result.
3152 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
3153 N2: Node->getOperand(Num: 1));
3154 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
3155}
3156
3157/// Open code the operations for PARITY of the specified operation.
3158SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3159 EVT VT = Op.getValueType();
3160 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3161 unsigned Sz = VT.getScalarSizeInBits();
3162
3163 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3164 SDValue Result;
3165 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3166 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3167 } else {
3168 Result = Op;
3169 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3170 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3171 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3172 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3173 }
3174 }
3175
3176 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3177}
3178
3179SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3180 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3181 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3182 : Node->getOperand(Num: 0).getSimpleValueType();
3183 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3184 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3185 MVT NewScalarVT = NewVecVT.getVectorElementType();
3186
3187 SDLoc DL(Node);
3188 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3189
3190 // FIXME: Support integer.
3191 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3192 "Only FP promotion is supported");
3193
3194 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3195 if (Node->getOperand(Num: j).getValueType().isVector() &&
3196 !(IsVPOpcode &&
3197 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3198 // promote the vector operand.
3199 // FIXME: Support integer.
3200 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3201 "Only FP promotion is supported");
3202 Operands[j] =
3203 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3204 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3205 // promote the initial value.
3206 Operands[j] =
3207 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3208 } else {
3209 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3210 }
3211
3212 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3213 Flags: Node->getFlags());
3214
3215 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3216 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3217 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3218}
3219
3220bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3221 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3222 SmallVector<SDValue, 8> Results;
3223 SDLoc dl(Node);
3224 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3225 bool NeedInvert;
3226 switch (Node->getOpcode()) {
3227 case ISD::ABS:
3228 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3229 Results.push_back(Elt: Tmp1);
3230 break;
3231 case ISD::ABDS:
3232 case ISD::ABDU:
3233 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3234 Results.push_back(Elt: Tmp1);
3235 break;
3236 case ISD::AVGCEILS:
3237 case ISD::AVGCEILU:
3238 case ISD::AVGFLOORS:
3239 case ISD::AVGFLOORU:
3240 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3241 Results.push_back(Elt: Tmp1);
3242 break;
3243 case ISD::CTPOP:
3244 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3245 Results.push_back(Elt: Tmp1);
3246 break;
3247 case ISD::CTLZ:
3248 case ISD::CTLZ_ZERO_UNDEF:
3249 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3250 Results.push_back(Elt: Tmp1);
3251 break;
3252 case ISD::CTLS:
3253 if ((Tmp1 = TLI.expandCTLS(N: Node, DAG)))
3254 Results.push_back(Elt: Tmp1);
3255 break;
3256 case ISD::CTTZ:
3257 case ISD::CTTZ_ZERO_UNDEF:
3258 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3259 Results.push_back(Elt: Tmp1);
3260 break;
3261 case ISD::BITREVERSE:
3262 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3263 Results.push_back(Elt: Tmp1);
3264 break;
3265 case ISD::BSWAP:
3266 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3267 Results.push_back(Elt: Tmp1);
3268 break;
3269 case ISD::PARITY:
3270 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3271 break;
3272 case ISD::FRAMEADDR:
3273 case ISD::RETURNADDR:
3274 case ISD::FRAME_TO_ARGS_OFFSET:
3275 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3276 break;
3277 case ISD::EH_DWARF_CFA: {
3278 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3279 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3280 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3281 VT: CfaArg.getValueType(),
3282 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3283 VT: CfaArg.getValueType()),
3284 N2: CfaArg);
3285 SDValue FA = DAG.getNode(
3286 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3287 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3288 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3289 N1: FA, N2: Offset));
3290 break;
3291 }
3292 case ISD::GET_ROUNDING:
3293 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3294 Results.push_back(Elt: Node->getOperand(Num: 0));
3295 break;
3296 case ISD::EH_RETURN:
3297 case ISD::EH_LABEL:
3298 case ISD::PREFETCH:
3299 case ISD::VAEND:
3300 case ISD::EH_SJLJ_LONGJMP:
3301 // If the target didn't expand these, there's nothing to do, so just
3302 // preserve the chain and be done.
3303 Results.push_back(Elt: Node->getOperand(Num: 0));
3304 break;
3305 case ISD::READCYCLECOUNTER:
3306 case ISD::READSTEADYCOUNTER:
3307 // If the target didn't expand this, just return 'zero' and preserve the
3308 // chain.
3309 Results.append(NumInputs: Node->getNumValues() - 1,
3310 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3311 Results.push_back(Elt: Node->getOperand(Num: 0));
3312 break;
3313 case ISD::EH_SJLJ_SETJMP:
3314 // If the target didn't expand this, just return 'zero' and preserve the
3315 // chain.
3316 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3317 Results.push_back(Elt: Node->getOperand(Num: 0));
3318 break;
3319 case ISD::ATOMIC_LOAD: {
3320 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3321 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3322 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3323 SDValue Swap = DAG.getAtomicCmpSwap(
3324 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3325 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3326 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3327 Results.push_back(Elt: Swap.getValue(R: 0));
3328 Results.push_back(Elt: Swap.getValue(R: 1));
3329 break;
3330 }
3331 case ISD::ATOMIC_STORE: {
3332 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3333 SDValue Swap = DAG.getAtomic(
3334 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3335 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3336 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3337 Results.push_back(Elt: Swap.getValue(R: 1));
3338 break;
3339 }
3340 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3341 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3342 // splits out the success value as a comparison. Expanding the resulting
3343 // ATOMIC_CMP_SWAP will produce a libcall.
3344 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3345 SDValue Res = DAG.getAtomicCmpSwap(
3346 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3347 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3348 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3349
3350 SDValue ExtRes = Res;
3351 SDValue LHS = Res;
3352 SDValue RHS = Node->getOperand(Num: 1);
3353
3354 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3355 EVT OuterType = Node->getValueType(ResNo: 0);
3356 switch (TLI.getExtendForAtomicOps()) {
3357 case ISD::SIGN_EXTEND:
3358 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3359 N2: DAG.getValueType(AtomicType));
3360 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3361 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3362 ExtRes = LHS;
3363 break;
3364 case ISD::ZERO_EXTEND:
3365 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3366 N2: DAG.getValueType(AtomicType));
3367 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3368 ExtRes = LHS;
3369 break;
3370 case ISD::ANY_EXTEND:
3371 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3372 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3373 break;
3374 default:
3375 llvm_unreachable("Invalid atomic op extension");
3376 }
3377
3378 SDValue Success =
3379 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3380
3381 Results.push_back(Elt: ExtRes.getValue(R: 0));
3382 Results.push_back(Elt: Success);
3383 Results.push_back(Elt: Res.getValue(R: 1));
3384 break;
3385 }
3386 case ISD::ATOMIC_LOAD_SUB: {
3387 SDLoc DL(Node);
3388 EVT VT = Node->getValueType(ResNo: 0);
3389 SDValue RHS = Node->getOperand(Num: 2);
3390 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3391 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3392 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3393 RHS = RHS->getOperand(Num: 0);
3394 SDValue NewRHS =
3395 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3396 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3397 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3398 Val: NewRHS, MMO: AN->getMemOperand());
3399 Results.push_back(Elt: Res);
3400 Results.push_back(Elt: Res.getValue(R: 1));
3401 break;
3402 }
3403 case ISD::DYNAMIC_STACKALLOC:
3404 ExpandDYNAMIC_STACKALLOC(Node, Results);
3405 break;
3406 case ISD::MERGE_VALUES:
3407 for (unsigned i = 0; i < Node->getNumValues(); i++)
3408 Results.push_back(Elt: Node->getOperand(Num: i));
3409 break;
3410 case ISD::POISON:
3411 case ISD::UNDEF: {
3412 EVT VT = Node->getValueType(ResNo: 0);
3413 if (VT.isInteger())
3414 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3415 else {
3416 assert(VT.isFloatingPoint() && "Unknown value type!");
3417 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3418 }
3419 break;
3420 }
3421 case ISD::STRICT_FP_ROUND:
3422 // When strict mode is enforced we can't do expansion because it
3423 // does not honor the "strict" properties. Only libcall is allowed.
3424 if (TLI.isStrictFPEnabled())
3425 break;
3426 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3427 // since this operation is more efficient than stack operation.
3428 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3429 VT: Node->getValueType(ResNo: 0))
3430 == TargetLowering::Legal)
3431 break;
3432 // We fall back to use stack operation when the FP_ROUND operation
3433 // isn't available.
3434 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3435 DestVT: Node->getValueType(ResNo: 0), dl,
3436 Chain: Node->getOperand(Num: 0)))) {
3437 ReplaceNode(Old: Node, New: Tmp1.getNode());
3438 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3439 return true;
3440 }
3441 break;
3442 case ISD::FP_ROUND: {
3443 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3444 Results.push_back(Elt: Tmp1);
3445 break;
3446 }
3447
3448 [[fallthrough]];
3449 }
3450 case ISD::BITCAST:
3451 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3452 DestVT: Node->getValueType(ResNo: 0), dl)))
3453 Results.push_back(Elt: Tmp1);
3454 break;
3455 case ISD::STRICT_FP_EXTEND:
3456 // When strict mode is enforced we can't do expansion because it
3457 // does not honor the "strict" properties. Only libcall is allowed.
3458 if (TLI.isStrictFPEnabled())
3459 break;
3460 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3461 // since this operation is more efficient than stack operation.
3462 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3463 VT: Node->getValueType(ResNo: 0))
3464 == TargetLowering::Legal)
3465 break;
3466 // We fall back to use stack operation when the FP_EXTEND operation
3467 // isn't available.
3468 if ((Tmp1 = EmitStackConvert(
3469 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3470 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3471 ReplaceNode(Old: Node, New: Tmp1.getNode());
3472 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3473 return true;
3474 }
3475 break;
3476 case ISD::FP_EXTEND: {
3477 SDValue Op = Node->getOperand(Num: 0);
3478 EVT SrcVT = Op.getValueType();
3479 EVT DstVT = Node->getValueType(ResNo: 0);
3480 if (SrcVT.getScalarType() == MVT::bf16) {
3481 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3482 break;
3483 }
3484
3485 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3486 Results.push_back(Elt: Tmp1);
3487 break;
3488 }
3489 case ISD::BF16_TO_FP: {
3490 // Always expand bf16 to f32 casts, they lower to ext + shift.
3491 //
3492 // Note that the operand of this code can be bf16 or an integer type in case
3493 // bf16 is not supported on the target and was softened.
3494 SDValue Op = Node->getOperand(Num: 0);
3495 if (Op.getValueType() == MVT::bf16) {
3496 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3497 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3498 } else {
3499 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3500 }
3501 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3502 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3503 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3504 // Add fp_extend in case the output is bigger than f32.
3505 if (Node->getValueType(ResNo: 0) != MVT::f32)
3506 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3507 Results.push_back(Elt: Op);
3508 break;
3509 }
3510 case ISD::FP_TO_BF16: {
3511 SDValue Op = Node->getOperand(Num: 0);
3512 if (Op.getValueType() != MVT::f32)
3513 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3514 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3515 // Certain SNaNs will turn into infinities if we do a simple shift right.
3516 if (!DAG.isKnownNeverSNaN(Op)) {
3517 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3518 }
3519 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
3520 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3521 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3522 // The result of this node can be bf16 or an integer type in case bf16 is
3523 // not supported on the target and was softened to i16 for storage.
3524 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3525 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3526 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3527 } else {
3528 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3529 }
3530 Results.push_back(Elt: Op);
3531 break;
3532 }
3533 case ISD::CONVERT_FROM_ARBITRARY_FP: {
3534 // Expand conversion from arbitrary FP format stored in an integer to a
3535 // native IEEE float type using integer bit manipulation.
3536 //
3537 // TODO: currently only conversions from FP4, FP6 and FP8 formats from OCP
3538 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3539 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3540 // Float8E8M0FNU.
3541 EVT DstVT = Node->getValueType(ResNo: 0);
3542
3543 SDValue IntVal = Node->getOperand(Num: 0);
3544 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
3545 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
3546
3547 // Supported source formats.
3548 switch (Sem) {
3549 case APFloatBase::S_Float8E5M2:
3550 case APFloatBase::S_Float8E4M3FN:
3551 case APFloatBase::S_Float6E3M2FN:
3552 case APFloatBase::S_Float6E2M3FN:
3553 case APFloatBase::S_Float4E2M1FN:
3554 break;
3555 default:
3556 DAG.getContext()->emitError(ErrorStr: "CONVERT_FROM_ARBITRARY_FP: not implemented "
3557 "source format (semantics enum " +
3558 Twine(SemEnum) + ")");
3559 Results.push_back(Elt: DAG.getPOISON(VT: DstVT));
3560 break;
3561 }
3562 if (!Results.empty())
3563 break;
3564
3565 const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(S: Sem);
3566
3567 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
3568 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
3569 const unsigned SrcMant = SrcPrecision - 1;
3570 const unsigned SrcExp = SrcBits - SrcMant - 1;
3571 const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
3572
3573 const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;
3574
3575 // Destination format parameters.
3576 const fltSemantics &DstSem = DstVT.getFltSemantics();
3577
3578 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
3579 const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
3580 const unsigned DstExpBits = DstBits - DstMant - 1;
3581 const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
3582 const int DstBias = 1 - DstMinExp;
3583 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
3584
3585 // Work in an integer type matching the destination float width.
3586 // Use zero-extend to preserve the raw bit-pattern.
3587 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstBits);
3588 SDValue Src = DAG.getZExtOrTrunc(Op: IntVal, DL: dl, VT: IntVT);
3589
3590 EVT SetCCVT = getSetCCResultType(VT: IntVT);
3591
3592 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
3593 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
3594
3595 // Extract bit fields.
3596 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
3597 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
3598
3599 SDValue MantField = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Src,
3600 N2: DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT));
3601
3602 SDValue ExpField =
3603 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT,
3604 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
3605 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl)),
3606 N2: DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT));
3607
3608 SDValue SignBit =
3609 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
3610 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl));
3611
3612 // Precompute sign shifted to MSB of destination.
3613 SDValue SignShifted =
3614 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: SignBit,
3615 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl));
3616
3617 // Classify the input value based on compile-time format properties.
3618 SDValue ExpAllOnes = DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT);
3619 SDValue IsExpAllOnes =
3620 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: ExpAllOnes, Cond: ISD::SETEQ);
3621 SDValue IsExpZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: Zero, Cond: ISD::SETEQ);
3622 SDValue IsMantZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETEQ);
3623 SDValue IsMantNonZero =
3624 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETNE);
3625
3626 // NaN detection.
3627 SDValue IsNaN;
3628 if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
3629 // FiniteOnly formats (E2M1FN, E3M2FN, E2M3FN) never produce NaN.
3630 IsNaN = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
3631 } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
3632 // E5M2 produces NaN when exp == all-ones AND mantissa != 0.
3633 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantNonZero);
3634 } else {
3635 // NanOnly + AllOnes (E4M3FN): NaN when all exp and mantissa bits are 1.
3636 assert(SrcSem.nanEncoding == fltNanEncoding::AllOnes);
3637 SDValue MantAllOnes = DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT);
3638 SDValue IsMantAllOnes =
3639 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: MantAllOnes, Cond: ISD::SETEQ);
3640 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantAllOnes);
3641 }
3642
3643 // Inf detection.
3644 SDValue IsInf;
3645 if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
3646 // E5M2: Inf when exp == all-ones AND mantissa == 0.
3647 IsInf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantZero);
3648 } else {
3649 // NanOnly and FiniteOnly formats have no Inf representation.
3650 IsInf = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
3651 }
3652
3653 // Zero detection.
3654 SDValue IsZero = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantZero);
3655
3656 // Denorm detection: exp == 0 AND mant != 0.
3657 SDValue IsDenorm =
3658 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantNonZero);
3659
3660 // Normal value conversion.
3661 // dst_exp = exp_field + (DstBias - SrcBias)
3662 // dst_mant = mant << (DstMant - SrcMant)
3663 const int BiasAdjust = DstBias - SrcBias;
3664 SDValue NormDstExp = DAG.getNode(
3665 Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: ExpField,
3666 N2: DAG.getConstant(Val: APInt(DstBits, BiasAdjust, true), DL: dl, VT: IntVT));
3667
3668 SDValue NormDstMant;
3669 if (DstMant > SrcMant) {
3670 SDValue NormDstMantShift =
3671 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
3672 NormDstMant =
3673 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: MantField, N2: NormDstMantShift);
3674 } else {
3675 NormDstMant = MantField;
3676 }
3677
3678 // Assemble normal result.
3679 SDValue DstMantShift = DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl);
3680 SDValue NormExpShifted =
3681 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: NormDstExp, N2: DstMantShift);
3682 SDValue NormResult = DAG.getNode(
3683 Opcode: ISD::OR, DL: dl, VT: IntVT,
3684 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted),
3685 N2: NormDstMant);
3686
3687 // Denormal value conversion.
3688 // For a denormal source (exp_field == 0, mant != 0), normalize by finding
3689 // the MSB position of mant using CTLZ, then compute the correct
3690 // exponent and mantissa for the destination format.
3691 SDValue DenormResult;
3692 {
3693 const unsigned IntVTBits = DstBits;
3694 SDValue LeadingZeros =
3695 DAG.getNode(Opcode: ISD::CTLZ_ZERO_UNDEF, DL: dl, VT: IntVT, Operand: MantField);
3696
3697 // dst_exp_denorm = (IntVTBits + DstBias - SrcBias - SrcMant) -
3698 // LeadingZeros
3699 const int DenormExpConst =
3700 (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
3701 SDValue DenormDstExp = DAG.getNode(
3702 Opcode: ISD::SUB, DL: dl, VT: IntVT,
3703 N1: DAG.getConstant(Val: APInt(DstBits, DenormExpConst, true), DL: dl, VT: IntVT),
3704 N2: LeadingZeros);
3705
3706 // MSB position of the mantissa (0-indexed from LSB).
3707 SDValue MantMSB =
3708 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT,
3709 N1: DAG.getConstant(Val: IntVTBits - 1, DL: dl, VT: IntVT), N2: LeadingZeros);
3710
3711 // leading_one = 1 << MantMSB
3712 SDValue LeadingOne = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: MantMSB);
3713
3714 // frac = mant XOR leading_one (strip the implicit 1)
3715 SDValue Frac = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: IntVT, N1: MantField, N2: LeadingOne);
3716
3717 // shift_amount = DstMant - MantMSB
3718 // = DstMant - (IntVTBits - 1 - LeadingZeros)
3719 // = LeadingZeros - (IntVTBits - 1 - DstMant)
3720 const unsigned ShiftSub = IntVTBits - 1 - DstMant; // always >= 0
3721 SDValue ShiftAmount = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: LeadingZeros,
3722 N2: DAG.getConstant(Val: ShiftSub, DL: dl, VT: IntVT));
3723
3724 SDValue DenormDstMant =
3725 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: Frac, N2: ShiftAmount);
3726
3727 // Assemble denorm as sign | (denorm_dst_exp << DstMant) | denorm_dst_mant
3728 SDValue DenormExpShifted =
3729 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormDstExp, N2: DstMantShift);
3730 DenormResult = DAG.getNode(
3731 Opcode: ISD::OR, DL: dl, VT: IntVT,
3732 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
3733 N2: DenormDstMant);
3734 }
3735
3736 // Select between normal and denorm paths.
3737 SDValue FiniteResult =
3738 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsDenorm, LHS: DenormResult, RHS: NormResult);
3739
3740 // Build special-value results.
3741 // NaN -> canonical quiet NaN: sign=0, exp=all-ones, qNaN bit set.
3742 // Encoding: (DstExpAllOnes << DstMant) | (1 << (DstMant - 1))
3743 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
3744 SDValue NaNResult =
3745 DAG.getConstant(Val: (DstExpAllOnes << DstMant) | QNaNBit, DL: dl, VT: IntVT);
3746
3747 // Inf -> destination Inf.
3748 // sign | (DstExpAllOnes << DstMant)
3749 SDValue InfResult =
3750 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
3751 N2: DAG.getConstant(Val: DstExpAllOnes << DstMant, DL: dl, VT: IntVT));
3752
3753 // Zero -> signed zero.
3754 // Sign bit only.
3755 SDValue ZeroResult = SignShifted;
3756
3757 // Final selection goes in order: NaN takes priority, then Inf, then Zero.
3758 SDValue Result = FiniteResult;
3759 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
3760 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
3761 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
3762
3763 // Bitcast integer result to destination float type.
3764 Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT, Operand: Result);
3765
3766 Results.push_back(Elt: Result);
3767 break;
3768 }
3769 case ISD::FCANONICALIZE: {
3770 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
3771 // suggested in
3772 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
3773 // It uses strict_fp operations even outside a strict_fp context in order
3774 // to guarantee that the canonicalization is not optimized away by later
3775 // passes. The result chain introduced by that is intentionally ignored
3776 // since no ordering requirement is intended here.
3777
3778 // Create strict multiplication by 1.0.
3779 SDValue Operand = Node->getOperand(Num: 0);
3780 EVT VT = Operand.getValueType();
3781 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
3782 SDValue Chain = DAG.getEntryNode();
3783 // Propagate existing flags on canonicalize, and additionally set
3784 // NoFPExcept.
3785 SDNodeFlags CanonicalizeFlags = Node->getFlags();
3786 CanonicalizeFlags.setNoFPExcept(true);
3787 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
3788 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
3789
3790 Results.push_back(Elt: Mul);
3791 break;
3792 }
3793 case ISD::SIGN_EXTEND_INREG: {
3794 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3795 EVT VT = Node->getValueType(ResNo: 0);
3796
3797 // An in-register sign-extend of a boolean is a negation:
3798 // 'true' (1) sign-extended is -1.
3799 // 'false' (0) sign-extended is 0.
3800 // However, we must mask the high bits of the source operand because the
3801 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3802
3803 // TODO: Do this for vectors too?
3804 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3805 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3806 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3807 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3808 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3809 Results.push_back(Elt: Neg);
3810 break;
3811 }
3812
3813 // NOTE: we could fall back on load/store here too for targets without
3814 // SRA. However, it is doubtful that any exist.
3815 unsigned BitsDiff = VT.getScalarSizeInBits() -
3816 ExtraVT.getScalarSizeInBits();
3817 SDValue ShiftCst = DAG.getShiftAmountConstant(Val: BitsDiff, VT, DL: dl);
3818 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: ShiftCst);
3819 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Tmp1, N2: ShiftCst);
3820 Results.push_back(Elt: Tmp1);
3821 break;
3822 }
3823 case ISD::UINT_TO_FP:
3824 case ISD::STRICT_UINT_TO_FP:
3825 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3826 Results.push_back(Elt: Tmp1);
3827 if (Node->isStrictFPOpcode())
3828 Results.push_back(Elt: Tmp2);
3829 break;
3830 }
3831 [[fallthrough]];
3832 case ISD::SINT_TO_FP:
3833 case ISD::STRICT_SINT_TO_FP:
3834 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3835 Results.push_back(Elt: Tmp1);
3836 if (Node->isStrictFPOpcode())
3837 Results.push_back(Elt: Tmp2);
3838 }
3839 break;
3840 case ISD::FP_TO_SINT:
3841 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3842 Results.push_back(Elt: Tmp1);
3843 break;
3844 case ISD::STRICT_FP_TO_SINT:
3845 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3846 ReplaceNode(Old: Node, New: Tmp1.getNode());
3847 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3848 return true;
3849 }
3850 break;
3851 case ISD::FP_TO_UINT:
3852 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3853 Results.push_back(Elt: Tmp1);
3854 break;
3855 case ISD::STRICT_FP_TO_UINT:
3856 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3857 // Relink the chain.
3858 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3859 // Replace the new UINT result.
3860 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3861 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3862 return true;
3863 }
3864 break;
3865 case ISD::FP_TO_SINT_SAT:
3866 case ISD::FP_TO_UINT_SAT:
3867 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3868 break;
3869 case ISD::LROUND:
3870 case ISD::LLROUND: {
3871 SDValue Arg = Node->getOperand(Num: 0);
3872 EVT ArgVT = Arg.getValueType();
3873 EVT ResVT = Node->getValueType(ResNo: 0);
3874 SDLoc dl(Node);
3875 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3876 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3877 break;
3878 }
3879 case ISD::VAARG:
3880 Results.push_back(Elt: DAG.expandVAArg(Node));
3881 Results.push_back(Elt: Results[0].getValue(R: 1));
3882 break;
3883 case ISD::VACOPY:
3884 Results.push_back(Elt: DAG.expandVACopy(Node));
3885 break;
3886 case ISD::EXTRACT_VECTOR_ELT:
3887 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3888 // This must be an access of the only element. Return it.
3889 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3890 Operand: Node->getOperand(Num: 0));
3891 else
3892 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3893 Results.push_back(Elt: Tmp1);
3894 break;
3895 case ISD::EXTRACT_SUBVECTOR:
3896 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3897 break;
3898 case ISD::INSERT_SUBVECTOR:
3899 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3900 break;
3901 case ISD::CONCAT_VECTORS:
3902 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3903 VectorValueType.isScalableVector() ||
3904 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3905 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3906 else
3907 Results.push_back(Elt: ExpandConcatVectors(Node));
3908 break;
3909 case ISD::SCALAR_TO_VECTOR:
3910 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3911 break;
3912 case ISD::INSERT_VECTOR_ELT:
3913 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3914 break;
3915 case ISD::VECTOR_SHUFFLE: {
3916 SmallVector<int, 32> NewMask;
3917 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3918
3919 EVT VT = Node->getValueType(ResNo: 0);
3920 EVT EltVT = VT.getVectorElementType();
3921 SDValue Op0 = Node->getOperand(Num: 0);
3922 SDValue Op1 = Node->getOperand(Num: 1);
3923 if (!TLI.isTypeLegal(VT: EltVT)) {
3924 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3925
3926 // BUILD_VECTOR operands are allowed to be wider than the element type.
3927 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3928 // it.
3929 if (NewEltVT.bitsLT(VT: EltVT)) {
3930 // Convert shuffle node.
3931 // If original node was v4i64 and the new EltVT is i32,
3932 // cast operands to v8i32 and re-build the mask.
3933
3934 // Calculate new VT, the size of the new VT should be equal to original.
3935 EVT NewVT =
3936 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3937 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3938 assert(NewVT.bitsEq(VT));
3939
3940 // cast operands to new VT
3941 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3942 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3943
3944 // Convert the shuffle mask
3945 unsigned int factor =
3946 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3947
3948 // EltVT gets smaller
3949 assert(factor > 0);
3950
3951 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3952 if (Mask[i] < 0) {
3953 for (unsigned fi = 0; fi < factor; ++fi)
3954 NewMask.push_back(Elt: Mask[i]);
3955 }
3956 else {
3957 for (unsigned fi = 0; fi < factor; ++fi)
3958 NewMask.push_back(Elt: Mask[i]*factor+fi);
3959 }
3960 }
3961 Mask = NewMask;
3962 VT = NewVT;
3963 }
3964 EltVT = NewEltVT;
3965 }
3966 unsigned NumElems = VT.getVectorNumElements();
3967 SmallVector<SDValue, 16> Ops;
3968 for (unsigned i = 0; i != NumElems; ++i) {
3969 if (Mask[i] < 0) {
3970 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3971 continue;
3972 }
3973 unsigned Idx = Mask[i];
3974 if (Idx < NumElems)
3975 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3976 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3977 else
3978 Ops.push_back(
3979 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3980 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3981 }
3982
3983 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3984 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3985 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3986 Results.push_back(Elt: Tmp1);
3987 break;
3988 }
3989 case ISD::VECTOR_SPLICE_LEFT:
3990 case ISD::VECTOR_SPLICE_RIGHT: {
3991 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3992 break;
3993 }
3994 case ISD::VECTOR_DEINTERLEAVE: {
3995 unsigned Factor = Node->getNumOperands();
3996 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
3997 break;
3998 SmallVector<SDValue, 8> Ops(Node->ops());
3999 EVT VecVT = Node->getValueType(ResNo: 0);
4000 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
4001 // Deinterleave at Factor/2 so each result contains two factors interleaved:
4002 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
4003 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
4004 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
4005 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
4006 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
4007 Results.resize(N: Factor);
4008 // Deinterleave the 2 factors out:
4009 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
4010 for (unsigned I = 0; I < Factor / 2; I++) {
4011 SDValue Deinterleave =
4012 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
4013 Ops: {L.getValue(R: I), R.getValue(R: I)});
4014 Results[I] = Deinterleave.getValue(R: 0);
4015 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
4016 }
4017 break;
4018 }
4019 case ISD::VECTOR_INTERLEAVE: {
4020 unsigned Factor = Node->getNumOperands();
4021 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
4022 break;
4023 EVT VecVT = Node->getValueType(ResNo: 0);
4024 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
4025 SmallVector<SDValue, 8> LOps, ROps;
4026 // Interleave so we have 2 factors per result:
4027 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
4028 for (unsigned I = 0; I < Factor / 2; I++) {
4029 SDValue Interleave =
4030 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
4031 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
4032 LOps.push_back(Elt: Interleave.getValue(R: 0));
4033 ROps.push_back(Elt: Interleave.getValue(R: 1));
4034 }
4035 // Interleave at Factor/2:
4036 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
4037 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: LOps);
4038 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: ROps);
4039 for (unsigned I = 0; I < Factor / 2; I++)
4040 Results.push_back(Elt: L.getValue(R: I));
4041 for (unsigned I = 0; I < Factor / 2; I++)
4042 Results.push_back(Elt: R.getValue(R: I));
4043 break;
4044 }
4045 case ISD::EXTRACT_ELEMENT: {
4046 EVT OpTy = Node->getOperand(Num: 0).getValueType();
4047 if (Node->getConstantOperandVal(Num: 1)) {
4048 // 1 -> Hi
4049 Tmp1 = DAG.getNode(
4050 Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
4051 N2: DAG.getShiftAmountConstant(Val: OpTy.getSizeInBits() / 2, VT: OpTy, DL: dl));
4052 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
4053 } else {
4054 // 0 -> Lo
4055 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
4056 Operand: Node->getOperand(Num: 0));
4057 }
4058 Results.push_back(Elt: Tmp1);
4059 break;
4060 }
4061 case ISD::STACKADDRESS:
4062 case ISD::STACKSAVE:
4063 // Expand to CopyFromReg if the target set
4064 // StackPointerRegisterToSaveRestore.
4065 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
4066 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
4067 VT: Node->getValueType(ResNo: 0)));
4068 Results.push_back(Elt: Results[0].getValue(R: 1));
4069 } else {
4070 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
4071 Results.push_back(Elt: Node->getOperand(Num: 0));
4072
4073 StringRef IntrinsicName = Node->getOpcode() == ISD::STACKADDRESS
4074 ? "llvm.stackaddress"
4075 : "llvm.stacksave";
4076 DAG.getContext()->diagnose(DI: DiagnosticInfoLegalizationFailure(
4077 Twine(IntrinsicName) + " is not supported on this target.",
4078 DAG.getMachineFunction().getFunction(), dl.getDebugLoc()));
4079 }
4080 break;
4081 case ISD::STACKRESTORE:
4082 // Expand to CopyToReg if the target set
4083 // StackPointerRegisterToSaveRestore.
4084 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
4085 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
4086 N: Node->getOperand(Num: 1)));
4087 } else {
4088 Results.push_back(Elt: Node->getOperand(Num: 0));
4089 }
4090 break;
4091 case ISD::GET_DYNAMIC_AREA_OFFSET:
4092 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
4093 Results.push_back(Elt: Results[0].getValue(R: 0));
4094 break;
4095 case ISD::FCOPYSIGN:
4096 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
4097 break;
4098 case ISD::FNEG:
4099 Results.push_back(Elt: ExpandFNEG(Node));
4100 break;
4101 case ISD::FABS:
4102 Results.push_back(Elt: ExpandFABS(Node));
4103 break;
4104 case ISD::IS_FPCLASS: {
4105 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
4106 if (SDValue Expanded =
4107 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
4108 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
4109 Results.push_back(Elt: Expanded);
4110 break;
4111 }
4112 case ISD::SMIN:
4113 case ISD::SMAX:
4114 case ISD::UMIN:
4115 case ISD::UMAX: {
4116 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
4117 ISD::CondCode Pred;
4118 switch (Node->getOpcode()) {
4119 default: llvm_unreachable("How did we get here?");
4120 case ISD::SMAX: Pred = ISD::SETGT; break;
4121 case ISD::SMIN: Pred = ISD::SETLT; break;
4122 case ISD::UMAX: Pred = ISD::SETUGT; break;
4123 case ISD::UMIN: Pred = ISD::SETULT; break;
4124 }
4125 Tmp1 = Node->getOperand(Num: 0);
4126 Tmp2 = Node->getOperand(Num: 1);
4127 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
4128 Results.push_back(Elt: Tmp1);
4129 break;
4130 }
4131 case ISD::FMINNUM:
4132 case ISD::FMAXNUM: {
4133 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
4134 Results.push_back(Elt: Expanded);
4135 break;
4136 }
4137 case ISD::FMINIMUM:
4138 case ISD::FMAXIMUM: {
4139 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
4140 Results.push_back(Elt: Expanded);
4141 break;
4142 }
4143 case ISD::FMINIMUMNUM:
4144 case ISD::FMAXIMUMNUM: {
4145 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
4146 break;
4147 }
4148 case ISD::FSIN:
4149 case ISD::FCOS: {
4150 EVT VT = Node->getValueType(ResNo: 0);
4151 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
4152 // fcos which share the same operand and both are used.
4153 if ((TLI.isOperationLegal(Op: ISD::FSINCOS, VT) ||
4154 isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls())) &&
4155 useSinCos(Node)) {
4156 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4157 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
4158 if (Node->getOpcode() == ISD::FCOS)
4159 Tmp1 = Tmp1.getValue(R: 1);
4160 Results.push_back(Elt: Tmp1);
4161 }
4162 break;
4163 }
4164 case ISD::FLDEXP:
4165 case ISD::STRICT_FLDEXP: {
4166 EVT VT = Node->getValueType(ResNo: 0);
4167 RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT);
4168 // Use the LibCall instead, it is very likely faster
4169 // FIXME: Use separate LibCall action.
4170 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
4171 break;
4172
4173 if (SDValue Expanded = expandLdexp(Node)) {
4174 Results.push_back(Elt: Expanded);
4175 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
4176 Results.push_back(Elt: Expanded.getValue(R: 1));
4177 }
4178
4179 break;
4180 }
4181 case ISD::FFREXP: {
4182 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0));
4183 // Use the LibCall instead, it is very likely faster
4184 // FIXME: Use separate LibCall action.
4185 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
4186 break;
4187
4188 if (SDValue Expanded = expandFrexp(Node)) {
4189 Results.push_back(Elt: Expanded);
4190 Results.push_back(Elt: Expanded.getValue(R: 1));
4191 }
4192 break;
4193 }
4194 case ISD::FMODF: {
4195 RTLIB::Libcall LC = RTLIB::getMODF(VT: Node->getValueType(ResNo: 0));
4196 // Use the LibCall instead, it is very likely faster
4197 // FIXME: Use separate LibCall action.
4198 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
4199 break;
4200
4201 if (SDValue Expanded = expandModf(Node)) {
4202 Results.push_back(Elt: Expanded);
4203 Results.push_back(Elt: Expanded.getValue(R: 1));
4204 }
4205 break;
4206 }
4207 case ISD::FSINCOS: {
4208 if (isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls()))
4209 break;
4210 EVT VT = Node->getValueType(ResNo: 0);
4211 SDValue Op = Node->getOperand(Num: 0);
4212 SDNodeFlags Flags = Node->getFlags();
4213 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
4214 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
4215 Results.append(IL: {Tmp1, Tmp2});
4216 break;
4217 }
4218 case ISD::FMAD:
4219 llvm_unreachable("Illegal fmad should never be formed");
4220
4221 case ISD::FP16_TO_FP:
4222 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4223 // We can extend to types bigger than f32 in two steps without changing
4224 // the result. Since "f16 -> f32" is much more commonly available, give
4225 // CodeGen the option of emitting that before resorting to a libcall.
4226 SDValue Res =
4227 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
4228 Results.push_back(
4229 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
4230 }
4231 break;
4232 case ISD::STRICT_BF16_TO_FP:
4233 case ISD::STRICT_FP16_TO_FP:
4234 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4235 // We can extend to types bigger than f32 in two steps without changing
4236 // the result. Since "f16 -> f32" is much more commonly available, give
4237 // CodeGen the option of emitting that before resorting to a libcall.
4238 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
4239 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
4240 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
4241 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
4242 Ops: {Res.getValue(R: 1), Res});
4243 Results.push_back(Elt: Res);
4244 Results.push_back(Elt: Res.getValue(R: 1));
4245 }
4246 break;
4247 case ISD::FP_TO_FP16:
4248 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
4249 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
4250 SDValue Op = Node->getOperand(Num: 0);
4251 MVT SVT = Op.getSimpleValueType();
4252 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
4253 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
4254 // Under fastmath, we can expand this node into a fround followed by
4255 // a float-half conversion.
4256 SDValue FloatVal =
4257 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
4258 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
4259 Results.push_back(
4260 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
4261 }
4262 }
4263 break;
4264 case ISD::ConstantFP: {
4265 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
4266 // Check to see if this FP immediate is already legal.
4267 // If this is a legal constant, turn it into a TargetConstantFP node.
4268 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
4269 ForCodeSize: DAG.shouldOptForSize()))
4270 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
4271 break;
4272 }
4273 case ISD::Constant: {
4274 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
4275 Results.push_back(Elt: ExpandConstant(CP));
4276 break;
4277 }
4278 case ISD::FSUB: {
4279 EVT VT = Node->getValueType(ResNo: 0);
4280 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
4281 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
4282 const SDNodeFlags Flags = Node->getFlags();
4283 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
4284 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
4285 Results.push_back(Elt: Tmp1);
4286 }
4287 break;
4288 }
4289 case ISD::SUB: {
4290 EVT VT = Node->getValueType(ResNo: 0);
4291 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
4292 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
4293 "Don't know how to expand this subtraction!");
4294 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
4295 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
4296 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
4297 break;
4298 }
4299 case ISD::UREM:
4300 case ISD::SREM:
4301 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
4302 Results.push_back(Elt: Tmp1);
4303 break;
4304 case ISD::UDIV:
4305 case ISD::SDIV: {
4306 bool isSigned = Node->getOpcode() == ISD::SDIV;
4307 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4308 EVT VT = Node->getValueType(ResNo: 0);
4309 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
4310 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4311 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4312 N2: Node->getOperand(Num: 1));
4313 Results.push_back(Elt: Tmp1);
4314 }
4315 break;
4316 }
4317 case ISD::MULHU:
4318 case ISD::MULHS: {
4319 unsigned ExpandOpcode =
4320 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
4321 EVT VT = Node->getValueType(ResNo: 0);
4322 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4323
4324 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4325 N2: Node->getOperand(Num: 1));
4326 Results.push_back(Elt: Tmp1.getValue(R: 1));
4327 break;
4328 }
4329 case ISD::UMUL_LOHI:
4330 case ISD::SMUL_LOHI: {
4331 SDValue LHS = Node->getOperand(Num: 0);
4332 SDValue RHS = Node->getOperand(Num: 1);
4333 EVT VT = LHS.getValueType();
4334 unsigned MULHOpcode =
4335 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
4336
4337 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
4338 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
4339 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
4340 break;
4341 }
4342
4343 SmallVector<SDValue, 4> Halves;
4344 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4345 assert(TLI.isTypeLegal(HalfType));
4346 if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
4347 HiLoVT: HalfType, DAG,
4348 Kind: TargetLowering::MulExpansionKind::Always)) {
4349 for (unsigned i = 0; i < 2; ++i) {
4350 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
4351 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
4352 SDValue Shift =
4353 DAG.getShiftAmountConstant(Val: HalfType.getScalarSizeInBits(), VT, DL: dl);
4354 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4355 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4356 }
4357 break;
4358 }
4359 break;
4360 }
4361 case ISD::MUL: {
4362 EVT VT = Node->getValueType(ResNo: 0);
4363 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4364 // See if multiply or divide can be lowered using two-result operations.
4365 // We just need the low half of the multiply; try both the signed
4366 // and unsigned forms. If the target supports both SMUL_LOHI and
4367 // UMUL_LOHI, form a preference by checking which forms of plain
4368 // MULH it supports.
4369 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
4370 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
4371 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
4372 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
4373 unsigned OpToUse = 0;
4374 if (HasSMUL_LOHI && !HasMULHS) {
4375 OpToUse = ISD::SMUL_LOHI;
4376 } else if (HasUMUL_LOHI && !HasMULHU) {
4377 OpToUse = ISD::UMUL_LOHI;
4378 } else if (HasSMUL_LOHI) {
4379 OpToUse = ISD::SMUL_LOHI;
4380 } else if (HasUMUL_LOHI) {
4381 OpToUse = ISD::UMUL_LOHI;
4382 }
4383 if (OpToUse) {
4384 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4385 N2: Node->getOperand(Num: 1)));
4386 break;
4387 }
4388
4389 SDValue Lo, Hi;
4390 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4391 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
4392 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
4393 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
4394 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
4395 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
4396 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4397 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
4398 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
4399 SDValue Shift =
4400 DAG.getShiftAmountConstant(Val: HalfType.getSizeInBits(), VT, DL: dl);
4401 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4402 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4403 }
4404 break;
4405 }
4406 case ISD::FSHL:
4407 case ISD::FSHR:
4408 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
4409 Results.push_back(Elt: Expanded);
4410 break;
4411 case ISD::ROTL:
4412 case ISD::ROTR:
4413 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
4414 Results.push_back(Elt: Expanded);
4415 break;
4416 case ISD::CLMUL:
4417 case ISD::CLMULR:
4418 case ISD::CLMULH:
4419 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG))
4420 Results.push_back(Elt: Expanded);
4421 break;
4422 case ISD::SADDSAT:
4423 case ISD::UADDSAT:
4424 case ISD::SSUBSAT:
4425 case ISD::USUBSAT:
4426 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
4427 break;
4428 case ISD::SCMP:
4429 case ISD::UCMP:
4430 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
4431 break;
4432 case ISD::SSHLSAT:
4433 case ISD::USHLSAT:
4434 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
4435 break;
4436 case ISD::SMULFIX:
4437 case ISD::SMULFIXSAT:
4438 case ISD::UMULFIX:
4439 case ISD::UMULFIXSAT:
4440 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
4441 break;
4442 case ISD::SDIVFIX:
4443 case ISD::SDIVFIXSAT:
4444 case ISD::UDIVFIX:
4445 case ISD::UDIVFIXSAT:
4446 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
4447 LHS: Node->getOperand(Num: 0),
4448 RHS: Node->getOperand(Num: 1),
4449 Scale: Node->getConstantOperandVal(Num: 2),
4450 DAG)) {
4451 Results.push_back(Elt: V);
4452 break;
4453 }
4454 // FIXME: We might want to retry here with a wider type if we fail, if that
4455 // type is legal.
4456 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4457 // <= 128 (which is the case for all of the default Embedded-C types),
4458 // we will only get here with types and scales that we could always expand
4459 // if we were allowed to generate libcalls to division functions of illegal
4460 // type. But we cannot do that.
4461 llvm_unreachable("Cannot expand DIVFIX!");
4462 case ISD::UADDO_CARRY:
4463 case ISD::USUBO_CARRY: {
4464 SDValue LHS = Node->getOperand(Num: 0);
4465 SDValue RHS = Node->getOperand(Num: 1);
4466 SDValue Carry = Node->getOperand(Num: 2);
4467
4468 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4469
4470 // Initial add of the 2 operands.
4471 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4472 EVT VT = LHS.getValueType();
4473 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4474
4475 // Initial check for overflow.
4476 EVT CarryType = Node->getValueType(ResNo: 1);
4477 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4478 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4479 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4480
4481 // Add of the sum and the carry.
4482 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4483 SDValue CarryExt =
4484 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4485 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4486
4487 // Second check for overflow. If we are adding, we can only overflow if the
4488 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4489 // If we are subtracting, we can only overflow if the initial sum is 0 and
4490 // the carry is set, resulting in a new sum of all 1s.
4491 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4492 SDValue Overflow2 =
4493 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4494 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4495 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4496 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4497
4498 SDValue ResultCarry =
4499 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4500
4501 Results.push_back(Elt: Sum2);
4502 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4503 break;
4504 }
4505 case ISD::SADDO:
4506 case ISD::SSUBO: {
4507 SDValue Result, Overflow;
4508 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4509 Results.push_back(Elt: Result);
4510 Results.push_back(Elt: Overflow);
4511 break;
4512 }
4513 case ISD::UADDO:
4514 case ISD::USUBO: {
4515 SDValue Result, Overflow;
4516 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4517 Results.push_back(Elt: Result);
4518 Results.push_back(Elt: Overflow);
4519 break;
4520 }
4521 case ISD::UMULO:
4522 case ISD::SMULO: {
4523 SDValue Result, Overflow;
4524 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4525 Results.push_back(Elt: Result);
4526 Results.push_back(Elt: Overflow);
4527 }
4528 break;
4529 }
4530 case ISD::BUILD_PAIR: {
4531 EVT PairTy = Node->getValueType(ResNo: 0);
4532 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4533 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4534 Tmp2 = DAG.getNode(
4535 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4536 N2: DAG.getShiftAmountConstant(Val: PairTy.getSizeInBits() / 2, VT: PairTy, DL: dl));
4537 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4538 break;
4539 }
4540 case ISD::SELECT:
4541 Tmp1 = Node->getOperand(Num: 0);
4542 Tmp2 = Node->getOperand(Num: 1);
4543 Tmp3 = Node->getOperand(Num: 2);
4544 if (Tmp1.getOpcode() == ISD::SETCC) {
4545 Tmp1 = DAG.getSelectCC(
4546 DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), True: Tmp2, False: Tmp3,
4547 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get(), Flags: Node->getFlags());
4548 } else {
4549 Tmp1 =
4550 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4551 True: Tmp2, False: Tmp3, Cond: ISD::SETNE, Flags: Node->getFlags());
4552 }
4553 Results.push_back(Elt: Tmp1);
4554 break;
4555 case ISD::BR_JT: {
4556 SDValue Chain = Node->getOperand(Num: 0);
4557 SDValue Table = Node->getOperand(Num: 1);
4558 SDValue Index = Node->getOperand(Num: 2);
4559 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4560
4561 const DataLayout &TD = DAG.getDataLayout();
4562 EVT PTy = TLI.getPointerTy(DL: TD);
4563
4564 unsigned EntrySize =
4565 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4566
4567 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4568 // This transformation needs to be done here since otherwise the MIPS
4569 // backend will end up emitting a three instruction multiply sequence
4570 // instead of a single shift and MSP430 will call a runtime function.
4571 if (llvm::isPowerOf2_32(Value: EntrySize))
4572 Index = DAG.getNode(
4573 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4574 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4575 else
4576 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4577 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4578 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4579
4580 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4581 SDValue LD = DAG.getExtLoad(
4582 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4583 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4584 Addr = LD;
4585 if (TLI.isJumpTableRelative()) {
4586 // For PIC, the sequence is:
4587 // BRIND(RelocBase + load(Jumptable + index))
4588 // RelocBase can be JumpTable, GOT or some sort of global base.
4589 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4590 Offset: Addr, DL: dl);
4591 }
4592
4593 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4594 Results.push_back(Elt: Tmp1);
4595 break;
4596 }
4597 case ISD::BRCOND:
4598 // Expand brcond's setcc into its constituent parts and create a BR_CC
4599 // Node.
4600 Tmp1 = Node->getOperand(Num: 0);
4601 Tmp2 = Node->getOperand(Num: 1);
4602 if (Tmp2.getOpcode() == ISD::SETCC &&
4603 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4604 VT: Tmp2.getOperand(i: 0).getValueType())) {
4605 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4606 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4607 N5: Node->getOperand(Num: 2));
4608 } else {
4609 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4610 if (Tmp2.isUndef() ||
4611 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4612 Tmp3 = Tmp2;
4613 else
4614 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4615 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4616 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4617 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4618 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4619 N5: Node->getOperand(Num: 2));
4620 }
4621 Results.push_back(Elt: Tmp1);
4622 break;
4623 case ISD::SETCC:
4624 case ISD::VP_SETCC:
4625 case ISD::STRICT_FSETCC:
4626 case ISD::STRICT_FSETCCS: {
4627 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4628 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4629 Node->getOpcode() == ISD::STRICT_FSETCCS;
4630 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4631 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4632 unsigned Offset = IsStrict ? 1 : 0;
4633 Tmp1 = Node->getOperand(Num: 0 + Offset);
4634 Tmp2 = Node->getOperand(Num: 1 + Offset);
4635 Tmp3 = Node->getOperand(Num: 2 + Offset);
4636 SDValue Mask, EVL;
4637 if (IsVP) {
4638 Mask = Node->getOperand(Num: 3 + Offset);
4639 EVL = Node->getOperand(Num: 4 + Offset);
4640 }
4641 bool Legalized = TLI.LegalizeSetCCCondCode(
4642 DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl,
4643 Chain, IsSignaling);
4644
4645 if (Legalized) {
4646 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4647 // condition code, create a new SETCC node.
4648 if (Tmp3.getNode()) {
4649 if (IsStrict) {
4650 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4651 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4652 Chain = Tmp1.getValue(R: 1);
4653 } else if (IsVP) {
4654 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0),
4655 Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags());
4656 } else {
4657 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4658 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4659 }
4660 }
4661
4662 // If we expanded the SETCC by inverting the condition code, then wrap
4663 // the existing SETCC in a NOT to restore the intended condition.
4664 if (NeedInvert) {
4665 if (!IsVP)
4666 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4667 else
4668 Tmp1 =
4669 DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0));
4670 }
4671
4672 Results.push_back(Elt: Tmp1);
4673 if (IsStrict)
4674 Results.push_back(Elt: Chain);
4675
4676 break;
4677 }
4678
4679 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4680 // understand if this code is useful for strict nodes.
4681 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4682
4683 // Otherwise, SETCC for the given comparison type must be completely
4684 // illegal; expand it into a SELECT_CC.
4685 // FIXME: This drops the mask/evl for VP_SETCC.
4686 EVT VT = Node->getValueType(ResNo: 0);
4687 EVT Tmp1VT = Tmp1.getValueType();
4688 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4689 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4690 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3,
4691 Flags: Node->getFlags());
4692 Results.push_back(Elt: Tmp1);
4693 break;
4694 }
4695 case ISD::SELECT_CC: {
4696 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4697 Tmp1 = Node->getOperand(Num: 0); // LHS
4698 Tmp2 = Node->getOperand(Num: 1); // RHS
4699 Tmp3 = Node->getOperand(Num: 2); // True
4700 Tmp4 = Node->getOperand(Num: 3); // False
4701 EVT VT = Node->getValueType(ResNo: 0);
4702 SDValue Chain;
4703 SDValue CC = Node->getOperand(Num: 4);
4704 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4705
4706 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4707 // If the condition code is legal, then we need to expand this
4708 // node using SETCC and SELECT.
4709 EVT CmpVT = Tmp1.getValueType();
4710 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4711 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4712 "expanded.");
4713 EVT CCVT = getSetCCResultType(VT: CmpVT);
4714 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4715 Results.push_back(
4716 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4717 break;
4718 }
4719
4720 // SELECT_CC is legal, so the condition code must not be.
4721 bool Legalized = false;
4722 // Try to legalize by inverting the condition. This is for targets that
4723 // might support an ordered version of a condition, but not the unordered
4724 // version (or vice versa).
4725 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4726 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4727 // Use the new condition code and swap true and false
4728 Legalized = true;
4729 Tmp1 =
4730 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC, Flags: Node->getFlags());
4731 } else {
4732 // If The inverse is not legal, then try to swap the arguments using
4733 // the inverse condition code.
4734 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4735 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4736 // The swapped inverse condition is legal, so swap true and false,
4737 // lhs and rhs.
4738 Legalized = true;
4739 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC,
4740 Flags: Node->getFlags());
4741 }
4742 }
4743
4744 if (!Legalized) {
4745 Legalized = TLI.LegalizeSetCCCondCode(
4746 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4747 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4748
4749 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4750
4751 // If we expanded the SETCC by inverting the condition code, then swap
4752 // the True/False operands to match.
4753 if (NeedInvert)
4754 std::swap(a&: Tmp3, b&: Tmp4);
4755
4756 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4757 // condition code, create a new SELECT_CC node.
4758 if (CC.getNode()) {
4759 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4760 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4761 } else {
4762 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4763 CC = DAG.getCondCode(Cond: ISD::SETNE);
4764 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4765 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4766 }
4767 }
4768 Results.push_back(Elt: Tmp1);
4769 break;
4770 }
4771 case ISD::BR_CC: {
4772 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4773 SDValue Chain;
4774 Tmp1 = Node->getOperand(Num: 0); // Chain
4775 Tmp2 = Node->getOperand(Num: 2); // LHS
4776 Tmp3 = Node->getOperand(Num: 3); // RHS
4777 Tmp4 = Node->getOperand(Num: 1); // CC
4778
4779 bool Legalized = TLI.LegalizeSetCCCondCode(
4780 DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4,
4781 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4782 (void)Legalized;
4783 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4784
4785 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4786 // node.
4787 if (Tmp4.getNode()) {
4788 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4789
4790 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4791 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4792 } else {
4793 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4794 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4795 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4796 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4797 }
4798 Results.push_back(Elt: Tmp1);
4799 break;
4800 }
4801 case ISD::BUILD_VECTOR:
4802 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4803 break;
4804 case ISD::SPLAT_VECTOR:
4805 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4806 break;
4807 case ISD::SRA:
4808 case ISD::SRL:
4809 case ISD::SHL: {
4810 // Scalarize vector SRA/SRL/SHL.
4811 EVT VT = Node->getValueType(ResNo: 0);
4812 assert(VT.isVector() && "Unable to legalize non-vector shift");
4813 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4814 unsigned NumElem = VT.getVectorNumElements();
4815
4816 SmallVector<SDValue, 8> Scalars;
4817 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4818 SDValue Ex =
4819 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4820 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4821 SDValue Sh =
4822 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4823 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4824 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4825 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4826 }
4827
4828 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4829 Results.push_back(Elt: Result);
4830 break;
4831 }
4832 case ISD::VECREDUCE_FADD:
4833 case ISD::VECREDUCE_FMUL:
4834 case ISD::VECREDUCE_ADD:
4835 case ISD::VECREDUCE_MUL:
4836 case ISD::VECREDUCE_AND:
4837 case ISD::VECREDUCE_OR:
4838 case ISD::VECREDUCE_XOR:
4839 case ISD::VECREDUCE_SMAX:
4840 case ISD::VECREDUCE_SMIN:
4841 case ISD::VECREDUCE_UMAX:
4842 case ISD::VECREDUCE_UMIN:
4843 case ISD::VECREDUCE_FMAX:
4844 case ISD::VECREDUCE_FMIN:
4845 case ISD::VECREDUCE_FMAXIMUM:
4846 case ISD::VECREDUCE_FMINIMUM:
4847 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4848 break;
4849 case ISD::VP_CTTZ_ELTS:
4850 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4851 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4852 break;
4853 case ISD::CLEAR_CACHE:
4854 // The default expansion of llvm.clear_cache is simply a no-op for those
4855 // targets where it is not needed.
4856 Results.push_back(Elt: Node->getOperand(Num: 0));
4857 break;
4858 case ISD::LRINT:
4859 case ISD::LLRINT: {
4860 SDValue Arg = Node->getOperand(Num: 0);
4861 EVT ArgVT = Arg.getValueType();
4862 EVT ResVT = Node->getValueType(ResNo: 0);
4863 SDLoc dl(Node);
4864 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL: dl, VT: ArgVT, Operand: Arg);
4865 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
4866 break;
4867 }
4868 case ISD::ADDRSPACECAST:
4869 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4870 break;
4871 case ISD::GLOBAL_OFFSET_TABLE:
4872 case ISD::GlobalAddress:
4873 case ISD::GlobalTLSAddress:
4874 case ISD::ExternalSymbol:
4875 case ISD::ConstantPool:
4876 case ISD::JumpTable:
4877 case ISD::INTRINSIC_W_CHAIN:
4878 case ISD::INTRINSIC_WO_CHAIN:
4879 case ISD::INTRINSIC_VOID:
4880 // FIXME: Custom lowering for these operations shouldn't return null!
4881 // Return true so that we don't call ConvertNodeToLibcall which also won't
4882 // do anything.
4883 return true;
4884 }
4885
4886 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4887 // FIXME: We were asked to expand a strict floating-point operation,
4888 // but there is currently no expansion implemented that would preserve
4889 // the "strict" properties. For now, we just fall back to the non-strict
4890 // version if that is legal on the target. The actual mutation of the
4891 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4892 switch (Node->getOpcode()) {
4893 default:
4894 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4895 VT: Node->getValueType(ResNo: 0))
4896 == TargetLowering::Legal)
4897 return true;
4898 break;
4899 case ISD::STRICT_FSUB: {
4900 if (TLI.getStrictFPOperationAction(
4901 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4902 return true;
4903 if (TLI.getStrictFPOperationAction(
4904 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4905 break;
4906
4907 EVT VT = Node->getValueType(ResNo: 0);
4908 const SDNodeFlags Flags = Node->getFlags();
4909 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4910 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4911 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4912 Flags);
4913
4914 Results.push_back(Elt: Fadd);
4915 Results.push_back(Elt: Fadd.getValue(R: 1));
4916 break;
4917 }
4918 case ISD::STRICT_SINT_TO_FP:
4919 case ISD::STRICT_UINT_TO_FP:
4920 case ISD::STRICT_LRINT:
4921 case ISD::STRICT_LLRINT:
4922 case ISD::STRICT_LROUND:
4923 case ISD::STRICT_LLROUND:
4924 // These are registered by the operand type instead of the value
4925 // type. Reflect that here.
4926 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4927 VT: Node->getOperand(Num: 1).getValueType())
4928 == TargetLowering::Legal)
4929 return true;
4930 break;
4931 }
4932 }
4933
4934 // Replace the original node with the legalized result.
4935 if (Results.empty()) {
4936 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4937 return false;
4938 }
4939
4940 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4941 ReplaceNode(Old: Node, New: Results.data());
4942 return true;
4943}
4944
4945/// Return if we can use the FAST_* variant of a math libcall for the node.
4946/// FIXME: This is just guessing, we probably should have unique specific sets
4947/// flags required per libcall.
4948static bool canUseFastMathLibcall(const SDNode *Node) {
4949 // FIXME: Probably should define fast to respect nan/inf and only be
4950 // approximate functions.
4951
4952 SDNodeFlags Flags = Node->getFlags();
4953 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4954 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4955}
4956
4957void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4958 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4959 SmallVector<SDValue, 8> Results;
4960 SDLoc dl(Node);
4961 TargetLowering::MakeLibCallOptions CallOptions;
4962 CallOptions.IsPostTypeLegalization = true;
4963 // FIXME: Check flags on the node to see if we can use a finite call.
4964 unsigned Opc = Node->getOpcode();
4965 switch (Opc) {
4966 case ISD::ATOMIC_FENCE: {
4967 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4968 // FIXME: handle "fence singlethread" more efficiently.
4969 TargetLowering::ArgListTy Args;
4970
4971 TargetLowering::CallLoweringInfo CLI(DAG);
4972 CLI.setDebugLoc(dl)
4973 .setChain(Node->getOperand(Num: 0))
4974 .setLibCallee(
4975 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4976 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4977 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4978 ArgsList: std::move(Args));
4979
4980 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4981
4982 Results.push_back(Elt: CallResult.second);
4983 break;
4984 }
4985 // By default, atomic intrinsics are marked Legal and lowered. Targets
4986 // which don't support them directly, however, may want libcalls, in which
4987 // case they mark them Expand, and we get here.
4988 case ISD::ATOMIC_SWAP:
4989 case ISD::ATOMIC_LOAD_ADD:
4990 case ISD::ATOMIC_LOAD_SUB:
4991 case ISD::ATOMIC_LOAD_AND:
4992 case ISD::ATOMIC_LOAD_CLR:
4993 case ISD::ATOMIC_LOAD_OR:
4994 case ISD::ATOMIC_LOAD_XOR:
4995 case ISD::ATOMIC_LOAD_NAND:
4996 case ISD::ATOMIC_LOAD_MIN:
4997 case ISD::ATOMIC_LOAD_MAX:
4998 case ISD::ATOMIC_LOAD_UMIN:
4999 case ISD::ATOMIC_LOAD_UMAX:
5000 case ISD::ATOMIC_CMP_SWAP: {
5001 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
5002 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
5003 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
5004 EVT RetVT = Node->getValueType(ResNo: 0);
5005 SmallVector<SDValue, 4> Ops;
5006 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported) {
5007 // If outline atomic available, prepare its arguments and expand.
5008 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
5009 Ops.push_back(Elt: Node->getOperand(Num: 1));
5010
5011 } else {
5012 LC = RTLIB::getSYNC(Opc, VT);
5013 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5014 "Unexpected atomic op or value type!");
5015 // Arguments for expansion to sync libcall
5016 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
5017 }
5018 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
5019 Ops, CallOptions,
5020 dl: SDLoc(Node),
5021 Chain: Node->getOperand(Num: 0));
5022 Results.push_back(Elt: Tmp.first);
5023 Results.push_back(Elt: Tmp.second);
5024 break;
5025 }
5026 case ISD::TRAP: {
5027 // If this operation is not supported, lower it to 'abort()' call
5028 TargetLowering::ArgListTy Args;
5029 TargetLowering::CallLoweringInfo CLI(DAG);
5030 CLI.setDebugLoc(dl)
5031 .setChain(Node->getOperand(Num: 0))
5032 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
5033 Target: DAG.getExternalSymbol(
5034 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
5035 ArgsList: std::move(Args));
5036 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
5037
5038 Results.push_back(Elt: CallResult.second);
5039 break;
5040 }
5041 case ISD::CLEAR_CACHE: {
5042 SDValue InputChain = Node->getOperand(Num: 0);
5043 SDValue StartVal = Node->getOperand(Num: 1);
5044 SDValue EndVal = Node->getOperand(Num: 2);
5045 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5046 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
5047 dl: SDLoc(Node), Chain: InputChain);
5048 Results.push_back(Elt: Tmp.second);
5049 break;
5050 }
5051 case ISD::FMINNUM:
5052 case ISD::STRICT_FMINNUM:
5053 ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
5054 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
5055 Call_PPCF128: RTLIB::FMIN_PPCF128, Results);
5056 break;
5057 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
5058 // libcall legalization for these nodes, but there is no default expasion for
5059 // these nodes either (see PR63267 for example).
5060 case ISD::FMAXNUM:
5061 case ISD::STRICT_FMAXNUM:
5062 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
5063 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
5064 Call_PPCF128: RTLIB::FMAX_PPCF128, Results);
5065 break;
5066 case ISD::FMINIMUMNUM:
5067 ExpandFPLibCall(Node, Call_F32: RTLIB::FMINIMUM_NUM_F32, Call_F64: RTLIB::FMINIMUM_NUM_F64,
5068 Call_F80: RTLIB::FMINIMUM_NUM_F80, Call_F128: RTLIB::FMINIMUM_NUM_F128,
5069 Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128, Results);
5070 break;
5071 case ISD::FMAXIMUMNUM:
5072 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAXIMUM_NUM_F32, Call_F64: RTLIB::FMAXIMUM_NUM_F64,
5073 Call_F80: RTLIB::FMAXIMUM_NUM_F80, Call_F128: RTLIB::FMAXIMUM_NUM_F128,
5074 Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128, Results);
5075 break;
5076 case ISD::FSQRT:
5077 case ISD::STRICT_FSQRT: {
5078 // FIXME: Probably should define fast to respect nan/inf and only be
5079 // approximate functions.
5080 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5081 Call_F32: {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
5082 Call_F64: {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
5083 Call_F80: {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
5084 Call_F128: {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
5085 Call_PPCF128: {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
5086 Results);
5087 break;
5088 }
5089 case ISD::FCBRT:
5090 ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64,
5091 Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128,
5092 Call_PPCF128: RTLIB::CBRT_PPCF128, Results);
5093 break;
5094 case ISD::FSIN:
5095 case ISD::STRICT_FSIN:
5096 ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
5097 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
5098 Call_PPCF128: RTLIB::SIN_PPCF128, Results);
5099 break;
5100 case ISD::FCOS:
5101 case ISD::STRICT_FCOS:
5102 ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
5103 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
5104 Call_PPCF128: RTLIB::COS_PPCF128, Results);
5105 break;
5106 case ISD::FTAN:
5107 case ISD::STRICT_FTAN:
5108 ExpandFPLibCall(Node, Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
5109 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128, Results);
5110 break;
5111 case ISD::FASIN:
5112 case ISD::STRICT_FASIN:
5113 ExpandFPLibCall(Node, Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
5114 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128, Results);
5115 break;
5116 case ISD::FACOS:
5117 case ISD::STRICT_FACOS:
5118 ExpandFPLibCall(Node, Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
5119 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128, Results);
5120 break;
5121 case ISD::FATAN:
5122 case ISD::STRICT_FATAN:
5123 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
5124 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128, Results);
5125 break;
5126 case ISD::FATAN2:
5127 case ISD::STRICT_FATAN2:
5128 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN2_F32, Call_F64: RTLIB::ATAN2_F64, Call_F80: RTLIB::ATAN2_F80,
5129 Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128, Results);
5130 break;
5131 case ISD::FSINH:
5132 case ISD::STRICT_FSINH:
5133 ExpandFPLibCall(Node, Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
5134 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128, Results);
5135 break;
5136 case ISD::FCOSH:
5137 case ISD::STRICT_FCOSH:
5138 ExpandFPLibCall(Node, Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
5139 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128, Results);
5140 break;
5141 case ISD::FTANH:
5142 case ISD::STRICT_FTANH:
5143 ExpandFPLibCall(Node, Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
5144 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128, Results);
5145 break;
5146 case ISD::FSINCOS:
5147 case ISD::FSINCOSPI: {
5148 EVT VT = Node->getValueType(ResNo: 0);
5149
5150 if (Node->getOpcode() == ISD::FSINCOS) {
5151 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(RetVT: VT);
5152 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
5153 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
5154 Results.push_back(Elt: Expanded);
5155 Results.push_back(Elt: Expanded.getValue(R: 1));
5156 break;
5157 }
5158 }
5159 }
5160
5161 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
5162 ? RTLIB::getSINCOS(RetVT: VT)
5163 : RTLIB::getSINCOSPI(RetVT: VT);
5164 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);
5165 if (!Expanded) {
5166 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
5167 Node->getOperationName(G: &DAG));
5168 SDValue Poison = DAG.getPOISON(VT);
5169 Results.push_back(Elt: Poison);
5170 Results.push_back(Elt: Poison);
5171 }
5172
5173 break;
5174 }
5175 case ISD::FLOG:
5176 case ISD::STRICT_FLOG:
5177 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80,
5178 Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results);
5179 break;
5180 case ISD::FLOG2:
5181 case ISD::STRICT_FLOG2:
5182 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80,
5183 Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results);
5184 break;
5185 case ISD::FLOG10:
5186 case ISD::STRICT_FLOG10:
5187 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80,
5188 Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results);
5189 break;
5190 case ISD::FEXP:
5191 case ISD::STRICT_FEXP:
5192 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80,
5193 Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results);
5194 break;
5195 case ISD::FEXP2:
5196 case ISD::STRICT_FEXP2:
5197 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80,
5198 Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results);
5199 break;
5200 case ISD::FEXP10:
5201 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
5202 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results);
5203 break;
5204 case ISD::FTRUNC:
5205 case ISD::STRICT_FTRUNC:
5206 ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
5207 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
5208 Call_PPCF128: RTLIB::TRUNC_PPCF128, Results);
5209 break;
5210 case ISD::FFLOOR:
5211 case ISD::STRICT_FFLOOR:
5212 ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
5213 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
5214 Call_PPCF128: RTLIB::FLOOR_PPCF128, Results);
5215 break;
5216 case ISD::FCEIL:
5217 case ISD::STRICT_FCEIL:
5218 ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
5219 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
5220 Call_PPCF128: RTLIB::CEIL_PPCF128, Results);
5221 break;
5222 case ISD::FRINT:
5223 case ISD::STRICT_FRINT:
5224 ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
5225 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
5226 Call_PPCF128: RTLIB::RINT_PPCF128, Results);
5227 break;
5228 case ISD::FNEARBYINT:
5229 case ISD::STRICT_FNEARBYINT:
5230 ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32,
5231 Call_F64: RTLIB::NEARBYINT_F64,
5232 Call_F80: RTLIB::NEARBYINT_F80,
5233 Call_F128: RTLIB::NEARBYINT_F128,
5234 Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results);
5235 break;
5236 case ISD::FROUND:
5237 case ISD::STRICT_FROUND:
5238 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32,
5239 Call_F64: RTLIB::ROUND_F64,
5240 Call_F80: RTLIB::ROUND_F80,
5241 Call_F128: RTLIB::ROUND_F128,
5242 Call_PPCF128: RTLIB::ROUND_PPCF128, Results);
5243 break;
5244 case ISD::FROUNDEVEN:
5245 case ISD::STRICT_FROUNDEVEN:
5246 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32,
5247 Call_F64: RTLIB::ROUNDEVEN_F64,
5248 Call_F80: RTLIB::ROUNDEVEN_F80,
5249 Call_F128: RTLIB::ROUNDEVEN_F128,
5250 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results);
5251 break;
5252 case ISD::FLDEXP:
5253 case ISD::STRICT_FLDEXP:
5254 ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80,
5255 Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results);
5256 break;
5257 case ISD::FMODF:
5258 case ISD::FFREXP: {
5259 EVT VT = Node->getValueType(ResNo: 0);
5260 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
5261 : RTLIB::getFREXP(RetVT: VT);
5262 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
5263 /*CallRetResNo=*/0);
5264 if (!Expanded)
5265 llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");
5266 break;
5267 }
5268 case ISD::FPOWI:
5269 case ISD::STRICT_FPOWI: {
5270 RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0));
5271 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
5272 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
5273 // Some targets don't have a powi libcall; use pow instead.
5274 if (Node->isStrictFPOpcode()) {
5275 SDValue Exponent =
5276 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
5277 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5278 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5279 SDValue FPOW =
5280 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
5281 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5282 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
5283 Results.push_back(Elt: FPOW);
5284 Results.push_back(Elt: FPOW.getValue(R: 1));
5285 } else {
5286 SDValue Exponent =
5287 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
5288 Operand: Node->getOperand(Num: 1));
5289 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
5290 VT: Node->getValueType(ResNo: 0),
5291 N1: Node->getOperand(Num: 0), N2: Exponent));
5292 }
5293 break;
5294 }
5295 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
5296 bool ExponentHasSizeOfInt =
5297 DAG.getLibInfo().getIntSize() ==
5298 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
5299 if (!ExponentHasSizeOfInt) {
5300 // If the exponent does not match with sizeof(int) a libcall to
5301 // RTLIB::POWI would use the wrong type for the argument.
5302 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
5303 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
5304 break;
5305 }
5306 ExpandFPLibCall(Node, LC, Results);
5307 break;
5308 }
5309 case ISD::FPOW:
5310 case ISD::STRICT_FPOW:
5311 ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80,
5312 Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results);
5313 break;
5314 case ISD::LROUND:
5315 case ISD::STRICT_LROUND:
5316 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
5317 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
5318 Call_F128: RTLIB::LROUND_F128,
5319 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
5320 break;
5321 case ISD::LLROUND:
5322 case ISD::STRICT_LLROUND:
5323 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
5324 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
5325 Call_F128: RTLIB::LLROUND_F128,
5326 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
5327 break;
5328 case ISD::LRINT:
5329 case ISD::STRICT_LRINT:
5330 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
5331 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
5332 Call_F128: RTLIB::LRINT_F128,
5333 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
5334 break;
5335 case ISD::LLRINT:
5336 case ISD::STRICT_LLRINT:
5337 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
5338 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
5339 Call_F128: RTLIB::LLRINT_F128,
5340 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
5341 break;
5342 case ISD::FDIV:
5343 case ISD::STRICT_FDIV: {
5344 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5345 Call_F32: {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5346 Call_F64: {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5347 Call_F80: {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5348 Call_F128: {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5349 Call_PPCF128: {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5350 break;
5351 }
5352 case ISD::FREM:
5353 case ISD::STRICT_FREM:
5354 ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
5355 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
5356 Call_PPCF128: RTLIB::REM_PPCF128, Results);
5357 break;
5358 case ISD::FMA:
5359 case ISD::STRICT_FMA:
5360 ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64,
5361 Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128,
5362 Call_PPCF128: RTLIB::FMA_PPCF128, Results);
5363 break;
5364 case ISD::FADD:
5365 case ISD::STRICT_FADD: {
5366 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5367 Call_F32: {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5368 Call_F64: {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5369 Call_F80: {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5370 Call_F128: {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5371 Call_PPCF128: {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5372 break;
5373 }
5374 case ISD::FMUL:
5375 case ISD::STRICT_FMUL: {
5376 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5377 Call_F32: {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5378 Call_F64: {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5379 Call_F80: {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5380 Call_F128: {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5381 Call_PPCF128: {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5382 break;
5383 }
5384 case ISD::FP16_TO_FP:
5385 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5386 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
5387 }
5388 break;
5389 case ISD::STRICT_BF16_TO_FP:
5390 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5391 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5392 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
5393 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5394 Results.push_back(Elt: Tmp.first);
5395 Results.push_back(Elt: Tmp.second);
5396 }
5397 break;
5398 case ISD::STRICT_FP16_TO_FP: {
5399 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5400 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5401 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
5402 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5403 Results.push_back(Elt: Tmp.first);
5404 Results.push_back(Elt: Tmp.second);
5405 }
5406 break;
5407 }
5408 case ISD::FP_TO_FP16: {
5409 RTLIB::Libcall LC =
5410 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
5411 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5412 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5413 break;
5414 }
5415 case ISD::FP_TO_BF16: {
5416 RTLIB::Libcall LC =
5417 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
5418 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5419 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5420 break;
5421 }
5422 case ISD::STRICT_SINT_TO_FP:
5423 case ISD::STRICT_UINT_TO_FP:
5424 case ISD::SINT_TO_FP:
5425 case ISD::UINT_TO_FP: {
5426 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
5427 bool IsStrict = Node->isStrictFPOpcode();
5428 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
5429 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
5430 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
5431 EVT RVT = Node->getValueType(ResNo: 0);
5432 EVT NVT = EVT();
5433 SDLoc dl(Node);
5434
5435 // Even if the input is legal, no libcall may exactly match, eg. we don't
5436 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
5437 // eg: i13 -> fp. Then, look for an appropriate libcall.
5438 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5439 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5440 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5441 ++t) {
5442 NVT = (MVT::SimpleValueType)t;
5443 // The source needs to big enough to hold the operand.
5444 if (NVT.bitsGE(VT: SVT))
5445 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
5446 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
5447 }
5448 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5449
5450 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5451 // Sign/zero extend the argument if the libcall takes a larger type.
5452 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
5453 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
5454 CallOptions.setIsSigned(Signed);
5455 std::pair<SDValue, SDValue> Tmp =
5456 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
5457 Results.push_back(Elt: Tmp.first);
5458 if (IsStrict)
5459 Results.push_back(Elt: Tmp.second);
5460 break;
5461 }
5462 case ISD::FP_TO_SINT:
5463 case ISD::FP_TO_UINT:
5464 case ISD::STRICT_FP_TO_SINT:
5465 case ISD::STRICT_FP_TO_UINT: {
5466 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
5467 bool IsStrict = Node->isStrictFPOpcode();
5468 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
5469 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
5470
5471 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5472 EVT SVT = Op.getValueType();
5473 EVT RVT = Node->getValueType(ResNo: 0);
5474 EVT NVT = EVT();
5475 SDLoc dl(Node);
5476
5477 // Even if the result is legal, no libcall may exactly match, eg. we don't
5478 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
5479 // eg: fp -> i32. Then, look for an appropriate libcall.
5480 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5481 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5482 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5483 ++IntVT) {
5484 NVT = (MVT::SimpleValueType)IntVT;
5485 // The type needs to big enough to hold the result.
5486 if (NVT.bitsGE(VT: RVT))
5487 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
5488 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
5489 }
5490 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5491
5492 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5493 std::pair<SDValue, SDValue> Tmp =
5494 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
5495
5496 // Truncate the result if the libcall returns a larger type.
5497 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
5498 if (IsStrict)
5499 Results.push_back(Elt: Tmp.second);
5500 break;
5501 }
5502
5503 case ISD::FP_ROUND:
5504 case ISD::STRICT_FP_ROUND: {
5505 // X = FP_ROUND(Y, TRUNC)
5506 // TRUNC is a flag, which is always an integer that is zero or one.
5507 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5508 // is known to not change the value of Y.
5509 // We can only expand it into libcall if the TRUNC is 0.
5510 bool IsStrict = Node->isStrictFPOpcode();
5511 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5512 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5513 EVT VT = Node->getValueType(ResNo: 0);
5514 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5515 "Unable to expand as libcall if it is not normal rounding");
5516
5517 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5518 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5519
5520 std::pair<SDValue, SDValue> Tmp =
5521 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5522 Results.push_back(Elt: Tmp.first);
5523 if (IsStrict)
5524 Results.push_back(Elt: Tmp.second);
5525 break;
5526 }
5527 case ISD::FP_EXTEND: {
5528 Results.push_back(
5529 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5530 RetVT: Node->getValueType(ResNo: 0)),
5531 Node, isSigned: false).first);
5532 break;
5533 }
5534 case ISD::STRICT_FP_EXTEND:
5535 case ISD::STRICT_FP_TO_FP16:
5536 case ISD::STRICT_FP_TO_BF16: {
5537 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5538 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5539 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5540 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5541 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5542 else
5543 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5544 RetVT: Node->getValueType(ResNo: 0));
5545
5546 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5547
5548 std::pair<SDValue, SDValue> Tmp =
5549 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5550 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5551 Results.push_back(Elt: Tmp.first);
5552 Results.push_back(Elt: Tmp.second);
5553 break;
5554 }
5555 case ISD::FSUB:
5556 case ISD::STRICT_FSUB: {
5557 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5558 Call_F32: {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5559 Call_F64: {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5560 Call_F80: {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5561 Call_F128: {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5562 Call_PPCF128: {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5563 break;
5564 }
5565 case ISD::SREM:
5566 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5567 Call_I8: RTLIB::SREM_I8,
5568 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5569 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5570 break;
5571 case ISD::UREM:
5572 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5573 Call_I8: RTLIB::UREM_I8,
5574 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5575 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5576 break;
5577 case ISD::SDIV:
5578 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5579 Call_I8: RTLIB::SDIV_I8,
5580 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5581 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5582 break;
5583 case ISD::UDIV:
5584 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5585 Call_I8: RTLIB::UDIV_I8,
5586 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5587 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5588 break;
5589 case ISD::SDIVREM:
5590 case ISD::UDIVREM:
5591 // Expand into divrem libcall
5592 ExpandDivRemLibCall(Node, Results);
5593 break;
5594 case ISD::MUL:
5595 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5596 Call_I8: RTLIB::MUL_I8,
5597 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5598 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5599 break;
5600 case ISD::CTLZ_ZERO_UNDEF:
5601 Results.push_back(Elt: ExpandBitCountingLibCall(
5602 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5603 break;
5604 case ISD::CTPOP:
5605 Results.push_back(Elt: ExpandBitCountingLibCall(
5606 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5607 break;
5608 case ISD::RESET_FPENV: {
5609 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5610 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5611 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5612 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5613 SDValue Chain = Node->getOperand(Num: 0);
5614 Results.push_back(
5615 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
5616 break;
5617 }
5618 case ISD::GET_FPENV_MEM: {
5619 SDValue Chain = Node->getOperand(Num: 0);
5620 SDValue EnvPtr = Node->getOperand(Num: 1);
5621 Results.push_back(
5622 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5623 break;
5624 }
5625 case ISD::SET_FPENV_MEM: {
5626 SDValue Chain = Node->getOperand(Num: 0);
5627 SDValue EnvPtr = Node->getOperand(Num: 1);
5628 Results.push_back(
5629 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5630 break;
5631 }
5632 case ISD::GET_FPMODE: {
5633 // Call fegetmode, which saves control modes into a stack slot. Then load
5634 // the value to return from the stack.
5635 EVT ModeVT = Node->getValueType(ResNo: 0);
5636 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5637 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5638 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5639 InChain: Node->getOperand(Num: 0), DLoc: dl);
5640 SDValue LdInst = DAG.getLoad(
5641 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5642 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5643 Results.push_back(Elt: LdInst);
5644 Results.push_back(Elt: LdInst.getValue(R: 1));
5645 break;
5646 }
5647 case ISD::SET_FPMODE: {
5648 // Move control modes to stack slot and then call fesetmode with the pointer
5649 // to the slot as argument.
5650 SDValue Mode = Node->getOperand(Num: 1);
5651 EVT ModeVT = Mode.getValueType();
5652 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5653 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5654 SDValue StInst = DAG.getStore(
5655 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5656 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5657 Results.push_back(
5658 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
5659 break;
5660 }
5661 case ISD::RESET_FPMODE: {
5662 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5663 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5664 // target must provide custom lowering.
5665 const DataLayout &DL = DAG.getDataLayout();
5666 EVT PtrTy = TLI.getPointerTy(DL);
5667 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5668 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5669 InChain: Node->getOperand(Num: 0), DLoc: dl));
5670 break;
5671 }
5672 }
5673
5674 // Replace the original node with the legalized result.
5675 if (!Results.empty()) {
5676 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5677 ReplaceNode(Old: Node, New: Results.data());
5678 } else
5679 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5680}
5681
5682// Determine the vector type to use in place of an original scalar element when
5683// promoting equally sized vectors.
5684static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5685 MVT EltVT, MVT NewEltVT) {
5686 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5687 MVT MidVT = OldEltsPerNewElt == 1
5688 ? NewEltVT
5689 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5690 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5691 return MidVT;
5692}
5693
5694void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5695 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5696 SmallVector<SDValue, 8> Results;
5697 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5698 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5699 Node->getOpcode() == ISD::SINT_TO_FP ||
5700 Node->getOpcode() == ISD::SETCC ||
5701 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5702 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5703 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5704 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5705 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5706 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM) {
5707 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5708 }
5709 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5710 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5711 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5712 Node->getOpcode() == ISD::STRICT_FSETCC ||
5713 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5714 Node->getOpcode() == ISD::STRICT_LRINT ||
5715 Node->getOpcode() == ISD::STRICT_LLRINT ||
5716 Node->getOpcode() == ISD::STRICT_LROUND ||
5717 Node->getOpcode() == ISD::STRICT_LLROUND ||
5718 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5719 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5720 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5721 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5722 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5723 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5724 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5725 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5726 if (Node->getOpcode() == ISD::BR_CC ||
5727 Node->getOpcode() == ISD::SELECT_CC)
5728 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5729 // Preserve fast math flags
5730 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5731 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5732 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5733 SDLoc dl(Node);
5734 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5735 switch (Node->getOpcode()) {
5736 case ISD::CTTZ:
5737 case ISD::CTTZ_ZERO_UNDEF:
5738 case ISD::CTLZ:
5739 case ISD::CTPOP: {
5740 // Zero extend the argument unless its cttz, then use any_extend.
5741 if (Node->getOpcode() == ISD::CTTZ ||
5742 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
5743 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5744 else
5745 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5746
5747 unsigned NewOpc = Node->getOpcode();
5748 if (NewOpc == ISD::CTTZ) {
5749 // The count is the same in the promoted type except if the original
5750 // value was zero. This can be handled by setting the bit just off
5751 // the top of the original type.
5752 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5753 BitNo: OVT.getSizeInBits());
5754 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5755 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5756 NewOpc = ISD::CTTZ_ZERO_UNDEF;
5757 }
5758 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
5759 // already the correct result.
5760 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5761 if (NewOpc == ISD::CTLZ) {
5762 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5763 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5764 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5765 OVT.getSizeInBits(), DL: dl, VT: NVT));
5766 }
5767 Results.push_back(
5768 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5769 break;
5770 }
5771 case ISD::CTLZ_ZERO_UNDEF: {
5772 // We know that the argument is unlikely to be zero, hence we can take a
5773 // different approach as compared to ISD::CTLZ
5774
5775 // Any Extend the argument
5776 auto AnyExtendedNode =
5777 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5778
5779 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5780 auto ShiftConstant = DAG.getShiftAmountConstant(
5781 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5782 auto LeftShiftResult =
5783 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5784
5785 // Perform the larger operation
5786 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5787 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5788 break;
5789 }
5790 case ISD::BITREVERSE:
5791 case ISD::BSWAP: {
5792 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5793 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5794 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5795 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5796 N2: DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl));
5797
5798 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5799 break;
5800 }
5801 case ISD::FP_TO_UINT:
5802 case ISD::STRICT_FP_TO_UINT:
5803 case ISD::FP_TO_SINT:
5804 case ISD::STRICT_FP_TO_SINT:
5805 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5806 break;
5807 case ISD::FP_TO_UINT_SAT:
5808 case ISD::FP_TO_SINT_SAT:
5809 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5810 break;
5811 case ISD::UINT_TO_FP:
5812 case ISD::STRICT_UINT_TO_FP:
5813 case ISD::SINT_TO_FP:
5814 case ISD::STRICT_SINT_TO_FP:
5815 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5816 break;
5817 case ISD::VAARG: {
5818 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5819 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5820
5821 unsigned TruncOp;
5822 if (OVT.isVector()) {
5823 TruncOp = ISD::BITCAST;
5824 } else {
5825 assert(OVT.isInteger()
5826 && "VAARG promotion is supported only for vectors or integer types");
5827 TruncOp = ISD::TRUNCATE;
5828 }
5829
5830 // Perform the larger operation, then convert back
5831 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5832 Align: Node->getConstantOperandVal(Num: 3));
5833 Chain = Tmp1.getValue(R: 1);
5834
5835 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5836
5837 // Modified the chain result - switch anything that used the old chain to
5838 // use the new one.
5839 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5840 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5841 if (UpdatedNodes) {
5842 UpdatedNodes->insert(X: Tmp2.getNode());
5843 UpdatedNodes->insert(X: Chain.getNode());
5844 }
5845 ReplacedNode(N: Node);
5846 break;
5847 }
5848 case ISD::MUL:
5849 case ISD::SDIV:
5850 case ISD::SREM:
5851 case ISD::UDIV:
5852 case ISD::UREM:
5853 case ISD::SMIN:
5854 case ISD::SMAX:
5855 case ISD::UMIN:
5856 case ISD::UMAX:
5857 case ISD::AND:
5858 case ISD::OR:
5859 case ISD::XOR: {
5860 unsigned ExtOp, TruncOp;
5861 if (OVT.isVector()) {
5862 ExtOp = ISD::BITCAST;
5863 TruncOp = ISD::BITCAST;
5864 } else {
5865 assert(OVT.isInteger() && "Cannot promote logic operation");
5866
5867 switch (Node->getOpcode()) {
5868 default:
5869 ExtOp = ISD::ANY_EXTEND;
5870 break;
5871 case ISD::SDIV:
5872 case ISD::SREM:
5873 case ISD::SMIN:
5874 case ISD::SMAX:
5875 ExtOp = ISD::SIGN_EXTEND;
5876 break;
5877 case ISD::UDIV:
5878 case ISD::UREM:
5879 ExtOp = ISD::ZERO_EXTEND;
5880 break;
5881 case ISD::UMIN:
5882 case ISD::UMAX:
5883 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5884 ExtOp = ISD::SIGN_EXTEND;
5885 else
5886 ExtOp = ISD::ZERO_EXTEND;
5887 break;
5888 }
5889 TruncOp = ISD::TRUNCATE;
5890 }
5891 // Promote each of the values to the new type.
5892 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5893 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5894 // Perform the larger operation, then convert back
5895 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5896 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5897 break;
5898 }
5899 case ISD::UMUL_LOHI:
5900 case ISD::SMUL_LOHI: {
5901 // Promote to a multiply in a wider integer type.
5902 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5903 : ISD::SIGN_EXTEND;
5904 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5905 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5906 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5907
5908 unsigned OriginalSize = OVT.getScalarSizeInBits();
5909 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5910 N2: DAG.getShiftAmountConstant(Val: OriginalSize, VT: NVT, DL: dl));
5911 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5912 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5913 break;
5914 }
5915 case ISD::SELECT: {
5916 unsigned ExtOp, TruncOp;
5917 if (Node->getValueType(ResNo: 0).isVector() ||
5918 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5919 ExtOp = ISD::BITCAST;
5920 TruncOp = ISD::BITCAST;
5921 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5922 ExtOp = ISD::ANY_EXTEND;
5923 TruncOp = ISD::TRUNCATE;
5924 } else {
5925 ExtOp = ISD::FP_EXTEND;
5926 TruncOp = ISD::FP_ROUND;
5927 }
5928 Tmp1 = Node->getOperand(Num: 0);
5929 // Promote each of the values to the new type.
5930 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5931 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5932 // Perform the larger operation, then round down.
5933 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5934 if (TruncOp != ISD::FP_ROUND)
5935 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5936 else
5937 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5938 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5939 Results.push_back(Elt: Tmp1);
5940 break;
5941 }
5942 case ISD::VECTOR_SHUFFLE: {
5943 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5944
5945 // Cast the two input vectors.
5946 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5947 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5948
5949 // Convert the shuffle mask to the right # elements.
5950 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5951 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5952 Results.push_back(Elt: Tmp1);
5953 break;
5954 }
5955 case ISD::VECTOR_SPLICE_LEFT:
5956 case ISD::VECTOR_SPLICE_RIGHT: {
5957 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5958 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5959 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5960 N3: Node->getOperand(Num: 2));
5961 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5962 break;
5963 }
5964 case ISD::SELECT_CC: {
5965 SDValue Cond = Node->getOperand(Num: 4);
5966 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5967 // Type of the comparison operands.
5968 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5969 assert(CVT == OVT && "not handled");
5970
5971 unsigned ExtOp = ISD::FP_EXTEND;
5972 if (NVT.isInteger()) {
5973 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5974 }
5975
5976 // Promote the comparison operands, if needed.
5977 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5978 Tmp1 = Node->getOperand(Num: 0);
5979 Tmp2 = Node->getOperand(Num: 1);
5980 } else {
5981 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5982 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5983 }
5984 // Cast the true/false operands.
5985 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5986 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5987
5988 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5989 Flags: Node->getFlags());
5990
5991 // Cast the result back to the original type.
5992 if (ExtOp != ISD::FP_EXTEND)
5993 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5994 else
5995 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5996 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5997
5998 Results.push_back(Elt: Tmp1);
5999 break;
6000 }
6001 case ISD::SETCC:
6002 case ISD::STRICT_FSETCC:
6003 case ISD::STRICT_FSETCCS: {
6004 unsigned ExtOp = ISD::FP_EXTEND;
6005 if (NVT.isInteger()) {
6006 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
6007 if (isSignedIntSetCC(Code: CCCode) ||
6008 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
6009 ExtOp = ISD::SIGN_EXTEND;
6010 else
6011 ExtOp = ISD::ZERO_EXTEND;
6012 }
6013 if (Node->isStrictFPOpcode()) {
6014 SDValue InChain = Node->getOperand(Num: 0);
6015 std::tie(args&: Tmp1, args: std::ignore) =
6016 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
6017 std::tie(args&: Tmp2, args: std::ignore) =
6018 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
6019 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
6020 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
6021 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
6022 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
6023 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
6024 Flags: Node->getFlags()));
6025 Results.push_back(Elt: Results.back().getValue(R: 1));
6026 break;
6027 }
6028 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6029 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
6030 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
6031 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
6032 break;
6033 }
6034 case ISD::BR_CC: {
6035 unsigned ExtOp = ISD::FP_EXTEND;
6036 if (NVT.isInteger()) {
6037 ISD::CondCode CCCode =
6038 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
6039 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
6040 }
6041 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
6042 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
6043 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
6044 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
6045 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
6046 break;
6047 }
6048 case ISD::FADD:
6049 case ISD::FSUB:
6050 case ISD::FMUL:
6051 case ISD::FDIV:
6052 case ISD::FREM:
6053 case ISD::FMINNUM:
6054 case ISD::FMAXNUM:
6055 case ISD::FMINIMUM:
6056 case ISD::FMAXIMUM:
6057 case ISD::FMINIMUMNUM:
6058 case ISD::FMAXIMUMNUM:
6059 case ISD::FPOW:
6060 case ISD::FATAN2:
6061 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6062 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
6063 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
6064 Results.push_back(
6065 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
6066 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6067 break;
6068
6069 case ISD::STRICT_FMINIMUM:
6070 case ISD::STRICT_FMAXIMUM: {
6071 SDValue InChain = Node->getOperand(Num: 0);
6072 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
6073 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
6074 N2: Node->getOperand(Num: 1));
6075 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
6076 N2: Node->getOperand(Num: 2));
6077 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
6078 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
6079 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
6080 N1: InChain, N2: Tmp3,
6081 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
6082 Results.push_back(Elt: Tmp4);
6083 Results.push_back(Elt: Tmp4.getValue(R: 1));
6084 break;
6085 }
6086
6087 case ISD::STRICT_FADD:
6088 case ISD::STRICT_FSUB:
6089 case ISD::STRICT_FMUL:
6090 case ISD::STRICT_FDIV:
6091 case ISD::STRICT_FMINNUM:
6092 case ISD::STRICT_FMAXNUM:
6093 case ISD::STRICT_FREM:
6094 case ISD::STRICT_FPOW:
6095 case ISD::STRICT_FATAN2:
6096 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6097 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6098 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6099 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
6100 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
6101 N2: Tmp2.getValue(R: 1));
6102 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6103 Ops: {Tmp3, Tmp1, Tmp2});
6104 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6105 Ops: {Tmp1.getValue(R: 1), Tmp1,
6106 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6107 Results.push_back(Elt: Tmp1);
6108 Results.push_back(Elt: Tmp1.getValue(R: 1));
6109 break;
6110 case ISD::FMA:
6111 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6112 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
6113 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
6114 Results.push_back(
6115 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
6116 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
6117 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6118 break;
6119 case ISD::STRICT_FMA:
6120 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6121 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6122 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6123 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
6124 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6125 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
6126 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
6127 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
6128 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6129 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
6130 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6131 Ops: {Tmp4.getValue(R: 1), Tmp4,
6132 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6133 Results.push_back(Elt: Tmp4);
6134 Results.push_back(Elt: Tmp4.getValue(R: 1));
6135 break;
6136 case ISD::FCOPYSIGN:
6137 case ISD::FLDEXP:
6138 case ISD::FPOWI: {
6139 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6140 Tmp2 = Node->getOperand(Num: 1);
6141 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
6142
6143 // fcopysign doesn't change anything but the sign bit, so
6144 // (fp_round (fcopysign (fpext a), b))
6145 // is as precise as
6146 // (fp_round (fpext a))
6147 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
6148 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
6149 Results.push_back(
6150 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
6151 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
6152 break;
6153 }
6154 case ISD::STRICT_FLDEXP: {
6155 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6156 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6157 Tmp2 = Node->getOperand(Num: 2);
6158 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
6159 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
6160 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6161 Ops: {Tmp3.getValue(R: 1), Tmp3,
6162 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6163 Results.push_back(Elt: Tmp4);
6164 Results.push_back(Elt: Tmp4.getValue(R: 1));
6165 break;
6166 }
6167 case ISD::STRICT_FPOWI:
6168 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6169 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6170 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6171 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
6172 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6173 Ops: {Tmp2.getValue(R: 1), Tmp2,
6174 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6175 Results.push_back(Elt: Tmp3);
6176 Results.push_back(Elt: Tmp3.getValue(R: 1));
6177 break;
6178 case ISD::FFREXP: {
6179 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6180 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
6181
6182 Results.push_back(
6183 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6184 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6185
6186 Results.push_back(Elt: Tmp2.getValue(R: 1));
6187 break;
6188 }
6189 case ISD::FMODF:
6190 case ISD::FSINCOS:
6191 case ISD::FSINCOSPI: {
6192 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6193 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
6194 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
6195 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
6196 Results.push_back(
6197 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
6198 break;
6199 }
6200 case ISD::FFLOOR:
6201 case ISD::FCEIL:
6202 case ISD::FRINT:
6203 case ISD::FNEARBYINT:
6204 case ISD::FROUND:
6205 case ISD::FROUNDEVEN:
6206 case ISD::FTRUNC:
6207 case ISD::FNEG:
6208 case ISD::FSQRT:
6209 case ISD::FSIN:
6210 case ISD::FCOS:
6211 case ISD::FTAN:
6212 case ISD::FASIN:
6213 case ISD::FACOS:
6214 case ISD::FATAN:
6215 case ISD::FSINH:
6216 case ISD::FCOSH:
6217 case ISD::FTANH:
6218 case ISD::FLOG:
6219 case ISD::FLOG2:
6220 case ISD::FLOG10:
6221 case ISD::FABS:
6222 case ISD::FEXP:
6223 case ISD::FEXP2:
6224 case ISD::FEXP10:
6225 case ISD::FCANONICALIZE:
6226 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6227 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6228 Results.push_back(
6229 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6230 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6231 break;
6232 case ISD::STRICT_FFLOOR:
6233 case ISD::STRICT_FCEIL:
6234 case ISD::STRICT_FRINT:
6235 case ISD::STRICT_FNEARBYINT:
6236 case ISD::STRICT_FROUND:
6237 case ISD::STRICT_FROUNDEVEN:
6238 case ISD::STRICT_FTRUNC:
6239 case ISD::STRICT_FSQRT:
6240 case ISD::STRICT_FSIN:
6241 case ISD::STRICT_FCOS:
6242 case ISD::STRICT_FTAN:
6243 case ISD::STRICT_FASIN:
6244 case ISD::STRICT_FACOS:
6245 case ISD::STRICT_FATAN:
6246 case ISD::STRICT_FSINH:
6247 case ISD::STRICT_FCOSH:
6248 case ISD::STRICT_FTANH:
6249 case ISD::STRICT_FLOG:
6250 case ISD::STRICT_FLOG2:
6251 case ISD::STRICT_FLOG10:
6252 case ISD::STRICT_FEXP:
6253 case ISD::STRICT_FEXP2:
6254 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6255 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6256 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6257 Ops: {Tmp1.getValue(R: 1), Tmp1});
6258 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6259 Ops: {Tmp2.getValue(R: 1), Tmp2,
6260 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6261 Results.push_back(Elt: Tmp3);
6262 Results.push_back(Elt: Tmp3.getValue(R: 1));
6263 break;
6264 case ISD::LLROUND:
6265 case ISD::LROUND:
6266 case ISD::LRINT:
6267 case ISD::LLRINT:
6268 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6269 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
6270 Results.push_back(Elt: Tmp2);
6271 break;
6272 case ISD::STRICT_LLROUND:
6273 case ISD::STRICT_LROUND:
6274 case ISD::STRICT_LRINT:
6275 case ISD::STRICT_LLRINT:
6276 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6277 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6278 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6279 Ops: {Tmp1.getValue(R: 1), Tmp1});
6280 Results.push_back(Elt: Tmp2);
6281 Results.push_back(Elt: Tmp2.getValue(R: 1));
6282 break;
6283 case ISD::BUILD_VECTOR: {
6284 MVT EltVT = OVT.getVectorElementType();
6285 MVT NewEltVT = NVT.getVectorElementType();
6286
6287 // Handle bitcasts to a different vector type with the same total bit size
6288 //
6289 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
6290 // =>
6291 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
6292
6293 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6294 "Invalid promote type for build_vector");
6295 assert(NewEltVT.bitsLE(EltVT) && "not handled");
6296
6297 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6298
6299 SmallVector<SDValue, 8> NewOps;
6300 for (const SDValue &Op : Node->op_values())
6301 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
6302
6303 SDLoc SL(Node);
6304 SDValue Concat =
6305 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
6306 DL: SL, VT: NVT, Ops: NewOps);
6307 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6308 Results.push_back(Elt: CvtVec);
6309 break;
6310 }
6311 case ISD::EXTRACT_VECTOR_ELT: {
6312 MVT EltVT = OVT.getVectorElementType();
6313 MVT NewEltVT = NVT.getVectorElementType();
6314
6315 // Handle bitcasts to a different vector type with the same total bit size.
6316 //
6317 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
6318 // =>
6319 // v4i32:castx = bitcast x:v2i64
6320 //
6321 // i64 = bitcast
6322 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
6323 // (i32 (extract_vector_elt castx, (2 * y + 1)))
6324 //
6325
6326 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6327 "Invalid promote type for extract_vector_elt");
6328 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6329
6330 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6331 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6332
6333 SDValue Idx = Node->getOperand(Num: 1);
6334 EVT IdxVT = Idx.getValueType();
6335 SDLoc SL(Node);
6336 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
6337 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6338
6339 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6340
6341 SmallVector<SDValue, 8> NewOps;
6342 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6343 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6344 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6345
6346 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6347 N1: CastVec, N2: TmpIdx);
6348 NewOps.push_back(Elt);
6349 }
6350
6351 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
6352 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
6353 break;
6354 }
6355 case ISD::INSERT_VECTOR_ELT: {
6356 MVT EltVT = OVT.getVectorElementType();
6357 MVT NewEltVT = NVT.getVectorElementType();
6358
6359 // Handle bitcasts to a different vector type with the same total bit size
6360 //
6361 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
6362 // =>
6363 // v4i32:castx = bitcast x:v2i64
6364 // v2i32:casty = bitcast y:i64
6365 //
6366 // v2i64 = bitcast
6367 // (v4i32 insert_vector_elt
6368 // (v4i32 insert_vector_elt v4i32:castx,
6369 // (extract_vector_elt casty, 0), 2 * z),
6370 // (extract_vector_elt casty, 1), (2 * z + 1))
6371
6372 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6373 "Invalid promote type for insert_vector_elt");
6374 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6375
6376 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6377 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6378
6379 SDValue Val = Node->getOperand(Num: 1);
6380 SDValue Idx = Node->getOperand(Num: 2);
6381 EVT IdxVT = Idx.getValueType();
6382 SDLoc SL(Node);
6383
6384 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
6385 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6386
6387 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6388 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6389
6390 SDValue NewVec = CastVec;
6391 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6392 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6393 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6394
6395 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6396 N1: CastVal, N2: IdxOffset);
6397
6398 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
6399 N1: NewVec, N2: Elt, N3: InEltIdx);
6400 }
6401
6402 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
6403 break;
6404 }
6405 case ISD::SCALAR_TO_VECTOR: {
6406 MVT EltVT = OVT.getVectorElementType();
6407 MVT NewEltVT = NVT.getVectorElementType();
6408
6409 // Handle bitcasts to different vector type with the same total bit size.
6410 //
6411 // e.g. v2i64 = scalar_to_vector x:i64
6412 // =>
6413 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
6414 //
6415
6416 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6417 SDValue Val = Node->getOperand(Num: 0);
6418 SDLoc SL(Node);
6419
6420 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6421 SDValue Undef = DAG.getUNDEF(VT: MidVT);
6422
6423 SmallVector<SDValue, 8> NewElts;
6424 NewElts.push_back(Elt: CastVal);
6425 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
6426 NewElts.push_back(Elt: Undef);
6427
6428 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
6429 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6430 Results.push_back(Elt: CvtVec);
6431 break;
6432 }
6433 case ISD::ATOMIC_SWAP:
6434 case ISD::ATOMIC_STORE: {
6435 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6436 SDLoc SL(Node);
6437 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
6438 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6439 "unexpected promotion type");
6440 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6441 "unexpected atomic_swap with illegal type");
6442
6443 SDValue Op0 = AM->getBasePtr();
6444 SDValue Op1 = CastVal;
6445
6446 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
6447 // but really it should merge with ISD::STORE.
6448 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6449 std::swap(a&: Op0, b&: Op1);
6450
6451 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
6452 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
6453
6454 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6455 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6456 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6457 } else
6458 Results.push_back(Elt: NewAtomic);
6459 break;
6460 }
6461 case ISD::ATOMIC_LOAD: {
6462 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6463 SDLoc SL(Node);
6464 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6465 "unexpected promotion type");
6466 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6467 "unexpected atomic_load with illegal type");
6468
6469 SDValue NewAtomic =
6470 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
6471 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
6472 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6473 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6474 break;
6475 }
6476 case ISD::SPLAT_VECTOR: {
6477 SDValue Scalar = Node->getOperand(Num: 0);
6478 MVT ScalarType = Scalar.getSimpleValueType();
6479 MVT NewScalarType = NVT.getVectorElementType();
6480 if (ScalarType.isInteger()) {
6481 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6482 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6483 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
6484 break;
6485 }
6486 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6487 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6488 Results.push_back(
6489 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6490 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6491 break;
6492 }
6493 case ISD::VECREDUCE_FMAX:
6494 case ISD::VECREDUCE_FMIN:
6495 case ISD::VECREDUCE_FMAXIMUM:
6496 case ISD::VECREDUCE_FMINIMUM:
6497 case ISD::VP_REDUCE_FMAX:
6498 case ISD::VP_REDUCE_FMIN:
6499 case ISD::VP_REDUCE_FMAXIMUM:
6500 case ISD::VP_REDUCE_FMINIMUM:
6501 Results.push_back(Elt: PromoteReduction(Node));
6502 break;
6503 }
6504
6505 // Replace the original node with the legalized result.
6506 if (!Results.empty()) {
6507 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6508 ReplaceNode(Old: Node, New: Results.data());
6509 } else
6510 LLVM_DEBUG(dbgs() << "Could not promote node\n");
6511}
6512
6513/// This is the entry point for the file.
6514void SelectionDAG::Legalize() {
6515 AssignTopologicalOrder();
6516
6517 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6518 // Use a delete listener to remove nodes which were deleted during
6519 // legalization from LegalizeNodes. This is needed to handle the situation
6520 // where a new node is allocated by the object pool to the same address of a
6521 // previously deleted node.
6522 DAGNodeDeletedListener DeleteListener(
6523 *this,
6524 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
6525
6526 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6527
6528 // Visit all the nodes. We start in topological order, so that we see
6529 // nodes with their original operands intact. Legalization can produce
6530 // new nodes which may themselves need to be legalized. Iterate until all
6531 // nodes have been legalized.
6532 while (true) {
6533 bool AnyLegalized = false;
6534 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6535 --NI;
6536
6537 SDNode *N = &*NI;
6538 if (N->use_empty() && N != getRoot().getNode()) {
6539 ++NI;
6540 DeleteNode(N);
6541 continue;
6542 }
6543
6544 if (LegalizedNodes.insert(Ptr: N).second) {
6545 AnyLegalized = true;
6546 Legalizer.LegalizeOp(Node: N);
6547
6548 if (N->use_empty() && N != getRoot().getNode()) {
6549 ++NI;
6550 DeleteNode(N);
6551 }
6552 }
6553 }
6554 if (!AnyLegalized)
6555 break;
6556
6557 }
6558
6559 // Remove dead nodes now.
6560 RemoveDeadNodes();
6561}
6562
6563bool SelectionDAG::LegalizeOp(SDNode *N,
6564 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6565 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6566 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6567
6568 // Directly insert the node in question, and legalize it. This will recurse
6569 // as needed through operands.
6570 LegalizedNodes.insert(Ptr: N);
6571 Legalizer.LegalizeOp(Node: N);
6572
6573 return LegalizedNodes.count(Ptr: N);
6574}
6575