1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/ConstantFolding.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/CodeGen/ISDOpcodes.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineJumpTableInfo.h"
27#include "llvm/CodeGen/MachineMemOperand.h"
28#include "llvm/CodeGen/RuntimeLibcallUtil.h"
29#include "llvm/CodeGen/SelectionDAG.h"
30#include "llvm/CodeGen/SelectionDAGNodes.h"
31#include "llvm/CodeGen/TargetFrameLowering.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetSubtargetInfo.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/CodeGenTypes/MachineValueType.h"
36#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/DerivedTypes.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/Metadata.h"
42#include "llvm/IR/Type.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/Compiler.h"
45#include "llvm/Support/Debug.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Support/MathExtras.h"
48#include "llvm/Support/raw_ostream.h"
49#include "llvm/Target/TargetMachine.h"
50#include "llvm/Target/TargetOptions.h"
51#include <cassert>
52#include <cstdint>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "legalizedag"
59
60namespace {
61
62/// Keeps track of state when getting the sign of a floating-point value as an
63/// integer.
64struct FloatSignAsInt {
65 EVT FloatVT;
66 SDValue Chain;
67 SDValue FloatPtr;
68 SDValue IntPtr;
69 MachinePointerInfo IntPointerInfo;
70 MachinePointerInfo FloatPointerInfo;
71 SDValue IntValue;
72 APInt SignMask;
73 uint8_t SignBit;
74};
75
76//===----------------------------------------------------------------------===//
77/// This takes an arbitrary SelectionDAG as input and
78/// hacks on it until the target machine can handle it. This involves
79/// eliminating value sizes the machine cannot handle (promoting small sizes to
80/// large sizes or splitting up large values into small values) as well as
81/// eliminating operations the machine cannot handle.
82///
83/// This code also does a small amount of optimization and recognition of idioms
84/// as part of its processing. For example, if a target does not support a
85/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
86/// will attempt merge setcc and brc instructions into brcc's.
87class SelectionDAGLegalize {
88 const TargetMachine &TM;
89 const TargetLowering &TLI;
90 SelectionDAG &DAG;
91
92 /// The set of nodes which have already been legalized. We hold a
93 /// reference to it in order to update as necessary on node deletion.
94 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
95
96 /// A set of all the nodes updated during legalization.
97 SmallSetVector<SDNode *, 16> *UpdatedNodes;
98
99 EVT getSetCCResultType(EVT VT) const {
100 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
101 }
102
103 // Libcall insertion helpers.
104
105public:
106 SelectionDAGLegalize(SelectionDAG &DAG,
107 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
108 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112 /// Legalizes the given operation.
113 void LegalizeOp(SDNode *Node);
114
115private:
116 SDValue OptimizeFloatStore(StoreSDNode *ST);
117
118 void LegalizeLoadOps(SDNode *Node);
119 void LegalizeStoreOps(SDNode *Node);
120
121 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
122
123 /// Return a vector shuffle operation which
124 /// performs the same shuffe in terms of order or result bytes, but on a type
125 /// whose vector element type is narrower than the original shuffle type.
126 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
127 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
128 SDValue N1, SDValue N2,
129 ArrayRef<int> Mask) const;
130
131 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
132 TargetLowering::ArgListTy &&Args, bool isSigned);
133 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
134
135 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
136 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
137 SmallVectorImpl<SDValue> &Results);
138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
139 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
140 RTLIB::Libcall Call_F128,
141 RTLIB::Libcall Call_PPCF128,
142 SmallVectorImpl<SDValue> &Results);
143 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
144 RTLIB::Libcall Call_I8,
145 RTLIB::Libcall Call_I16,
146 RTLIB::Libcall Call_I32,
147 RTLIB::Libcall Call_I64,
148 RTLIB::Libcall Call_I128);
149 void ExpandArgFPLibCall(SDNode *Node,
150 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
151 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
152 RTLIB::Libcall Call_PPCF128,
153 SmallVectorImpl<SDValue> &Results);
154 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156
157 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
158 const SDLoc &dl);
159 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160 const SDLoc &dl, SDValue ChainIn);
161 SDValue ExpandBUILD_VECTOR(SDNode *Node);
162 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
163 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
164 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
165 SmallVectorImpl<SDValue> &Results);
166 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
167 SDValue Value) const;
168 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
169 SDValue NewIntValue) const;
170 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
171 SDValue ExpandFABS(SDNode *Node) const;
172 SDValue ExpandFNEG(SDNode *Node) const;
173 SDValue expandLdexp(SDNode *Node) const;
174 SDValue expandFrexp(SDNode *Node) const;
175
176 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
177 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
178 SmallVectorImpl<SDValue> &Results);
179 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
180 SmallVectorImpl<SDValue> &Results);
181 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
182
183 /// Implements vector reduce operation promotion.
184 ///
185 /// All vector operands are promoted to a vector type with larger element
186 /// type, and the start value is promoted to a larger scalar type. Then the
187 /// result is truncated back to the original scalar type.
188 SDValue PromoteReduction(SDNode *Node);
189
190 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
191
192 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
193 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
194 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
195
196 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
197 SDValue ExpandConstant(ConstantSDNode *CP);
198
199 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
200 bool ExpandNode(SDNode *Node);
201 void ConvertNodeToLibcall(SDNode *Node);
202 void PromoteNode(SDNode *Node);
203
204public:
205 // Node replacement helpers
206
207 void ReplacedNode(SDNode *N) {
208 LegalizedNodes.erase(Ptr: N);
209 if (UpdatedNodes)
210 UpdatedNodes->insert(X: N);
211 }
212
213 void ReplaceNode(SDNode *Old, SDNode *New) {
214 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
215 dbgs() << " with: "; New->dump(&DAG));
216
217 assert(Old->getNumValues() == New->getNumValues() &&
218 "Replacing one node with another that produces a different number "
219 "of values!");
220 DAG.ReplaceAllUsesWith(From: Old, To: New);
221 if (UpdatedNodes)
222 UpdatedNodes->insert(X: New);
223 ReplacedNode(N: Old);
224 }
225
226 void ReplaceNode(SDValue Old, SDValue New) {
227 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
228 dbgs() << " with: "; New->dump(&DAG));
229
230 DAG.ReplaceAllUsesWith(From: Old, To: New);
231 if (UpdatedNodes)
232 UpdatedNodes->insert(X: New.getNode());
233 ReplacedNode(N: Old.getNode());
234 }
235
236 void ReplaceNode(SDNode *Old, const SDValue *New) {
237 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
238
239 DAG.ReplaceAllUsesWith(From: Old, To: New);
240 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
241 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
242 New[i]->dump(&DAG));
243 if (UpdatedNodes)
244 UpdatedNodes->insert(X: New[i].getNode());
245 }
246 ReplacedNode(N: Old);
247 }
248
249 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
250 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
251 dbgs() << " with: "; New->dump(&DAG));
252
253 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
254 if (UpdatedNodes)
255 UpdatedNodes->insert(X: New.getNode());
256 ReplacedNode(N: Old.getNode());
257 }
258};
259
260} // end anonymous namespace
261
262// Helper function that generates an MMO that considers the alignment of the
263// stack, and the size of the stack object
264static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
265 MachineFunction &MF,
266 bool isObjectScalable) {
267 auto &MFI = MF.getFrameInfo();
268 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
269 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
270 LocationSize ObjectSize = isObjectScalable
271 ? LocationSize::beforeOrAfterPointer()
272 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
273 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
274 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
275}
276
277/// Return a vector shuffle operation which
278/// performs the same shuffle in terms of order or result bytes, but on a type
279/// whose vector element type is narrower than the original shuffle type.
280/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
281SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
282 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
283 ArrayRef<int> Mask) const {
284 unsigned NumMaskElts = VT.getVectorNumElements();
285 unsigned NumDestElts = NVT.getVectorNumElements();
286 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
287
288 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
289
290 if (NumEltsGrowth == 1)
291 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
292
293 SmallVector<int, 8> NewMask;
294 for (unsigned i = 0; i != NumMaskElts; ++i) {
295 int Idx = Mask[i];
296 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
297 if (Idx < 0)
298 NewMask.push_back(Elt: -1);
299 else
300 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
301 }
302 }
303 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
304 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
305 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
306}
307
308/// Expands the ConstantFP node to an integer constant or
309/// a load from the constant pool.
310SDValue
311SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
312 bool Extend = false;
313 SDLoc dl(CFP);
314
315 // If a FP immediate is precise when represented as a float and if the
316 // target can do an extending load from float to double, we put it into
317 // the constant pool as a float, even if it's is statically typed as a
318 // double. This shrinks FP constants and canonicalizes them for targets where
319 // an FP extending load is the same cost as a normal load (such as on the x87
320 // fp stack or PPC FP unit).
321 EVT VT = CFP->getValueType(ResNo: 0);
322 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
323 if (!UseCP) {
324 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
325 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
326 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
327 }
328
329 APFloat APF = CFP->getValueAPF();
330 EVT OrigVT = VT;
331 EVT SVT = VT;
332
333 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
334 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
335 if (!APF.isSignaling()) {
336 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
337 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
338 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
339 // Only do this if the target has a native EXTLOAD instruction from
340 // smaller type.
341 TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: OrigVT, MemVT: SVT) &&
342 TLI.ShouldShrinkFPConstant(OrigVT)) {
343 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
344 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
345 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
346 VT = SVT;
347 Extend = true;
348 }
349 }
350 }
351
352 SDValue CPIdx =
353 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
354 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
355 if (Extend) {
356 SDValue Result = DAG.getExtLoad(
357 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
358 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
359 Alignment);
360 return Result;
361 }
362 SDValue Result = DAG.getLoad(
363 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
364 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
365 return Result;
366}
367
368/// Expands the Constant node to a load from the constant pool.
369SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
370 SDLoc dl(CP);
371 EVT VT = CP->getValueType(ResNo: 0);
372 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
373 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
374 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
375 SDValue Result = DAG.getLoad(
376 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
377 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
378 return Result;
379}
380
381SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
382 SDValue Vec = Op.getOperand(i: 0);
383 SDValue Val = Op.getOperand(i: 1);
384 SDValue Idx = Op.getOperand(i: 2);
385 SDLoc dl(Op);
386
387 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
388 // SCALAR_TO_VECTOR requires that the type of the value being inserted
389 // match the element type of the vector being created, except for
390 // integers in which case the inserted value can be over width.
391 EVT EltVT = Vec.getValueType().getVectorElementType();
392 if (Val.getValueType() == EltVT ||
393 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
394 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
395 VT: Vec.getValueType(), Operand: Val);
396
397 unsigned NumElts = Vec.getValueType().getVectorNumElements();
398 // We generate a shuffle of InVec and ScVec, so the shuffle mask
399 // should be 0,1,2,3,4,5... with the appropriate element replaced with
400 // elt 0 of the RHS.
401 SmallVector<int, 8> ShufOps;
402 for (unsigned i = 0; i != NumElts; ++i)
403 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
404
405 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
406 }
407 }
408 return ExpandInsertToVectorThroughStack(Op);
409}
410
411SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
412 if (!ISD::isNormalStore(N: ST))
413 return SDValue();
414
415 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
416 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
417 // FIXME: move this to the DAG Combiner! Note that we can't regress due
418 // to phase ordering between legalized code and the dag combiner. This
419 // probably means that we need to integrate dag combiner and legalizer
420 // together.
421 // We generally can't do this one for long doubles.
422 SDValue Chain = ST->getChain();
423 SDValue Ptr = ST->getBasePtr();
424 SDValue Value = ST->getValue();
425 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
426 AAMDNodes AAInfo = ST->getAAInfo();
427 SDLoc dl(ST);
428
429 // Don't optimise TargetConstantFP
430 if (Value.getOpcode() == ISD::TargetConstantFP)
431 return SDValue();
432
433 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
434 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
435 TLI.isTypeLegal(VT: MVT::i32)) {
436 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
437 bitcastToAPInt().zextOrTrunc(width: 32),
438 DL: SDLoc(CFP), VT: MVT::i32);
439 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
440 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
441 }
442
443 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
444 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
445 // If this target supports 64-bit registers, do a single 64-bit store.
446 if (TLI.isTypeLegal(VT: MVT::i64)) {
447 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
448 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
449 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
450 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
451 }
452
453 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
454 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
455 // stores. If the target supports neither 32- nor 64-bits, this
456 // xform is certainly not worth it.
457 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
458 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
459 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
460 if (DAG.getDataLayout().isBigEndian())
461 std::swap(a&: Lo, b&: Hi);
462
463 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
464 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
465 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
466 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
467 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
468 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
469
470 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
471 }
472 }
473 }
474 return SDValue();
475}
476
477void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
478 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
479 SDValue Chain = ST->getChain();
480 SDValue Ptr = ST->getBasePtr();
481 SDLoc dl(Node);
482
483 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
484 AAMDNodes AAInfo = ST->getAAInfo();
485
486 if (!ST->isTruncatingStore()) {
487 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
488 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
489 ReplaceNode(Old: ST, New: OptStore);
490 return;
491 }
492
493 SDValue Value = ST->getValue();
494 MVT VT = Value.getSimpleValueType();
495 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
496 default: llvm_unreachable("This action is not supported yet!");
497 case TargetLowering::Legal: {
498 // If this is an unaligned store and the target doesn't support it,
499 // expand it.
500 EVT MemVT = ST->getMemoryVT();
501 const DataLayout &DL = DAG.getDataLayout();
502 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
503 MMO: *ST->getMemOperand())) {
504 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
505 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
506 ReplaceNode(Old: SDValue(ST, 0), New: Result);
507 } else
508 LLVM_DEBUG(dbgs() << "Legal store\n");
509 break;
510 }
511 case TargetLowering::Custom: {
512 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
513 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
514 if (Res && Res != SDValue(Node, 0))
515 ReplaceNode(Old: SDValue(Node, 0), New: Res);
516 return;
517 }
518 case TargetLowering::Promote: {
519 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
520 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
521 "Can only promote stores to same size type");
522 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
523 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
524 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
525 ReplaceNode(Old: SDValue(Node, 0), New: Result);
526 break;
527 }
528 }
529 return;
530 }
531
532 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
533 SDValue Value = ST->getValue();
534 EVT StVT = ST->getMemoryVT();
535 TypeSize StWidth = StVT.getSizeInBits();
536 TypeSize StSize = StVT.getStoreSizeInBits();
537 auto &DL = DAG.getDataLayout();
538
539 if (StWidth != StSize) {
540 // Promote to a byte-sized store with upper bits zero if not
541 // storing an integral number of bytes. For example, promote
542 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
543 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
544 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
545 SDValue Result =
546 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
547 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
548 ReplaceNode(Old: SDValue(Node, 0), New: Result);
549 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
550 // If not storing a power-of-2 number of bits, expand as two stores.
551 assert(!StVT.isVector() && "Unsupported truncstore!");
552 unsigned StWidthBits = StWidth.getFixedValue();
553 unsigned LogStWidth = Log2_32(Value: StWidthBits);
554 assert(LogStWidth < 32);
555 unsigned RoundWidth = 1 << LogStWidth;
556 assert(RoundWidth < StWidthBits);
557 unsigned ExtraWidth = StWidthBits - RoundWidth;
558 assert(ExtraWidth < RoundWidth);
559 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
560 "Store size not an integral number of bytes!");
561 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
562 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
563 SDValue Lo, Hi;
564 unsigned IncrementSize;
565
566 if (DL.isLittleEndian()) {
567 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
568 // Store the bottom RoundWidth bits.
569 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
570 SVT: RoundVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
571
572 // Store the remaining ExtraWidth bits.
573 IncrementSize = RoundWidth / 8;
574 Ptr =
575 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
576 Hi = DAG.getNode(
577 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
578 N2: DAG.getConstant(Val: RoundWidth, DL: dl,
579 VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL)));
580 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
581 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
582 SVT: ExtraVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
583 } else {
584 // Big endian - avoid unaligned stores.
585 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
586 // Store the top RoundWidth bits.
587 Hi = DAG.getNode(
588 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
589 N2: DAG.getConstant(Val: ExtraWidth, DL: dl,
590 VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL)));
591 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
592 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
593
594 // Store the remaining ExtraWidth bits.
595 IncrementSize = RoundWidth / 8;
596 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
597 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
598 VT: Ptr.getValueType()));
599 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
600 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
601 SVT: ExtraVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
602 }
603
604 // The order of the stores doesn't matter.
605 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
606 ReplaceNode(Old: SDValue(Node, 0), New: Result);
607 } else {
608 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT)) {
609 default: llvm_unreachable("This action is not supported yet!");
610 case TargetLowering::Legal: {
611 EVT MemVT = ST->getMemoryVT();
612 // If this is an unaligned store and the target doesn't support it,
613 // expand it.
614 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
615 MMO: *ST->getMemOperand())) {
616 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
617 ReplaceNode(Old: SDValue(ST, 0), New: Result);
618 }
619 break;
620 }
621 case TargetLowering::Custom: {
622 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
623 if (Res && Res != SDValue(Node, 0))
624 ReplaceNode(Old: SDValue(Node, 0), New: Res);
625 return;
626 }
627 case TargetLowering::Expand:
628 assert(!StVT.isVector() &&
629 "Vector Stores are handled in LegalizeVectorOps");
630
631 SDValue Result;
632
633 // TRUNCSTORE:i16 i32 -> STORE i16
634 if (TLI.isTypeLegal(VT: StVT)) {
635 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
636 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
637 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
638 } else {
639 // The in-memory type isn't legal. Truncate to the type it would promote
640 // to, and then do a truncstore.
641 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
642 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
643 Operand: Value);
644 Result =
645 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: StVT,
646 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
647 }
648
649 ReplaceNode(Old: SDValue(Node, 0), New: Result);
650 break;
651 }
652 }
653}
654
655void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
656 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
657 SDValue Chain = LD->getChain(); // The chain.
658 SDValue Ptr = LD->getBasePtr(); // The base pointer.
659 SDValue Value; // The value returned by the load op.
660 SDLoc dl(Node);
661
662 ISD::LoadExtType ExtType = LD->getExtensionType();
663 if (ExtType == ISD::NON_EXTLOAD) {
664 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
665 MVT VT = Node->getSimpleValueType(ResNo: 0);
666 SDValue RVal = SDValue(Node, 0);
667 SDValue RChain = SDValue(Node, 1);
668
669 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
670 default: llvm_unreachable("This action is not supported yet!");
671 case TargetLowering::Legal: {
672 EVT MemVT = LD->getMemoryVT();
673 const DataLayout &DL = DAG.getDataLayout();
674 // If this is an unaligned load and the target doesn't support it,
675 // expand it.
676 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
677 MMO: *LD->getMemOperand())) {
678 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
679 }
680 break;
681 }
682 case TargetLowering::Custom:
683 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
684 RVal = Res;
685 RChain = Res.getValue(R: 1);
686 }
687 break;
688
689 case TargetLowering::Promote: {
690 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
691 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
692 "Can only promote loads to same size type");
693
694 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
695 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
696 RChain = Res.getValue(R: 1);
697 break;
698 }
699 }
700 if (RChain.getNode() != Node) {
701 assert(RVal.getNode() != Node && "Load must be completely replaced");
702 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
703 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
704 if (UpdatedNodes) {
705 UpdatedNodes->insert(X: RVal.getNode());
706 UpdatedNodes->insert(X: RChain.getNode());
707 }
708 ReplacedNode(N: Node);
709 }
710 return;
711 }
712
713 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
714 EVT SrcVT = LD->getMemoryVT();
715 TypeSize SrcWidth = SrcVT.getSizeInBits();
716 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
717 AAMDNodes AAInfo = LD->getAAInfo();
718
719 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
720 // Some targets pretend to have an i1 loading operation, and actually
721 // load an i8. This trick is correct for ZEXTLOAD because the top 7
722 // bits are guaranteed to be zero; it helps the optimizers understand
723 // that these bits are zero. It is also useful for EXTLOAD, since it
724 // tells the optimizers that those bits are undefined. It would be
725 // nice to have an effective generic way of getting these benefits...
726 // Until such a way is found, don't insist on promoting i1 here.
727 (SrcVT != MVT::i1 ||
728 TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1) ==
729 TargetLowering::Promote)) {
730 // Promote to a byte-sized load if not loading an integral number of
731 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
732 unsigned NewWidth = SrcVT.getStoreSizeInBits();
733 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
734 SDValue Ch;
735
736 // The extra bits are guaranteed to be zero, since we stored them that
737 // way. A zext load from NVT thus automatically gives zext from SrcVT.
738
739 ISD::LoadExtType NewExtType =
740 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
741
742 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
743 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
744 Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
745
746 Ch = Result.getValue(R: 1); // The chain.
747
748 if (ExtType == ISD::SEXTLOAD)
749 // Having the top bits zero doesn't help when sign extending.
750 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
751 VT: Result.getValueType(),
752 N1: Result, N2: DAG.getValueType(SrcVT));
753 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
754 // All the top bits are guaranteed to be zero - inform the optimizers.
755 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
756 VT: Result.getValueType(), N1: Result,
757 N2: DAG.getValueType(SrcVT));
758
759 Value = Result;
760 Chain = Ch;
761 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
762 // If not loading a power-of-2 number of bits, expand as two loads.
763 assert(!SrcVT.isVector() && "Unsupported extload!");
764 unsigned SrcWidthBits = SrcWidth.getFixedValue();
765 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
766 assert(LogSrcWidth < 32);
767 unsigned RoundWidth = 1 << LogSrcWidth;
768 assert(RoundWidth < SrcWidthBits);
769 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
770 assert(ExtraWidth < RoundWidth);
771 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
772 "Load size not an integral number of bytes!");
773 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
774 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
775 SDValue Lo, Hi, Ch;
776 unsigned IncrementSize;
777 auto &DL = DAG.getDataLayout();
778
779 if (DL.isLittleEndian()) {
780 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
781 // Load the bottom RoundWidth bits.
782 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
783 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getOriginalAlign(),
784 MMOFlags, AAInfo);
785
786 // Load the remaining ExtraWidth bits.
787 IncrementSize = RoundWidth / 8;
788 Ptr =
789 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
790 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
791 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
792 MemVT: ExtraVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
793
794 // Build a factor node to remember that this load is independent of
795 // the other one.
796 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
797 N2: Hi.getValue(R: 1));
798
799 // Move the top bits to the right place.
800 Hi = DAG.getNode(
801 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
802 N2: DAG.getConstant(Val: RoundWidth, DL: dl,
803 VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL)));
804
805 // Join the hi and lo parts.
806 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
807 } else {
808 // Big endian - avoid unaligned loads.
809 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
810 // Load the top RoundWidth bits.
811 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
812 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getOriginalAlign(),
813 MMOFlags, AAInfo);
814
815 // Load the remaining ExtraWidth bits.
816 IncrementSize = RoundWidth / 8;
817 Ptr =
818 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
819 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
820 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
821 MemVT: ExtraVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
822
823 // Build a factor node to remember that this load is independent of
824 // the other one.
825 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
826 N2: Hi.getValue(R: 1));
827
828 // Move the top bits to the right place.
829 Hi = DAG.getNode(
830 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
831 N2: DAG.getConstant(Val: ExtraWidth, DL: dl,
832 VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL)));
833
834 // Join the hi and lo parts.
835 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
836 }
837
838 Chain = Ch;
839 } else {
840 bool isCustom = false;
841 switch (TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0),
842 MemVT: SrcVT.getSimpleVT())) {
843 default: llvm_unreachable("This action is not supported yet!");
844 case TargetLowering::Custom:
845 isCustom = true;
846 [[fallthrough]];
847 case TargetLowering::Legal:
848 Value = SDValue(Node, 0);
849 Chain = SDValue(Node, 1);
850
851 if (isCustom) {
852 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
853 Value = Res;
854 Chain = Res.getValue(R: 1);
855 }
856 } else {
857 // If this is an unaligned load and the target doesn't support it,
858 // expand it.
859 EVT MemVT = LD->getMemoryVT();
860 const DataLayout &DL = DAG.getDataLayout();
861 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
862 MMO: *LD->getMemOperand())) {
863 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
864 }
865 }
866 break;
867
868 case TargetLowering::Expand: {
869 EVT DestVT = Node->getValueType(ResNo: 0);
870 if (!TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SrcVT)) {
871 // If the source type is not legal, see if there is a legal extload to
872 // an intermediate type that we can then extend further.
873 EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT());
874 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
875 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
876 TLI.isLoadExtLegal(ExtType, ValVT: LoadVT, MemVT: SrcVT))) {
877 // If we are loading a legal type, this is a non-extload followed by a
878 // full extend.
879 ISD::LoadExtType MidExtType =
880 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
881
882 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
883 MemVT: SrcVT, MMO: LD->getMemOperand());
884 unsigned ExtendOp =
885 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
886 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
887 Chain = Load.getValue(R: 1);
888 break;
889 }
890
891 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
892 // normal undefined upper bits behavior to allow using an in-reg extend
893 // with the illegal FP type, so load as an integer and do the
894 // from-integer conversion.
895 EVT SVT = SrcVT.getScalarType();
896 if (SVT == MVT::f16 || SVT == MVT::bf16) {
897 EVT ISrcVT = SrcVT.changeTypeToInteger();
898 EVT IDestVT = DestVT.changeTypeToInteger();
899 EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT());
900
901 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
902 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
903 Value =
904 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
905 DL: dl, VT: DestVT, Operand: Result);
906 Chain = Result.getValue(R: 1);
907 break;
908 }
909 }
910
911 assert(!SrcVT.isVector() &&
912 "Vector Loads are handled in LegalizeVectorOps");
913
914 // FIXME: This does not work for vectors on most targets. Sign-
915 // and zero-extend operations are currently folded into extending
916 // loads, whether they are legal or not, and then we end up here
917 // without any support for legalizing them.
918 assert(ExtType != ISD::EXTLOAD &&
919 "EXTLOAD should always be supported!");
920 // Turn the unsupported load into an EXTLOAD followed by an
921 // explicit zero/sign extend inreg.
922 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
923 VT: Node->getValueType(ResNo: 0),
924 Chain, Ptr, MemVT: SrcVT,
925 MMO: LD->getMemOperand());
926 SDValue ValRes;
927 if (ExtType == ISD::SEXTLOAD)
928 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
929 VT: Result.getValueType(),
930 N1: Result, N2: DAG.getValueType(SrcVT));
931 else
932 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
933 Value = ValRes;
934 Chain = Result.getValue(R: 1);
935 break;
936 }
937 }
938 }
939
940 // Since loads produce two values, make sure to remember that we legalized
941 // both of them.
942 if (Chain.getNode() != Node) {
943 assert(Value.getNode() != Node && "Load must be completely replaced");
944 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
945 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
946 if (UpdatedNodes) {
947 UpdatedNodes->insert(X: Value.getNode());
948 UpdatedNodes->insert(X: Chain.getNode());
949 }
950 ReplacedNode(N: Node);
951 }
952}
953
954/// Return a legal replacement for the given operation, with all legal operands.
955void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
956 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
957
958 // Allow illegal target nodes and illegal registers.
959 if (Node->getOpcode() == ISD::TargetConstant ||
960 Node->getOpcode() == ISD::Register)
961 return;
962
963#ifndef NDEBUG
964 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
965 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
966 TargetLowering::TypeLegal &&
967 "Unexpected illegal type!");
968
969 for (const SDValue &Op : Node->op_values())
970 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
971 TargetLowering::TypeLegal ||
972 Op.getOpcode() == ISD::TargetConstant ||
973 Op.getOpcode() == ISD::Register) &&
974 "Unexpected illegal type!");
975#endif
976
977 // Figure out the correct action; the way to query this varies by opcode
978 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
979 bool SimpleFinishLegalizing = true;
980 switch (Node->getOpcode()) {
981 case ISD::INTRINSIC_W_CHAIN:
982 case ISD::INTRINSIC_WO_CHAIN:
983 case ISD::INTRINSIC_VOID:
984 case ISD::STACKSAVE:
985 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
986 break;
987 case ISD::GET_DYNAMIC_AREA_OFFSET:
988 Action = TLI.getOperationAction(Op: Node->getOpcode(),
989 VT: Node->getValueType(ResNo: 0));
990 break;
991 case ISD::VAARG:
992 Action = TLI.getOperationAction(Op: Node->getOpcode(),
993 VT: Node->getValueType(ResNo: 0));
994 if (Action != TargetLowering::Promote)
995 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
996 break;
997 case ISD::SET_FPENV:
998 case ISD::SET_FPMODE:
999 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1000 VT: Node->getOperand(Num: 1).getValueType());
1001 break;
1002 case ISD::FP_TO_FP16:
1003 case ISD::FP_TO_BF16:
1004 case ISD::SINT_TO_FP:
1005 case ISD::UINT_TO_FP:
1006 case ISD::EXTRACT_VECTOR_ELT:
1007 case ISD::LROUND:
1008 case ISD::LLROUND:
1009 case ISD::LRINT:
1010 case ISD::LLRINT:
1011 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1012 VT: Node->getOperand(Num: 0).getValueType());
1013 break;
1014 case ISD::STRICT_FP_TO_FP16:
1015 case ISD::STRICT_FP_TO_BF16:
1016 case ISD::STRICT_SINT_TO_FP:
1017 case ISD::STRICT_UINT_TO_FP:
1018 case ISD::STRICT_LRINT:
1019 case ISD::STRICT_LLRINT:
1020 case ISD::STRICT_LROUND:
1021 case ISD::STRICT_LLROUND:
1022 // These pseudo-ops are the same as the other STRICT_ ops except
1023 // they are registered with setOperationAction() using the input type
1024 // instead of the output type.
1025 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1026 VT: Node->getOperand(Num: 1).getValueType());
1027 break;
1028 case ISD::SIGN_EXTEND_INREG: {
1029 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1030 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1031 break;
1032 }
1033 case ISD::ATOMIC_STORE:
1034 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1035 VT: Node->getOperand(Num: 1).getValueType());
1036 break;
1037 case ISD::SELECT_CC:
1038 case ISD::STRICT_FSETCC:
1039 case ISD::STRICT_FSETCCS:
1040 case ISD::SETCC:
1041 case ISD::SETCCCARRY:
1042 case ISD::VP_SETCC:
1043 case ISD::BR_CC: {
1044 unsigned Opc = Node->getOpcode();
1045 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1046 : Opc == ISD::STRICT_FSETCC ? 3
1047 : Opc == ISD::STRICT_FSETCCS ? 3
1048 : Opc == ISD::SETCCCARRY ? 3
1049 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1050 : 1;
1051 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1052 : Opc == ISD::STRICT_FSETCC ? 1
1053 : Opc == ISD::STRICT_FSETCCS ? 1
1054 : 0;
1055 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1056 ISD::CondCode CCCode =
1057 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1058 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1059 if (Action == TargetLowering::Legal) {
1060 if (Node->getOpcode() == ISD::SELECT_CC)
1061 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1062 VT: Node->getValueType(ResNo: 0));
1063 else
1064 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1065 }
1066 break;
1067 }
1068 case ISD::LOAD:
1069 case ISD::STORE:
1070 // FIXME: Model these properly. LOAD and STORE are complicated, and
1071 // STORE expects the unlegalized operand in some cases.
1072 SimpleFinishLegalizing = false;
1073 break;
1074 case ISD::CALLSEQ_START:
1075 case ISD::CALLSEQ_END:
1076 // FIXME: This shouldn't be necessary. These nodes have special properties
1077 // dealing with the recursive nature of legalization. Removing this
1078 // special case should be done as part of making LegalizeDAG non-recursive.
1079 SimpleFinishLegalizing = false;
1080 break;
1081 case ISD::EXTRACT_ELEMENT:
1082 case ISD::GET_ROUNDING:
1083 case ISD::MERGE_VALUES:
1084 case ISD::EH_RETURN:
1085 case ISD::FRAME_TO_ARGS_OFFSET:
1086 case ISD::EH_DWARF_CFA:
1087 case ISD::EH_SJLJ_SETJMP:
1088 case ISD::EH_SJLJ_LONGJMP:
1089 case ISD::EH_SJLJ_SETUP_DISPATCH:
1090 // These operations lie about being legal: when they claim to be legal,
1091 // they should actually be expanded.
1092 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1093 if (Action == TargetLowering::Legal)
1094 Action = TargetLowering::Expand;
1095 break;
1096 case ISD::INIT_TRAMPOLINE:
1097 case ISD::ADJUST_TRAMPOLINE:
1098 case ISD::FRAMEADDR:
1099 case ISD::RETURNADDR:
1100 case ISD::ADDROFRETURNADDR:
1101 case ISD::SPONENTRY:
1102 // These operations lie about being legal: when they claim to be legal,
1103 // they should actually be custom-lowered.
1104 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1105 if (Action == TargetLowering::Legal)
1106 Action = TargetLowering::Custom;
1107 break;
1108 case ISD::CLEAR_CACHE:
1109 // This operation is typically going to be LibCall unless the target wants
1110 // something differrent.
1111 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1112 break;
1113 case ISD::READCYCLECOUNTER:
1114 case ISD::READSTEADYCOUNTER:
1115 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1116 // legalization might have expanded that to several smaller types.
1117 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1118 break;
1119 case ISD::READ_REGISTER:
1120 case ISD::WRITE_REGISTER:
1121 // Named register is legal in the DAG, but blocked by register name
1122 // selection if not implemented by target (to chose the correct register)
1123 // They'll be converted to Copy(To/From)Reg.
1124 Action = TargetLowering::Legal;
1125 break;
1126 case ISD::UBSANTRAP:
1127 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1128 if (Action == TargetLowering::Expand) {
1129 // replace ISD::UBSANTRAP with ISD::TRAP
1130 SDValue NewVal;
1131 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1132 N: Node->getOperand(Num: 0));
1133 ReplaceNode(Old: Node, New: NewVal.getNode());
1134 LegalizeOp(Node: NewVal.getNode());
1135 return;
1136 }
1137 break;
1138 case ISD::DEBUGTRAP:
1139 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1140 if (Action == TargetLowering::Expand) {
1141 // replace ISD::DEBUGTRAP with ISD::TRAP
1142 SDValue NewVal;
1143 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1144 N: Node->getOperand(Num: 0));
1145 ReplaceNode(Old: Node, New: NewVal.getNode());
1146 LegalizeOp(Node: NewVal.getNode());
1147 return;
1148 }
1149 break;
1150 case ISD::SADDSAT:
1151 case ISD::UADDSAT:
1152 case ISD::SSUBSAT:
1153 case ISD::USUBSAT:
1154 case ISD::SSHLSAT:
1155 case ISD::USHLSAT:
1156 case ISD::SCMP:
1157 case ISD::UCMP:
1158 case ISD::FP_TO_SINT_SAT:
1159 case ISD::FP_TO_UINT_SAT:
1160 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1161 break;
1162 case ISD::SMULFIX:
1163 case ISD::SMULFIXSAT:
1164 case ISD::UMULFIX:
1165 case ISD::UMULFIXSAT:
1166 case ISD::SDIVFIX:
1167 case ISD::SDIVFIXSAT:
1168 case ISD::UDIVFIX:
1169 case ISD::UDIVFIXSAT: {
1170 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1171 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1172 VT: Node->getValueType(ResNo: 0), Scale);
1173 break;
1174 }
1175 case ISD::MSCATTER:
1176 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1177 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1178 break;
1179 case ISD::MSTORE:
1180 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1181 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1182 break;
1183 case ISD::VP_SCATTER:
1184 Action = TLI.getOperationAction(
1185 Op: Node->getOpcode(),
1186 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1187 break;
1188 case ISD::VP_STORE:
1189 Action = TLI.getOperationAction(
1190 Op: Node->getOpcode(),
1191 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1192 break;
1193 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1194 Action = TLI.getOperationAction(
1195 Op: Node->getOpcode(),
1196 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1197 break;
1198 case ISD::VECREDUCE_FADD:
1199 case ISD::VECREDUCE_FMUL:
1200 case ISD::VECREDUCE_ADD:
1201 case ISD::VECREDUCE_MUL:
1202 case ISD::VECREDUCE_AND:
1203 case ISD::VECREDUCE_OR:
1204 case ISD::VECREDUCE_XOR:
1205 case ISD::VECREDUCE_SMAX:
1206 case ISD::VECREDUCE_SMIN:
1207 case ISD::VECREDUCE_UMAX:
1208 case ISD::VECREDUCE_UMIN:
1209 case ISD::VECREDUCE_FMAX:
1210 case ISD::VECREDUCE_FMIN:
1211 case ISD::VECREDUCE_FMAXIMUM:
1212 case ISD::VECREDUCE_FMINIMUM:
1213 case ISD::IS_FPCLASS:
1214 Action = TLI.getOperationAction(
1215 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1216 break;
1217 case ISD::VECREDUCE_SEQ_FADD:
1218 case ISD::VECREDUCE_SEQ_FMUL:
1219 case ISD::VP_REDUCE_FADD:
1220 case ISD::VP_REDUCE_FMUL:
1221 case ISD::VP_REDUCE_ADD:
1222 case ISD::VP_REDUCE_MUL:
1223 case ISD::VP_REDUCE_AND:
1224 case ISD::VP_REDUCE_OR:
1225 case ISD::VP_REDUCE_XOR:
1226 case ISD::VP_REDUCE_SMAX:
1227 case ISD::VP_REDUCE_SMIN:
1228 case ISD::VP_REDUCE_UMAX:
1229 case ISD::VP_REDUCE_UMIN:
1230 case ISD::VP_REDUCE_FMAX:
1231 case ISD::VP_REDUCE_FMIN:
1232 case ISD::VP_REDUCE_FMAXIMUM:
1233 case ISD::VP_REDUCE_FMINIMUM:
1234 case ISD::VP_REDUCE_SEQ_FADD:
1235 case ISD::VP_REDUCE_SEQ_FMUL:
1236 Action = TLI.getOperationAction(
1237 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1238 break;
1239 case ISD::VP_CTTZ_ELTS:
1240 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1241 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1242 VT: Node->getOperand(Num: 0).getValueType());
1243 break;
1244 default:
1245 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1246 Action = TLI.getCustomOperationAction(Op&: *Node);
1247 } else {
1248 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1249 }
1250 break;
1251 }
1252
1253 if (SimpleFinishLegalizing) {
1254 SDNode *NewNode = Node;
1255 switch (Node->getOpcode()) {
1256 default: break;
1257 case ISD::SHL:
1258 case ISD::SRL:
1259 case ISD::SRA:
1260 case ISD::ROTL:
1261 case ISD::ROTR: {
1262 // Legalizing shifts/rotates requires adjusting the shift amount
1263 // to the appropriate width.
1264 SDValue Op0 = Node->getOperand(Num: 0);
1265 SDValue Op1 = Node->getOperand(Num: 1);
1266 if (!Op1.getValueType().isVector()) {
1267 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1268 // The getShiftAmountOperand() may create a new operand node or
1269 // return the existing one. If new operand is created we need
1270 // to update the parent node.
1271 // Do not try to legalize SAO here! It will be automatically legalized
1272 // in the next round.
1273 if (SAO != Op1)
1274 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1275 }
1276 }
1277 break;
1278 case ISD::FSHL:
1279 case ISD::FSHR:
1280 case ISD::SRL_PARTS:
1281 case ISD::SRA_PARTS:
1282 case ISD::SHL_PARTS: {
1283 // Legalizing shifts/rotates requires adjusting the shift amount
1284 // to the appropriate width.
1285 SDValue Op0 = Node->getOperand(Num: 0);
1286 SDValue Op1 = Node->getOperand(Num: 1);
1287 SDValue Op2 = Node->getOperand(Num: 2);
1288 if (!Op2.getValueType().isVector()) {
1289 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1290 // The getShiftAmountOperand() may create a new operand node or
1291 // return the existing one. If new operand is created we need
1292 // to update the parent node.
1293 if (SAO != Op2)
1294 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1295 }
1296 break;
1297 }
1298 }
1299
1300 if (NewNode != Node) {
1301 ReplaceNode(Old: Node, New: NewNode);
1302 Node = NewNode;
1303 }
1304 switch (Action) {
1305 case TargetLowering::Legal:
1306 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1307 return;
1308 case TargetLowering::Custom:
1309 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1310 // FIXME: The handling for custom lowering with multiple results is
1311 // a complete mess.
1312 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1313 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1314 return;
1315
1316 if (Node->getNumValues() == 1) {
1317 // Verify the new types match the original. Glue is waived because
1318 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1319 assert((Res.getValueType() == Node->getValueType(0) ||
1320 Node->getValueType(0) == MVT::Glue) &&
1321 "Type mismatch for custom legalized operation");
1322 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1323 // We can just directly replace this node with the lowered value.
1324 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1325 return;
1326 }
1327
1328 SmallVector<SDValue, 8> ResultVals;
1329 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1330 // Verify the new types match the original. Glue is waived because
1331 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1332 assert((Res->getValueType(i) == Node->getValueType(i) ||
1333 Node->getValueType(i) == MVT::Glue) &&
1334 "Type mismatch for custom legalized operation");
1335 ResultVals.push_back(Elt: Res.getValue(R: i));
1336 }
1337 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1338 ReplaceNode(Old: Node, New: ResultVals.data());
1339 return;
1340 }
1341 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1342 [[fallthrough]];
1343 case TargetLowering::Expand:
1344 if (ExpandNode(Node))
1345 return;
1346 [[fallthrough]];
1347 case TargetLowering::LibCall:
1348 ConvertNodeToLibcall(Node);
1349 return;
1350 case TargetLowering::Promote:
1351 PromoteNode(Node);
1352 return;
1353 }
1354 }
1355
1356 switch (Node->getOpcode()) {
1357 default:
1358#ifndef NDEBUG
1359 dbgs() << "NODE: ";
1360 Node->dump( &DAG);
1361 dbgs() << "\n";
1362#endif
1363 llvm_unreachable("Do not know how to legalize this operator!");
1364
1365 case ISD::CALLSEQ_START:
1366 case ISD::CALLSEQ_END:
1367 break;
1368 case ISD::LOAD:
1369 return LegalizeLoadOps(Node);
1370 case ISD::STORE:
1371 return LegalizeStoreOps(Node);
1372 }
1373}
1374
1375SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1376 SDValue Vec = Op.getOperand(i: 0);
1377 SDValue Idx = Op.getOperand(i: 1);
1378 SDLoc dl(Op);
1379
1380 // Before we generate a new store to a temporary stack slot, see if there is
1381 // already one that we can use. There often is because when we scalarize
1382 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1383 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1384 // the vector. If all are expanded here, we don't want one store per vector
1385 // element.
1386
1387 // Caches for hasPredecessorHelper
1388 SmallPtrSet<const SDNode *, 32> Visited;
1389 SmallVector<const SDNode *, 16> Worklist;
1390 Visited.insert(Ptr: Op.getNode());
1391 Worklist.push_back(Elt: Idx.getNode());
1392 SDValue StackPtr, Ch;
1393 for (SDNode *User : Vec.getNode()->uses()) {
1394 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1395 if (ST->isIndexed() || ST->isTruncatingStore() ||
1396 ST->getValue() != Vec)
1397 continue;
1398
1399 // Make sure that nothing else could have stored into the destination of
1400 // this store.
1401 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1402 continue;
1403
1404 // If the index is dependent on the store we will introduce a cycle when
1405 // creating the load (the load uses the index, and by replacing the chain
1406 // we will make the index dependent on the load). Also, the store might be
1407 // dependent on the extractelement and introduce a cycle when creating
1408 // the load.
1409 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1410 ST->hasPredecessor(N: Op.getNode()))
1411 continue;
1412
1413 StackPtr = ST->getBasePtr();
1414 Ch = SDValue(ST, 0);
1415 break;
1416 }
1417 }
1418
1419 EVT VecVT = Vec.getValueType();
1420
1421 if (!Ch.getNode()) {
1422 // Store the value to a temporary stack slot, then LOAD the returned part.
1423 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1424 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1425 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1426 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1427 }
1428
1429 SDValue NewLoad;
1430 Align ElementAlignment =
1431 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1432 b: DAG.getDataLayout().getPrefTypeAlign(
1433 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1434
1435 if (Op.getValueType().isVector()) {
1436 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1437 SubVecVT: Op.getValueType(), Index: Idx);
1438 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1439 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1440 } else {
1441 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1442 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1443 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1444 Alignment: ElementAlignment);
1445 }
1446
1447 // Replace the chain going out of the store, by the one out of the load.
1448 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1449
1450 // We introduced a cycle though, so update the loads operands, making sure
1451 // to use the original store's chain as an incoming chain.
1452 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1453 NewLoad->op_end());
1454 NewLoadOperands[0] = Ch;
1455 NewLoad =
1456 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1457 return NewLoad;
1458}
1459
1460SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1461 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1462
1463 SDValue Vec = Op.getOperand(i: 0);
1464 SDValue Part = Op.getOperand(i: 1);
1465 SDValue Idx = Op.getOperand(i: 2);
1466 SDLoc dl(Op);
1467
1468 // Store the value to a temporary stack slot, then LOAD the returned part.
1469 EVT VecVT = Vec.getValueType();
1470 EVT PartVT = Part.getValueType();
1471 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1472 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1473 MachinePointerInfo PtrInfo =
1474 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1475
1476 // First store the whole vector.
1477 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo);
1478
1479 // Freeze the index so we don't poison the clamping code we're about to emit.
1480 Idx = DAG.getFreeze(V: Idx);
1481
1482 // Then store the inserted part.
1483 if (PartVT.isVector()) {
1484 SDValue SubStackPtr =
1485 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1486
1487 // Store the subvector.
1488 Ch = DAG.getStore(
1489 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1490 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
1491 } else {
1492 SDValue SubStackPtr =
1493 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1494
1495 // Store the scalar value.
1496 Ch = DAG.getTruncStore(
1497 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1498 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1499 SVT: VecVT.getVectorElementType());
1500 }
1501
1502 // Finally, load the updated vector.
1503 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo);
1504}
1505
1506SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1507 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1508 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1509 "Unexpected opcode!");
1510
1511 // We can't handle this case efficiently. Allocate a sufficiently
1512 // aligned object on the stack, store each operand into it, then load
1513 // the result as a vector.
1514 // Create the stack frame object.
1515 EVT VT = Node->getValueType(ResNo: 0);
1516 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1517 : Node->getOperand(Num: 0).getValueType();
1518 SDLoc dl(Node);
1519 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1520 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1521 MachinePointerInfo PtrInfo =
1522 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1523
1524 // Emit a store of each element to the stack slot.
1525 SmallVector<SDValue, 8> Stores;
1526 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1527 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1528
1529 // If the destination vector element type of a BUILD_VECTOR is narrower than
1530 // the source element type, only store the bits necessary.
1531 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1532 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1533
1534 // Store (in the right endianness) the elements to memory.
1535 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1536 // Ignore undef elements.
1537 if (Node->getOperand(Num: i).isUndef()) continue;
1538
1539 unsigned Offset = TypeByteSize*i;
1540
1541 SDValue Idx =
1542 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1543
1544 if (Truncate)
1545 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1546 Val: Node->getOperand(Num: i), Ptr: Idx,
1547 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1548 else
1549 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1550 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1551 }
1552
1553 SDValue StoreChain;
1554 if (!Stores.empty()) // Not all undef elements?
1555 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1556 else
1557 StoreChain = DAG.getEntryNode();
1558
1559 // Result is a load from the stack slot.
1560 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1561}
1562
1563/// Bitcast a floating-point value to an integer value. Only bitcast the part
1564/// containing the sign bit if the target has no integer value capable of
1565/// holding all bits of the floating-point value.
1566void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1567 const SDLoc &DL,
1568 SDValue Value) const {
1569 EVT FloatVT = Value.getValueType();
1570 unsigned NumBits = FloatVT.getScalarSizeInBits();
1571 State.FloatVT = FloatVT;
1572 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1573 // Convert to an integer of the same size.
1574 if (TLI.isTypeLegal(VT: IVT)) {
1575 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1576 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1577 State.SignBit = NumBits - 1;
1578 return;
1579 }
1580
1581 auto &DataLayout = DAG.getDataLayout();
1582 // Store the float to memory, then load the sign part out as an integer.
1583 MVT LoadTy = TLI.getRegisterType(VT: MVT::i8);
1584 // First create a temporary that is aligned for both the load and store.
1585 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1586 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1587 // Then store the float to it.
1588 State.FloatPtr = StackPtr;
1589 MachineFunction &MF = DAG.getMachineFunction();
1590 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1591 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1592 PtrInfo: State.FloatPointerInfo);
1593
1594 SDValue IntPtr;
1595 if (DataLayout.isBigEndian()) {
1596 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1597 // Load out a legal integer with the same sign bit as the float.
1598 IntPtr = StackPtr;
1599 State.IntPointerInfo = State.FloatPointerInfo;
1600 } else {
1601 // Advance the pointer so that the loaded byte will contain the sign bit.
1602 unsigned ByteOffset = (NumBits / 8) - 1;
1603 IntPtr =
1604 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1605 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1606 Offset: ByteOffset);
1607 }
1608
1609 State.IntPtr = IntPtr;
1610 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1611 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1612 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1613 State.SignBit = 7;
1614}
1615
1616/// Replace the integer value produced by getSignAsIntValue() with a new value
1617/// and cast the result back to a floating-point type.
1618SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1619 const SDLoc &DL,
1620 SDValue NewIntValue) const {
1621 if (!State.Chain)
1622 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1623
1624 // Override the part containing the sign bit in the value stored on the stack.
1625 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1626 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1627 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1628 PtrInfo: State.FloatPointerInfo);
1629}
1630
1631SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1632 SDLoc DL(Node);
1633 SDValue Mag = Node->getOperand(Num: 0);
1634 SDValue Sign = Node->getOperand(Num: 1);
1635
1636 // Get sign bit into an integer value.
1637 FloatSignAsInt SignAsInt;
1638 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1639
1640 EVT IntVT = SignAsInt.IntValue.getValueType();
1641 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1642 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1643 N2: SignMask);
1644
1645 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1646 EVT FloatVT = Mag.getValueType();
1647 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1648 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1649 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1650 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1651 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1652 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1653 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1654 }
1655
1656 // Transform Mag value to integer, and clear the sign bit.
1657 FloatSignAsInt MagAsInt;
1658 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1659 EVT MagVT = MagAsInt.IntValue.getValueType();
1660 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1661 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1662 N2: ClearSignMask);
1663
1664 // Get the signbit at the right position for MagAsInt.
1665 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1666 EVT ShiftVT = IntVT;
1667 if (SignBit.getScalarValueSizeInBits() <
1668 ClearedSign.getScalarValueSizeInBits()) {
1669 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1670 ShiftVT = MagVT;
1671 }
1672 if (ShiftAmount > 0) {
1673 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1674 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1675 } else if (ShiftAmount < 0) {
1676 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1677 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1678 }
1679 if (SignBit.getScalarValueSizeInBits() >
1680 ClearedSign.getScalarValueSizeInBits()) {
1681 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1682 }
1683
1684 SDNodeFlags Flags;
1685 Flags.setDisjoint(true);
1686
1687 // Store the part with the modified sign and convert back to float.
1688 SDValue CopiedSign =
1689 DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit, Flags);
1690
1691 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1692}
1693
1694SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1695 // Get the sign bit as an integer.
1696 SDLoc DL(Node);
1697 FloatSignAsInt SignAsInt;
1698 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1699 EVT IntVT = SignAsInt.IntValue.getValueType();
1700
1701 // Flip the sign.
1702 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1703 SDValue SignFlip =
1704 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1705
1706 // Convert back to float.
1707 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1708}
1709
1710SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1711 SDLoc DL(Node);
1712 SDValue Value = Node->getOperand(Num: 0);
1713
1714 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1715 EVT FloatVT = Value.getValueType();
1716 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1717 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1718 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1719 }
1720
1721 // Transform value to integer, clear the sign bit and transform back.
1722 FloatSignAsInt ValueAsInt;
1723 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1724 EVT IntVT = ValueAsInt.IntValue.getValueType();
1725 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1726 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1727 N2: ClearSignMask);
1728 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1729}
1730
1731void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1732 SmallVectorImpl<SDValue> &Results) {
1733 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1734 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1735 " not tell us which reg is the stack pointer!");
1736 SDLoc dl(Node);
1737 EVT VT = Node->getValueType(ResNo: 0);
1738 SDValue Tmp1 = SDValue(Node, 0);
1739 SDValue Tmp2 = SDValue(Node, 1);
1740 SDValue Tmp3 = Node->getOperand(Num: 2);
1741 SDValue Chain = Tmp1.getOperand(i: 0);
1742
1743 // Chain the dynamic stack allocation so that it doesn't modify the stack
1744 // pointer when other instructions are using the stack.
1745 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1746
1747 SDValue Size = Tmp2.getOperand(i: 1);
1748 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1749 Chain = SP.getValue(R: 1);
1750 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1751 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1752 unsigned Opc =
1753 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1754 ISD::ADD : ISD::SUB;
1755
1756 Align StackAlign = TFL->getStackAlign();
1757 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1758 if (Alignment > StackAlign)
1759 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1760 N2: DAG.getConstant(Val: -Alignment.value(), DL: dl, VT));
1761 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1762
1763 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1764
1765 Results.push_back(Elt: Tmp1);
1766 Results.push_back(Elt: Tmp2);
1767}
1768
1769/// Emit a store/load combination to the stack. This stores
1770/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1771/// a load from the stack slot to DestVT, extending it if needed.
1772/// The resultant code need not be legal.
1773SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1774 EVT DestVT, const SDLoc &dl) {
1775 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1776}
1777
1778SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1779 EVT DestVT, const SDLoc &dl,
1780 SDValue Chain) {
1781 EVT SrcVT = SrcOp.getValueType();
1782 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1783 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1784
1785 // Don't convert with stack if the load/store is expensive.
1786 if ((SrcVT.bitsGT(VT: SlotVT) &&
1787 !TLI.isTruncStoreLegalOrCustom(ValVT: SrcOp.getValueType(), MemVT: SlotVT)) ||
1788 (SlotVT.bitsLT(VT: DestVT) &&
1789 !TLI.isLoadExtLegalOrCustom(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SlotVT)))
1790 return SDValue();
1791
1792 // Create the stack frame object.
1793 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1794 Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1795 SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1796
1797 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1798 int SPFI = StackPtrFI->getIndex();
1799 MachinePointerInfo PtrInfo =
1800 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1801
1802 // Emit a store to the stack slot. Use a truncstore if the input value is
1803 // later than DestVT.
1804 SDValue Store;
1805
1806 if (SrcVT.bitsGT(VT: SlotVT))
1807 Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1808 SVT: SlotVT, Alignment: SrcAlign);
1809 else {
1810 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1811 Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1812 }
1813
1814 // Result is a load from the stack slot.
1815 if (SlotVT.bitsEq(VT: DestVT))
1816 return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1817
1818 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1819 return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1820 Alignment: DestAlign);
1821}
1822
1823SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1824 SDLoc dl(Node);
1825 // Create a vector sized/aligned stack slot, store the value to element #0,
1826 // then load the whole vector back out.
1827 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1828
1829 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1830 int SPFI = StackPtrFI->getIndex();
1831
1832 SDValue Ch = DAG.getTruncStore(
1833 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1834 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1835 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1836 return DAG.getLoad(
1837 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1838 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1839}
1840
1841static bool
1842ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1843 const TargetLowering &TLI, SDValue &Res) {
1844 unsigned NumElems = Node->getNumOperands();
1845 SDLoc dl(Node);
1846 EVT VT = Node->getValueType(ResNo: 0);
1847
1848 // Try to group the scalars into pairs, shuffle the pairs together, then
1849 // shuffle the pairs of pairs together, etc. until the vector has
1850 // been built. This will work only if all of the necessary shuffle masks
1851 // are legal.
1852
1853 // We do this in two phases; first to check the legality of the shuffles,
1854 // and next, assuming that all shuffles are legal, to create the new nodes.
1855 for (int Phase = 0; Phase < 2; ++Phase) {
1856 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1857 NewIntermedVals;
1858 for (unsigned i = 0; i < NumElems; ++i) {
1859 SDValue V = Node->getOperand(Num: i);
1860 if (V.isUndef())
1861 continue;
1862
1863 SDValue Vec;
1864 if (Phase)
1865 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1866 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1867 }
1868
1869 while (IntermedVals.size() > 2) {
1870 NewIntermedVals.clear();
1871 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1872 // This vector and the next vector are shuffled together (simply to
1873 // append the one to the other).
1874 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1875
1876 SmallVector<int, 16> FinalIndices;
1877 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1878 IntermedVals[i+1].second.size());
1879
1880 int k = 0;
1881 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1882 ++j, ++k) {
1883 ShuffleVec[k] = j;
1884 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1885 }
1886 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1887 ++j, ++k) {
1888 ShuffleVec[k] = NumElems + j;
1889 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1890 }
1891
1892 SDValue Shuffle;
1893 if (Phase)
1894 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1895 N2: IntermedVals[i+1].first,
1896 Mask: ShuffleVec);
1897 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1898 return false;
1899 NewIntermedVals.push_back(
1900 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1901 }
1902
1903 // If we had an odd number of defined values, then append the last
1904 // element to the array of new vectors.
1905 if ((IntermedVals.size() & 1) != 0)
1906 NewIntermedVals.push_back(Elt: IntermedVals.back());
1907
1908 IntermedVals.swap(RHS&: NewIntermedVals);
1909 }
1910
1911 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1912 "Invalid number of intermediate vectors");
1913 SDValue Vec1 = IntermedVals[0].first;
1914 SDValue Vec2;
1915 if (IntermedVals.size() > 1)
1916 Vec2 = IntermedVals[1].first;
1917 else if (Phase)
1918 Vec2 = DAG.getUNDEF(VT);
1919
1920 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1921 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1922 ShuffleVec[IntermedVals[0].second[i]] = i;
1923 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1924 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1925
1926 if (Phase)
1927 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1928 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1929 return false;
1930 }
1931
1932 return true;
1933}
1934
1935/// Expand a BUILD_VECTOR node on targets that don't
1936/// support the operation, but do support the resultant vector type.
1937SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1938 unsigned NumElems = Node->getNumOperands();
1939 SDValue Value1, Value2;
1940 SDLoc dl(Node);
1941 EVT VT = Node->getValueType(ResNo: 0);
1942 EVT OpVT = Node->getOperand(Num: 0).getValueType();
1943 EVT EltVT = VT.getVectorElementType();
1944
1945 // If the only non-undef value is the low element, turn this into a
1946 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1947 bool isOnlyLowElement = true;
1948 bool MoreThanTwoValues = false;
1949 bool isConstant = true;
1950 for (unsigned i = 0; i < NumElems; ++i) {
1951 SDValue V = Node->getOperand(Num: i);
1952 if (V.isUndef())
1953 continue;
1954 if (i > 0)
1955 isOnlyLowElement = false;
1956 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
1957 isConstant = false;
1958
1959 if (!Value1.getNode()) {
1960 Value1 = V;
1961 } else if (!Value2.getNode()) {
1962 if (V != Value1)
1963 Value2 = V;
1964 } else if (V != Value1 && V != Value2) {
1965 MoreThanTwoValues = true;
1966 }
1967 }
1968
1969 if (!Value1.getNode())
1970 return DAG.getUNDEF(VT);
1971
1972 if (isOnlyLowElement)
1973 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
1974
1975 // If all elements are constants, create a load from the constant pool.
1976 if (isConstant) {
1977 SmallVector<Constant*, 16> CV;
1978 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1979 if (ConstantFPSDNode *V =
1980 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
1981 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
1982 } else if (ConstantSDNode *V =
1983 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
1984 if (OpVT==EltVT)
1985 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
1986 else {
1987 // If OpVT and EltVT don't match, EltVT is not legal and the
1988 // element values have been promoted/truncated earlier. Undo this;
1989 // we don't want a v16i8 to become a v16i32 for example.
1990 const ConstantInt *CI = V->getConstantIntValue();
1991 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
1992 V: CI->getZExtValue()));
1993 }
1994 } else {
1995 assert(Node->getOperand(i).isUndef());
1996 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
1997 CV.push_back(Elt: UndefValue::get(T: OpNTy));
1998 }
1999 }
2000 Constant *CP = ConstantVector::get(V: CV);
2001 SDValue CPIdx =
2002 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2003 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2004 return DAG.getLoad(
2005 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2006 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2007 Alignment);
2008 }
2009
2010 SmallSet<SDValue, 16> DefinedValues;
2011 for (unsigned i = 0; i < NumElems; ++i) {
2012 if (Node->getOperand(Num: i).isUndef())
2013 continue;
2014 DefinedValues.insert(V: Node->getOperand(Num: i));
2015 }
2016
2017 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2018 if (!MoreThanTwoValues) {
2019 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2020 for (unsigned i = 0; i < NumElems; ++i) {
2021 SDValue V = Node->getOperand(Num: i);
2022 if (V.isUndef())
2023 continue;
2024 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2025 }
2026 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2027 // Get the splatted value into the low element of a vector register.
2028 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2029 SDValue Vec2;
2030 if (Value2.getNode())
2031 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2032 else
2033 Vec2 = DAG.getUNDEF(VT);
2034
2035 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2036 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2037 }
2038 } else {
2039 SDValue Res;
2040 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2041 return Res;
2042 }
2043 }
2044
2045 // Otherwise, we can't handle this case efficiently.
2046 return ExpandVectorBuildThroughStack(Node);
2047}
2048
2049SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2050 SDLoc DL(Node);
2051 EVT VT = Node->getValueType(ResNo: 0);
2052 SDValue SplatVal = Node->getOperand(Num: 0);
2053
2054 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2055}
2056
2057// Expand a node into a call to a libcall, returning the value as the first
2058// result and the chain as the second. If the result value does not fit into a
2059// register, return the lo part and set the hi part to the by-reg argument in
2060// the first. If it does fit into a single register, return the result and
2061// leave the Hi part unset.
2062std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2063 TargetLowering::ArgListTy &&Args,
2064 bool isSigned) {
2065 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2066 SDValue Callee;
2067 if (const char *LibcallName = TLI.getLibcallName(Call: LC))
2068 Callee = DAG.getExternalSymbol(Sym: LibcallName, VT: CodePtrTy);
2069 else {
2070 Callee = DAG.getUNDEF(VT: CodePtrTy);
2071 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2072 Node->getOperationName(G: &DAG));
2073 }
2074
2075 EVT RetVT = Node->getValueType(ResNo: 0);
2076 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2077
2078 // By default, the input chain to this libcall is the entry node of the
2079 // function. If the libcall is going to be emitted as a tail call then
2080 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2081 // node which is being folded has a non-entry input chain.
2082 SDValue InChain = DAG.getEntryNode();
2083
2084 // isTailCall may be true since the callee does not reference caller stack
2085 // frame. Check if it's in the right position and that the return types match.
2086 SDValue TCChain = InChain;
2087 const Function &F = DAG.getMachineFunction().getFunction();
2088 bool isTailCall =
2089 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2090 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2091 if (isTailCall)
2092 InChain = TCChain;
2093
2094 TargetLowering::CallLoweringInfo CLI(DAG);
2095 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Type: RetVT, IsSigned: isSigned);
2096 CLI.setDebugLoc(SDLoc(Node))
2097 .setChain(InChain)
2098 .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee,
2099 ArgsList: std::move(Args))
2100 .setTailCall(isTailCall)
2101 .setSExtResult(signExtend)
2102 .setZExtResult(!signExtend)
2103 .setIsPostTypeLegalization(true);
2104
2105 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2106
2107 if (!CallInfo.second.getNode()) {
2108 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2109 // It's a tailcall, return the chain (which is the DAG root).
2110 return {DAG.getRoot(), DAG.getRoot()};
2111 }
2112
2113 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2114 return CallInfo;
2115}
2116
2117std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2118 bool isSigned) {
2119 TargetLowering::ArgListTy Args;
2120 TargetLowering::ArgListEntry Entry;
2121 for (const SDValue &Op : Node->op_values()) {
2122 EVT ArgVT = Op.getValueType();
2123 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2124 Entry.Node = Op;
2125 Entry.Ty = ArgTy;
2126 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Type: ArgVT, IsSigned: isSigned);
2127 Entry.IsZExt = !Entry.IsSExt;
2128 Args.push_back(x: Entry);
2129 }
2130
2131 return ExpandLibCall(LC, Node, Args: std::move(Args), isSigned);
2132}
2133
2134void SelectionDAGLegalize::ExpandFrexpLibCall(
2135 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
2136 SDLoc dl(Node);
2137 EVT VT = Node->getValueType(ResNo: 0);
2138 EVT ExpVT = Node->getValueType(ResNo: 1);
2139
2140 SDValue FPOp = Node->getOperand(Num: 0);
2141
2142 EVT ArgVT = FPOp.getValueType();
2143 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2144
2145 TargetLowering::ArgListEntry FPArgEntry;
2146 FPArgEntry.Node = FPOp;
2147 FPArgEntry.Ty = ArgTy;
2148
2149 SDValue StackSlot = DAG.CreateStackTemporary(VT: ExpVT);
2150 TargetLowering::ArgListEntry PtrArgEntry;
2151 PtrArgEntry.Node = StackSlot;
2152 PtrArgEntry.Ty = PointerType::get(C&: *DAG.getContext(),
2153 AddressSpace: DAG.getDataLayout().getAllocaAddrSpace());
2154
2155 TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2156
2157 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT);
2158 auto [Call, Chain] = ExpandLibCall(LC, Node, Args: std::move(Args), isSigned: false);
2159
2160 // FIXME: Get type of int for libcall declaration and cast
2161
2162 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
2163 auto PtrInfo =
2164 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2165
2166 SDValue LoadExp = DAG.getLoad(VT: ExpVT, dl, Chain, Ptr: StackSlot, PtrInfo);
2167 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2168 N1: LoadExp.getValue(R: 1), N2: DAG.getRoot());
2169 DAG.setRoot(OutputChain);
2170
2171 Results.push_back(Elt: Call);
2172 Results.push_back(Elt: LoadExp);
2173}
2174
2175void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2176 RTLIB::Libcall LC,
2177 SmallVectorImpl<SDValue> &Results) {
2178 if (LC == RTLIB::UNKNOWN_LIBCALL)
2179 llvm_unreachable("Can't create an unknown libcall!");
2180
2181 if (Node->isStrictFPOpcode()) {
2182 EVT RetVT = Node->getValueType(ResNo: 0);
2183 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2184 TargetLowering::MakeLibCallOptions CallOptions;
2185 // FIXME: This doesn't support tail calls.
2186 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2187 Ops, CallOptions,
2188 dl: SDLoc(Node),
2189 Chain: Node->getOperand(Num: 0));
2190 Results.push_back(Elt: Tmp.first);
2191 Results.push_back(Elt: Tmp.second);
2192 } else {
2193 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2194 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2195 Results.push_back(Elt: Tmp);
2196 }
2197}
2198
2199/// Expand the node to a libcall based on the result type.
2200void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2201 RTLIB::Libcall Call_F32,
2202 RTLIB::Libcall Call_F64,
2203 RTLIB::Libcall Call_F80,
2204 RTLIB::Libcall Call_F128,
2205 RTLIB::Libcall Call_PPCF128,
2206 SmallVectorImpl<SDValue> &Results) {
2207 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0),
2208 Call_F32, Call_F64, Call_F80,
2209 Call_F128, Call_PPCF128);
2210 ExpandFPLibCall(Node, LC, Results);
2211}
2212
2213SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2214 RTLIB::Libcall Call_I8,
2215 RTLIB::Libcall Call_I16,
2216 RTLIB::Libcall Call_I32,
2217 RTLIB::Libcall Call_I64,
2218 RTLIB::Libcall Call_I128) {
2219 RTLIB::Libcall LC;
2220 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2221 default: llvm_unreachable("Unexpected request for libcall!");
2222 case MVT::i8: LC = Call_I8; break;
2223 case MVT::i16: LC = Call_I16; break;
2224 case MVT::i32: LC = Call_I32; break;
2225 case MVT::i64: LC = Call_I64; break;
2226 case MVT::i128: LC = Call_I128; break;
2227 }
2228 return ExpandLibCall(LC, Node, isSigned).first;
2229}
2230
2231/// Expand the node to a libcall based on first argument type (for instance
2232/// lround and its variant).
2233void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2234 RTLIB::Libcall Call_F32,
2235 RTLIB::Libcall Call_F64,
2236 RTLIB::Libcall Call_F80,
2237 RTLIB::Libcall Call_F128,
2238 RTLIB::Libcall Call_PPCF128,
2239 SmallVectorImpl<SDValue> &Results) {
2240 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2241 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2242 Call_F32, Call_F64, Call_F80,
2243 Call_F128, Call_PPCF128);
2244 ExpandFPLibCall(Node, LC, Results);
2245}
2246
2247/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2248void
2249SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2250 SmallVectorImpl<SDValue> &Results) {
2251 unsigned Opcode = Node->getOpcode();
2252 bool isSigned = Opcode == ISD::SDIVREM;
2253
2254 RTLIB::Libcall LC;
2255 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2256 default: llvm_unreachable("Unexpected request for libcall!");
2257 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2258 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2259 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2260 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2261 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2262 }
2263
2264 // The input chain to this libcall is the entry node of the function.
2265 // Legalizing the call will automatically add the previous call to the
2266 // dependence.
2267 SDValue InChain = DAG.getEntryNode();
2268
2269 EVT RetVT = Node->getValueType(ResNo: 0);
2270 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2271
2272 TargetLowering::ArgListTy Args;
2273 TargetLowering::ArgListEntry Entry;
2274 for (const SDValue &Op : Node->op_values()) {
2275 EVT ArgVT = Op.getValueType();
2276 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2277 Entry.Node = Op;
2278 Entry.Ty = ArgTy;
2279 Entry.IsSExt = isSigned;
2280 Entry.IsZExt = !isSigned;
2281 Args.push_back(x: Entry);
2282 }
2283
2284 // Also pass the return address of the remainder.
2285 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2286 Entry.Node = FIPtr;
2287 Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2288 Entry.IsSExt = isSigned;
2289 Entry.IsZExt = !isSigned;
2290 Args.push_back(x: Entry);
2291
2292 SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2293 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2294
2295 SDLoc dl(Node);
2296 TargetLowering::CallLoweringInfo CLI(DAG);
2297 CLI.setDebugLoc(dl)
2298 .setChain(InChain)
2299 .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee,
2300 ArgsList: std::move(Args))
2301 .setSExtResult(isSigned)
2302 .setZExtResult(!isSigned);
2303
2304 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2305
2306 // Remainder is loaded back from the stack frame.
2307 SDValue Rem =
2308 DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo: MachinePointerInfo());
2309 Results.push_back(Elt: CallInfo.first);
2310 Results.push_back(Elt: Rem);
2311}
2312
2313/// Return true if sincos libcall is available.
2314static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2315 RTLIB::Libcall LC;
2316 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2317 default: llvm_unreachable("Unexpected request for libcall!");
2318 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2319 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2320 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2321 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2322 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2323 }
2324 return TLI.getLibcallName(Call: LC) != nullptr;
2325}
2326
2327/// Only issue sincos libcall if both sin and cos are needed.
2328static bool useSinCos(SDNode *Node) {
2329 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2330 ? ISD::FCOS : ISD::FSIN;
2331
2332 SDValue Op0 = Node->getOperand(Num: 0);
2333 for (const SDNode *User : Op0.getNode()->uses()) {
2334 if (User == Node)
2335 continue;
2336 // The other user might have been turned into sincos already.
2337 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2338 return true;
2339 }
2340 return false;
2341}
2342
2343/// Issue libcalls to sincos to compute sin / cos pairs.
2344void
2345SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2346 SmallVectorImpl<SDValue> &Results) {
2347 RTLIB::Libcall LC;
2348 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2349 default: llvm_unreachable("Unexpected request for libcall!");
2350 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2351 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2352 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2353 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2354 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2355 }
2356
2357 // The input chain to this libcall is the entry node of the function.
2358 // Legalizing the call will automatically add the previous call to the
2359 // dependence.
2360 SDValue InChain = DAG.getEntryNode();
2361
2362 EVT RetVT = Node->getValueType(ResNo: 0);
2363 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2364
2365 TargetLowering::ArgListTy Args;
2366 TargetLowering::ArgListEntry Entry;
2367
2368 // Pass the argument.
2369 Entry.Node = Node->getOperand(Num: 0);
2370 Entry.Ty = RetTy;
2371 Entry.IsSExt = false;
2372 Entry.IsZExt = false;
2373 Args.push_back(x: Entry);
2374
2375 // Pass the return address of sin.
2376 SDValue SinPtr = DAG.CreateStackTemporary(VT: RetVT);
2377 Entry.Node = SinPtr;
2378 Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2379 Entry.IsSExt = false;
2380 Entry.IsZExt = false;
2381 Args.push_back(x: Entry);
2382
2383 // Also pass the return address of the cos.
2384 SDValue CosPtr = DAG.CreateStackTemporary(VT: RetVT);
2385 Entry.Node = CosPtr;
2386 Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2387 Entry.IsSExt = false;
2388 Entry.IsZExt = false;
2389 Args.push_back(x: Entry);
2390
2391 SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2392 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2393
2394 SDLoc dl(Node);
2395 TargetLowering::CallLoweringInfo CLI(DAG);
2396 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2397 CC: TLI.getLibcallCallingConv(Call: LC), ResultType: Type::getVoidTy(C&: *DAG.getContext()), Target: Callee,
2398 ArgsList: std::move(Args));
2399
2400 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2401
2402 Results.push_back(
2403 Elt: DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: SinPtr, PtrInfo: MachinePointerInfo()));
2404 Results.push_back(
2405 Elt: DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: CosPtr, PtrInfo: MachinePointerInfo()));
2406}
2407
2408SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2409 SDLoc dl(Node);
2410 EVT VT = Node->getValueType(ResNo: 0);
2411 SDValue X = Node->getOperand(Num: 0);
2412 SDValue N = Node->getOperand(Num: 1);
2413 EVT ExpVT = N.getValueType();
2414 EVT AsIntVT = VT.changeTypeToInteger();
2415 if (AsIntVT == EVT()) // TODO: How to handle f80?
2416 return SDValue();
2417
2418 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2419 return SDValue();
2420
2421 SDNodeFlags NSW;
2422 NSW.setNoSignedWrap(true);
2423 SDNodeFlags NUW_NSW;
2424 NUW_NSW.setNoUnsignedWrap(true);
2425 NUW_NSW.setNoSignedWrap(true);
2426
2427 EVT SetCCVT =
2428 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2429 const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2430
2431 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2432 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2433 const int Precision = APFloat::semanticsPrecision(FltSem);
2434
2435 const SDValue MaxExp = DAG.getConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2436 const SDValue MinExp = DAG.getConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2437
2438 const SDValue DoubleMaxExp = DAG.getConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2439
2440 const APFloat One(FltSem, "1.0");
2441 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2442
2443 // Offset by precision to avoid denormal range.
2444 APFloat ScaleDownK =
2445 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2446
2447 // TODO: Should really introduce control flow and use a block for the >
2448 // MaxExp, < MinExp cases
2449
2450 // First, handle exponents Exp > MaxExp and scale down.
2451 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2452
2453 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2454 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2455 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2456 SDValue DecN1 =
2457 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2458
2459 SDValue ScaleUpTwice =
2460 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2461
2462 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2463 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2464 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2465
2466 SDValue SelectN_Big =
2467 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2468 SDValue SelectX_Big =
2469 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2470
2471 // Now handle exponents Exp < MinExp
2472 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2473
2474 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2475 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2476
2477 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2478
2479 SDValue ClampMinVal =
2480 DAG.getConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2481 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2482 SDValue IncN1 =
2483 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2484
2485 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2486 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2487 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2488
2489 SDValue ScaleDownTwice = DAG.getSetCC(
2490 DL: dl, VT: SetCCVT, LHS: N, RHS: DAG.getConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT),
2491 Cond: ISD::SETULT);
2492
2493 SDValue SelectN_Small =
2494 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2495 SDValue SelectX_Small =
2496 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2497
2498 // Now combine the two out of range exponent handling cases with the base
2499 // case.
2500 SDValue NewX = DAG.getNode(
2501 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2502 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2503
2504 SDValue NewN = DAG.getNode(
2505 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2506 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2507
2508 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2509
2510 SDValue ExponentShiftAmt =
2511 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2512 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2513
2514 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2515 N2: ExponentShiftAmt, Flags: NUW_NSW);
2516 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2517 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2518}
2519
2520SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2521 SDLoc dl(Node);
2522 SDValue Val = Node->getOperand(Num: 0);
2523 EVT VT = Val.getValueType();
2524 EVT ExpVT = Node->getValueType(ResNo: 1);
2525 EVT AsIntVT = VT.changeTypeToInteger();
2526 if (AsIntVT == EVT()) // TODO: How to handle f80?
2527 return SDValue();
2528
2529 const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2530 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2531 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2532 const unsigned BitSize = VT.getScalarSizeInBits();
2533
2534 // TODO: Could introduce control flow and skip over the denormal handling.
2535
2536 // scale_up = fmul value, scalbn(1.0, precision + 1)
2537 // extracted_exp = (bitcast value to uint) >> precision - 1
2538 // biased_exp = extracted_exp + min_exp
2539 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2540 //
2541 // is_denormal = val < smallest_normalized
2542 // computed_fract = is_denormal ? scale_up : extracted_fract
2543 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2544 //
2545 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2546 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2547
2548 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2549 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2550 VT: AsIntVT);
2551
2552 SDValue SmallestNormalizedInt = DAG.getConstant(
2553 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2554 VT: AsIntVT);
2555
2556 // Masks out the exponent bits.
2557 SDValue ExpMask =
2558 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2559
2560 // Mask out the exponent part of the value.
2561 //
2562 // e.g, for f32 FractSignMaskVal = 0x807fffff
2563 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2564 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2565
2566 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2567 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2568
2569 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2570
2571 const APFloat One(FltSem, "1.0");
2572 // Scale a possible denormal input.
2573 // e.g., for f64, 0x1p+54
2574 APFloat ScaleUpKVal =
2575 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2576
2577 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2578 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2579
2580 EVT SetCCVT =
2581 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2582
2583 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2584
2585 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2586
2587 SDValue AddNegSmallestNormal =
2588 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2589 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2590 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2591
2592 SDValue IsDenormal =
2593 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2594
2595 SDValue MinExp = DAG.getConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2596 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2597
2598 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2599 SDValue ScaledSelect =
2600 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2601
2602 SDValue ExpMaskScaled =
2603 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2604
2605 SDValue ScaledValue =
2606 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2607
2608 // Extract the exponent bits.
2609 SDValue ExponentShiftAmt =
2610 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2611 SDValue ShiftedExp =
2612 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2613 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2614
2615 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2616 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2617 SDValue DenormalExpBias =
2618 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2619
2620 SDValue MaskedFractAsInt =
2621 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2622 const APFloat Half(FltSem, "0.5");
2623 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2624 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2625 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2626
2627 SDValue ComputedExp =
2628 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2629
2630 SDValue Result0 =
2631 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2632
2633 SDValue Result1 =
2634 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2635
2636 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2637}
2638
2639/// This function is responsible for legalizing a
2640/// INT_TO_FP operation of the specified operand when the target requests that
2641/// we expand it. At this point, we know that the result and operand types are
2642/// legal for the target.
2643SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2644 SDValue &Chain) {
2645 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2646 Node->getOpcode() == ISD::SINT_TO_FP);
2647 EVT DestVT = Node->getValueType(ResNo: 0);
2648 SDLoc dl(Node);
2649 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2650 SDValue Op0 = Node->getOperand(Num: OpNo);
2651 EVT SrcVT = Op0.getValueType();
2652
2653 // TODO: Should any fast-math-flags be set for the created nodes?
2654 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2655 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2656 (DestVT.bitsLE(VT: MVT::f64) ||
2657 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2658 : ISD::FP_EXTEND,
2659 VT: DestVT))) {
2660 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2661 "expansion\n");
2662
2663 // Get the stack frame index of a 8 byte buffer.
2664 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2665
2666 SDValue Lo = Op0;
2667 // if signed map to unsigned space
2668 if (isSigned) {
2669 // Invert sign bit (signed to unsigned mapping).
2670 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2671 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2672 }
2673 // Initial hi portion of constructed double.
2674 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2675
2676 // If this a big endian target, swap the lo and high data.
2677 if (DAG.getDataLayout().isBigEndian())
2678 std::swap(a&: Lo, b&: Hi);
2679
2680 SDValue MemChain = DAG.getEntryNode();
2681
2682 // Store the lo of the constructed double.
2683 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2684 PtrInfo: MachinePointerInfo());
2685 // Store the hi of the constructed double.
2686 SDValue HiPtr =
2687 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2688 SDValue Store2 =
2689 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2690 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2691
2692 // load the constructed double
2693 SDValue Load =
2694 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2695 // FP constant to bias correct the final result
2696 SDValue Bias = DAG.getConstantFP(
2697 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2698 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2699 DL: dl, VT: MVT::f64);
2700 // Subtract the bias and get the final result.
2701 SDValue Sub;
2702 SDValue Result;
2703 if (Node->isStrictFPOpcode()) {
2704 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2705 Ops: {Node->getOperand(Num: 0), Load, Bias});
2706 Chain = Sub.getValue(R: 1);
2707 if (DestVT != Sub.getValueType()) {
2708 std::pair<SDValue, SDValue> ResultPair;
2709 ResultPair =
2710 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2711 Result = ResultPair.first;
2712 Chain = ResultPair.second;
2713 }
2714 else
2715 Result = Sub;
2716 } else {
2717 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2718 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2719 }
2720 return Result;
2721 }
2722
2723 if (isSigned)
2724 return SDValue();
2725
2726 // TODO: Generalize this for use with other types.
2727 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2728 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2729 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2730 // For unsigned conversions, convert them to signed conversions using the
2731 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2732 // should be valid for i32->f32 as well.
2733
2734 // More generally this transform should be valid if there are 3 more bits
2735 // in the integer type than the significand. Rounding uses the first bit
2736 // after the width of the significand and the OR of all bits after that. So
2737 // we need to be able to OR the shifted out bit into one of the bits that
2738 // participate in the OR.
2739
2740 // TODO: This really should be implemented using a branch rather than a
2741 // select. We happen to get lucky and machinesink does the right
2742 // thing most of the time. This would be a good candidate for a
2743 // pseudo-op, or, even better, for whole-function isel.
2744 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2745
2746 SDValue SignBitTest = DAG.getSetCC(
2747 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2748
2749 EVT ShiftVT = TLI.getShiftAmountTy(LHSTy: SrcVT, DL: DAG.getDataLayout());
2750 SDValue ShiftConst = DAG.getConstant(Val: 1, DL: dl, VT: ShiftVT);
2751 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2752 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2753 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2754 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2755
2756 SDValue Slow, Fast;
2757 if (Node->isStrictFPOpcode()) {
2758 // In strict mode, we must avoid spurious exceptions, and therefore
2759 // must make sure to only emit a single STRICT_SINT_TO_FP.
2760 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2761 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2762 Ops: { Node->getOperand(Num: 0), InCvt });
2763 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
2764 Ops: { Fast.getValue(R: 1), Fast, Fast });
2765 Chain = Slow.getValue(R: 1);
2766 // The STRICT_SINT_TO_FP inherits the exception mode from the
2767 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2768 // never raise any exception.
2769 SDNodeFlags Flags;
2770 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2771 Fast->setFlags(Flags);
2772 Flags.setNoFPExcept(true);
2773 Slow->setFlags(Flags);
2774 } else {
2775 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2776 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2777 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2778 }
2779
2780 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2781 }
2782
2783 // Don't expand it if there isn't cheap fadd.
2784 if (!TLI.isOperationLegalOrCustom(
2785 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2786 return SDValue();
2787
2788 // The following optimization is valid only if every value in SrcVT (when
2789 // treated as signed) is representable in DestVT. Check that the mantissa
2790 // size of DestVT is >= than the number of bits in SrcVT -1.
2791 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2792 SrcVT.getSizeInBits() - 1 &&
2793 "Cannot perform lossless SINT_TO_FP!");
2794
2795 SDValue Tmp1;
2796 if (Node->isStrictFPOpcode()) {
2797 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2798 Ops: { Node->getOperand(Num: 0), Op0 });
2799 } else
2800 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2801
2802 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2803 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2804 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2805 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2806 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2807 Cond: SignSet, LHS: Four, RHS: Zero);
2808
2809 // If the sign bit of the integer is set, the large number will be treated
2810 // as a negative number. To counteract this, the dynamic code adds an
2811 // offset depending on the data type.
2812 uint64_t FF;
2813 switch (SrcVT.getSimpleVT().SimpleTy) {
2814 default:
2815 return SDValue();
2816 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2817 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2818 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2819 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2820 }
2821 if (DAG.getDataLayout().isLittleEndian())
2822 FF <<= 32;
2823 Constant *FudgeFactor = ConstantInt::get(
2824 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2825
2826 SDValue CPIdx =
2827 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2828 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2829 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2830 Alignment = commonAlignment(A: Alignment, Offset: 4);
2831 SDValue FudgeInReg;
2832 if (DestVT == MVT::f32)
2833 FudgeInReg = DAG.getLoad(
2834 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2835 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2836 Alignment);
2837 else {
2838 SDValue Load = DAG.getExtLoad(
2839 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2840 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
2841 Alignment);
2842 HandleSDNode Handle(Load);
2843 LegalizeOp(Node: Load.getNode());
2844 FudgeInReg = Handle.getValue();
2845 }
2846
2847 if (Node->isStrictFPOpcode()) {
2848 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
2849 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
2850 Chain = Result.getValue(R: 1);
2851 return Result;
2852 }
2853
2854 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
2855}
2856
2857/// This function is responsible for legalizing a
2858/// *INT_TO_FP operation of the specified operand when the target requests that
2859/// we promote it. At this point, we know that the result and operand types are
2860/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2861/// operation that takes a larger input.
2862void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2863 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2864 bool IsStrict = N->isStrictFPOpcode();
2865 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2866 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2867 EVT DestVT = N->getValueType(ResNo: 0);
2868 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
2869 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2870 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2871
2872 // First step, figure out the appropriate *INT_TO_FP operation to use.
2873 EVT NewInTy = LegalOp.getValueType();
2874
2875 unsigned OpToUse = 0;
2876
2877 // Scan for the appropriate larger type to use.
2878 while (true) {
2879 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2880 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2881
2882 // If the target supports SINT_TO_FP of this type, use it.
2883 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
2884 OpToUse = SIntOp;
2885 break;
2886 }
2887 if (IsSigned)
2888 continue;
2889
2890 // If the target supports UINT_TO_FP of this type, use it.
2891 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
2892 OpToUse = UIntOp;
2893 break;
2894 }
2895
2896 // Otherwise, try a larger type.
2897 }
2898
2899 // Okay, we found the operation and type to use. Zero extend our input to the
2900 // desired type then run the operation on it.
2901 if (IsStrict) {
2902 SDValue Res =
2903 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
2904 Ops: {N->getOperand(Num: 0),
2905 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2906 DL: dl, VT: NewInTy, Operand: LegalOp)});
2907 Results.push_back(Elt: Res);
2908 Results.push_back(Elt: Res.getValue(R: 1));
2909 return;
2910 }
2911
2912 Results.push_back(
2913 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
2914 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2915 DL: dl, VT: NewInTy, Operand: LegalOp)));
2916}
2917
2918/// This function is responsible for legalizing a
2919/// FP_TO_*INT operation of the specified operand when the target requests that
2920/// we promote it. At this point, we know that the result and operand types are
2921/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2922/// operation that returns a larger result.
2923void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2924 SmallVectorImpl<SDValue> &Results) {
2925 bool IsStrict = N->isStrictFPOpcode();
2926 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2927 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2928 EVT DestVT = N->getValueType(ResNo: 0);
2929 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
2930 // First step, figure out the appropriate FP_TO*INT operation to use.
2931 EVT NewOutTy = DestVT;
2932
2933 unsigned OpToUse = 0;
2934
2935 // Scan for the appropriate larger type to use.
2936 while (true) {
2937 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2938 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2939
2940 // A larger signed type can hold all unsigned values of the requested type,
2941 // so using FP_TO_SINT is valid
2942 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2943 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
2944 break;
2945
2946 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2947 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2948 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
2949 break;
2950
2951 // Otherwise, try a larger type.
2952 }
2953
2954 // Okay, we found the operation and type to use.
2955 SDValue Operation;
2956 if (IsStrict) {
2957 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
2958 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
2959 } else
2960 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
2961
2962 // Truncate the result of the extended FP_TO_*INT operation to the desired
2963 // size.
2964 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
2965 Results.push_back(Elt: Trunc);
2966 if (IsStrict)
2967 Results.push_back(Elt: Operation.getValue(R: 1));
2968}
2969
2970/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2971/// the result and operand types are legal and there must be a legal
2972/// FP_TO_*INT_SAT operation for a larger result type.
2973SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2974 const SDLoc &dl) {
2975 unsigned Opcode = Node->getOpcode();
2976
2977 // Scan for the appropriate larger type to use.
2978 EVT NewOutTy = Node->getValueType(ResNo: 0);
2979 while (true) {
2980 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2981 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2982
2983 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
2984 break;
2985 }
2986
2987 // Saturation width is determined by second operand, so we don't have to
2988 // perform any fixup and can directly truncate the result.
2989 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
2990 N2: Node->getOperand(Num: 1));
2991 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
2992}
2993
2994/// Open code the operations for PARITY of the specified operation.
2995SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2996 EVT VT = Op.getValueType();
2997 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
2998 unsigned Sz = VT.getScalarSizeInBits();
2999
3000 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3001 SDValue Result;
3002 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3003 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3004 } else {
3005 Result = Op;
3006 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3007 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3008 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3009 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3010 }
3011 }
3012
3013 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3014}
3015
3016SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3017 MVT VecVT = Node->getOperand(Num: 1).getSimpleValueType();
3018 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3019 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3020 MVT NewScalarVT = NewVecVT.getVectorElementType();
3021
3022 SDLoc DL(Node);
3023 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3024
3025 // promote the initial value.
3026 // FIXME: Support integer.
3027 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3028 "Only FP promotion is supported");
3029 Operands[0] =
3030 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: 0));
3031
3032 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
3033 if (Node->getOperand(Num: j).getValueType().isVector() &&
3034 !(ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
3035 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3036 // promote the vector operand.
3037 // FIXME: Support integer.
3038 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3039 "Only FP promotion is supported");
3040 Operands[j] =
3041 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3042 } else {
3043 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3044 }
3045
3046 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3047 Flags: Node->getFlags());
3048
3049 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3050 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3051 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3052}
3053
3054bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3055 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3056 SmallVector<SDValue, 8> Results;
3057 SDLoc dl(Node);
3058 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3059 bool NeedInvert;
3060 switch (Node->getOpcode()) {
3061 case ISD::ABS:
3062 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3063 Results.push_back(Elt: Tmp1);
3064 break;
3065 case ISD::ABDS:
3066 case ISD::ABDU:
3067 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3068 Results.push_back(Elt: Tmp1);
3069 break;
3070 case ISD::AVGCEILS:
3071 case ISD::AVGCEILU:
3072 case ISD::AVGFLOORS:
3073 case ISD::AVGFLOORU:
3074 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3075 Results.push_back(Elt: Tmp1);
3076 break;
3077 case ISD::CTPOP:
3078 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3079 Results.push_back(Elt: Tmp1);
3080 break;
3081 case ISD::CTLZ:
3082 case ISD::CTLZ_ZERO_UNDEF:
3083 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3084 Results.push_back(Elt: Tmp1);
3085 break;
3086 case ISD::CTTZ:
3087 case ISD::CTTZ_ZERO_UNDEF:
3088 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3089 Results.push_back(Elt: Tmp1);
3090 break;
3091 case ISD::BITREVERSE:
3092 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3093 Results.push_back(Elt: Tmp1);
3094 break;
3095 case ISD::BSWAP:
3096 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3097 Results.push_back(Elt: Tmp1);
3098 break;
3099 case ISD::PARITY:
3100 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3101 break;
3102 case ISD::FRAMEADDR:
3103 case ISD::RETURNADDR:
3104 case ISD::FRAME_TO_ARGS_OFFSET:
3105 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3106 break;
3107 case ISD::EH_DWARF_CFA: {
3108 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3109 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3110 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3111 VT: CfaArg.getValueType(),
3112 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3113 VT: CfaArg.getValueType()),
3114 N2: CfaArg);
3115 SDValue FA = DAG.getNode(
3116 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3117 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3118 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3119 N1: FA, N2: Offset));
3120 break;
3121 }
3122 case ISD::GET_ROUNDING:
3123 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3124 Results.push_back(Elt: Node->getOperand(Num: 0));
3125 break;
3126 case ISD::EH_RETURN:
3127 case ISD::EH_LABEL:
3128 case ISD::PREFETCH:
3129 case ISD::VAEND:
3130 case ISD::EH_SJLJ_LONGJMP:
3131 // If the target didn't expand these, there's nothing to do, so just
3132 // preserve the chain and be done.
3133 Results.push_back(Elt: Node->getOperand(Num: 0));
3134 break;
3135 case ISD::READCYCLECOUNTER:
3136 case ISD::READSTEADYCOUNTER:
3137 // If the target didn't expand this, just return 'zero' and preserve the
3138 // chain.
3139 Results.append(NumInputs: Node->getNumValues() - 1,
3140 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3141 Results.push_back(Elt: Node->getOperand(Num: 0));
3142 break;
3143 case ISD::EH_SJLJ_SETJMP:
3144 // If the target didn't expand this, just return 'zero' and preserve the
3145 // chain.
3146 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3147 Results.push_back(Elt: Node->getOperand(Num: 0));
3148 break;
3149 case ISD::ATOMIC_LOAD: {
3150 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3151 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3152 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3153 SDValue Swap = DAG.getAtomicCmpSwap(
3154 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3155 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3156 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3157 Results.push_back(Elt: Swap.getValue(R: 0));
3158 Results.push_back(Elt: Swap.getValue(R: 1));
3159 break;
3160 }
3161 case ISD::ATOMIC_STORE: {
3162 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3163 SDValue Swap = DAG.getAtomic(
3164 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3165 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3166 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3167 Results.push_back(Elt: Swap.getValue(R: 1));
3168 break;
3169 }
3170 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3171 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3172 // splits out the success value as a comparison. Expanding the resulting
3173 // ATOMIC_CMP_SWAP will produce a libcall.
3174 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3175 SDValue Res = DAG.getAtomicCmpSwap(
3176 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3177 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3178 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3179
3180 SDValue ExtRes = Res;
3181 SDValue LHS = Res;
3182 SDValue RHS = Node->getOperand(Num: 1);
3183
3184 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3185 EVT OuterType = Node->getValueType(ResNo: 0);
3186 switch (TLI.getExtendForAtomicOps()) {
3187 case ISD::SIGN_EXTEND:
3188 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3189 N2: DAG.getValueType(AtomicType));
3190 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3191 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3192 ExtRes = LHS;
3193 break;
3194 case ISD::ZERO_EXTEND:
3195 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3196 N2: DAG.getValueType(AtomicType));
3197 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3198 ExtRes = LHS;
3199 break;
3200 case ISD::ANY_EXTEND:
3201 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3202 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3203 break;
3204 default:
3205 llvm_unreachable("Invalid atomic op extension");
3206 }
3207
3208 SDValue Success =
3209 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3210
3211 Results.push_back(Elt: ExtRes.getValue(R: 0));
3212 Results.push_back(Elt: Success);
3213 Results.push_back(Elt: Res.getValue(R: 1));
3214 break;
3215 }
3216 case ISD::ATOMIC_LOAD_SUB: {
3217 SDLoc DL(Node);
3218 EVT VT = Node->getValueType(ResNo: 0);
3219 SDValue RHS = Node->getOperand(Num: 2);
3220 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3221 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3222 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3223 RHS = RHS->getOperand(Num: 0);
3224 SDValue NewRHS =
3225 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3226 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3227 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3228 Val: NewRHS, MMO: AN->getMemOperand());
3229 Results.push_back(Elt: Res);
3230 Results.push_back(Elt: Res.getValue(R: 1));
3231 break;
3232 }
3233 case ISD::DYNAMIC_STACKALLOC:
3234 ExpandDYNAMIC_STACKALLOC(Node, Results);
3235 break;
3236 case ISD::MERGE_VALUES:
3237 for (unsigned i = 0; i < Node->getNumValues(); i++)
3238 Results.push_back(Elt: Node->getOperand(Num: i));
3239 break;
3240 case ISD::UNDEF: {
3241 EVT VT = Node->getValueType(ResNo: 0);
3242 if (VT.isInteger())
3243 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3244 else {
3245 assert(VT.isFloatingPoint() && "Unknown value type!");
3246 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3247 }
3248 break;
3249 }
3250 case ISD::STRICT_FP_ROUND:
3251 // When strict mode is enforced we can't do expansion because it
3252 // does not honor the "strict" properties. Only libcall is allowed.
3253 if (TLI.isStrictFPEnabled())
3254 break;
3255 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3256 // since this operation is more efficient than stack operation.
3257 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3258 VT: Node->getValueType(ResNo: 0))
3259 == TargetLowering::Legal)
3260 break;
3261 // We fall back to use stack operation when the FP_ROUND operation
3262 // isn't available.
3263 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3264 DestVT: Node->getValueType(ResNo: 0), dl,
3265 Chain: Node->getOperand(Num: 0)))) {
3266 ReplaceNode(Old: Node, New: Tmp1.getNode());
3267 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3268 return true;
3269 }
3270 break;
3271 case ISD::FP_ROUND: {
3272 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3273 Results.push_back(Elt: Tmp1);
3274 break;
3275 }
3276
3277 [[fallthrough]];
3278 }
3279 case ISD::BITCAST:
3280 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3281 DestVT: Node->getValueType(ResNo: 0), dl)))
3282 Results.push_back(Elt: Tmp1);
3283 break;
3284 case ISD::STRICT_FP_EXTEND:
3285 // When strict mode is enforced we can't do expansion because it
3286 // does not honor the "strict" properties. Only libcall is allowed.
3287 if (TLI.isStrictFPEnabled())
3288 break;
3289 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3290 // since this operation is more efficient than stack operation.
3291 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3292 VT: Node->getValueType(ResNo: 0))
3293 == TargetLowering::Legal)
3294 break;
3295 // We fall back to use stack operation when the FP_EXTEND operation
3296 // isn't available.
3297 if ((Tmp1 = EmitStackConvert(
3298 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3299 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3300 ReplaceNode(Old: Node, New: Tmp1.getNode());
3301 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3302 return true;
3303 }
3304 break;
3305 case ISD::FP_EXTEND: {
3306 SDValue Op = Node->getOperand(Num: 0);
3307 EVT SrcVT = Op.getValueType();
3308 EVT DstVT = Node->getValueType(ResNo: 0);
3309 if (SrcVT.getScalarType() == MVT::bf16) {
3310 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3311 break;
3312 }
3313
3314 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3315 Results.push_back(Elt: Tmp1);
3316 break;
3317 }
3318 case ISD::BF16_TO_FP: {
3319 // Always expand bf16 to f32 casts, they lower to ext + shift.
3320 //
3321 // Note that the operand of this code can be bf16 or an integer type in case
3322 // bf16 is not supported on the target and was softened.
3323 SDValue Op = Node->getOperand(Num: 0);
3324 if (Op.getValueType() == MVT::bf16) {
3325 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3326 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3327 } else {
3328 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3329 }
3330 Op = DAG.getNode(
3331 Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3332 N2: DAG.getConstant(Val: 16, DL: dl,
3333 VT: TLI.getShiftAmountTy(LHSTy: MVT::i32, DL: DAG.getDataLayout())));
3334 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3335 // Add fp_extend in case the output is bigger than f32.
3336 if (Node->getValueType(ResNo: 0) != MVT::f32)
3337 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3338 Results.push_back(Elt: Op);
3339 break;
3340 }
3341 case ISD::FP_TO_BF16: {
3342 SDValue Op = Node->getOperand(Num: 0);
3343 if (Op.getValueType() != MVT::f32)
3344 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3345 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3346 // Certain SNaNs will turn into infinities if we do a simple shift right.
3347 if (!DAG.isKnownNeverSNaN(Op)) {
3348 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3349 }
3350 Op = DAG.getNode(
3351 Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3352 N2: DAG.getConstant(Val: 16, DL: dl,
3353 VT: TLI.getShiftAmountTy(LHSTy: MVT::i32, DL: DAG.getDataLayout())));
3354 // The result of this node can be bf16 or an integer type in case bf16 is
3355 // not supported on the target and was softened to i16 for storage.
3356 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3357 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3358 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3359 } else {
3360 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3361 }
3362 Results.push_back(Elt: Op);
3363 break;
3364 }
3365 case ISD::SIGN_EXTEND_INREG: {
3366 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3367 EVT VT = Node->getValueType(ResNo: 0);
3368
3369 // An in-register sign-extend of a boolean is a negation:
3370 // 'true' (1) sign-extended is -1.
3371 // 'false' (0) sign-extended is 0.
3372 // However, we must mask the high bits of the source operand because the
3373 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3374
3375 // TODO: Do this for vectors too?
3376 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3377 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3378 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3379 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3380 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3381 Results.push_back(Elt: Neg);
3382 break;
3383 }
3384
3385 // NOTE: we could fall back on load/store here too for targets without
3386 // SRA. However, it is doubtful that any exist.
3387 EVT ShiftAmountTy = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3388 unsigned BitsDiff = VT.getScalarSizeInBits() -
3389 ExtraVT.getScalarSizeInBits();
3390 SDValue ShiftCst = DAG.getConstant(Val: BitsDiff, DL: dl, VT: ShiftAmountTy);
3391 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Node->getValueType(ResNo: 0),
3392 N1: Node->getOperand(Num: 0), N2: ShiftCst);
3393 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: ShiftCst);
3394 Results.push_back(Elt: Tmp1);
3395 break;
3396 }
3397 case ISD::UINT_TO_FP:
3398 case ISD::STRICT_UINT_TO_FP:
3399 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3400 Results.push_back(Elt: Tmp1);
3401 if (Node->isStrictFPOpcode())
3402 Results.push_back(Elt: Tmp2);
3403 break;
3404 }
3405 [[fallthrough]];
3406 case ISD::SINT_TO_FP:
3407 case ISD::STRICT_SINT_TO_FP:
3408 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3409 Results.push_back(Elt: Tmp1);
3410 if (Node->isStrictFPOpcode())
3411 Results.push_back(Elt: Tmp2);
3412 }
3413 break;
3414 case ISD::FP_TO_SINT:
3415 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3416 Results.push_back(Elt: Tmp1);
3417 break;
3418 case ISD::STRICT_FP_TO_SINT:
3419 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3420 ReplaceNode(Old: Node, New: Tmp1.getNode());
3421 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3422 return true;
3423 }
3424 break;
3425 case ISD::FP_TO_UINT:
3426 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3427 Results.push_back(Elt: Tmp1);
3428 break;
3429 case ISD::STRICT_FP_TO_UINT:
3430 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3431 // Relink the chain.
3432 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3433 // Replace the new UINT result.
3434 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3435 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3436 return true;
3437 }
3438 break;
3439 case ISD::FP_TO_SINT_SAT:
3440 case ISD::FP_TO_UINT_SAT:
3441 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3442 break;
3443 case ISD::VAARG:
3444 Results.push_back(Elt: DAG.expandVAArg(Node));
3445 Results.push_back(Elt: Results[0].getValue(R: 1));
3446 break;
3447 case ISD::VACOPY:
3448 Results.push_back(Elt: DAG.expandVACopy(Node));
3449 break;
3450 case ISD::EXTRACT_VECTOR_ELT:
3451 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3452 // This must be an access of the only element. Return it.
3453 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3454 Operand: Node->getOperand(Num: 0));
3455 else
3456 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3457 Results.push_back(Elt: Tmp1);
3458 break;
3459 case ISD::EXTRACT_SUBVECTOR:
3460 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3461 break;
3462 case ISD::INSERT_SUBVECTOR:
3463 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3464 break;
3465 case ISD::CONCAT_VECTORS:
3466 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3467 break;
3468 case ISD::SCALAR_TO_VECTOR:
3469 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3470 break;
3471 case ISD::INSERT_VECTOR_ELT:
3472 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3473 break;
3474 case ISD::VECTOR_SHUFFLE: {
3475 SmallVector<int, 32> NewMask;
3476 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3477
3478 EVT VT = Node->getValueType(ResNo: 0);
3479 EVT EltVT = VT.getVectorElementType();
3480 SDValue Op0 = Node->getOperand(Num: 0);
3481 SDValue Op1 = Node->getOperand(Num: 1);
3482 if (!TLI.isTypeLegal(VT: EltVT)) {
3483 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3484
3485 // BUILD_VECTOR operands are allowed to be wider than the element type.
3486 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3487 // it.
3488 if (NewEltVT.bitsLT(VT: EltVT)) {
3489 // Convert shuffle node.
3490 // If original node was v4i64 and the new EltVT is i32,
3491 // cast operands to v8i32 and re-build the mask.
3492
3493 // Calculate new VT, the size of the new VT should be equal to original.
3494 EVT NewVT =
3495 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3496 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3497 assert(NewVT.bitsEq(VT));
3498
3499 // cast operands to new VT
3500 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3501 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3502
3503 // Convert the shuffle mask
3504 unsigned int factor =
3505 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3506
3507 // EltVT gets smaller
3508 assert(factor > 0);
3509
3510 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3511 if (Mask[i] < 0) {
3512 for (unsigned fi = 0; fi < factor; ++fi)
3513 NewMask.push_back(Elt: Mask[i]);
3514 }
3515 else {
3516 for (unsigned fi = 0; fi < factor; ++fi)
3517 NewMask.push_back(Elt: Mask[i]*factor+fi);
3518 }
3519 }
3520 Mask = NewMask;
3521 VT = NewVT;
3522 }
3523 EltVT = NewEltVT;
3524 }
3525 unsigned NumElems = VT.getVectorNumElements();
3526 SmallVector<SDValue, 16> Ops;
3527 for (unsigned i = 0; i != NumElems; ++i) {
3528 if (Mask[i] < 0) {
3529 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3530 continue;
3531 }
3532 unsigned Idx = Mask[i];
3533 if (Idx < NumElems)
3534 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3535 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3536 else
3537 Ops.push_back(
3538 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3539 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3540 }
3541
3542 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3543 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3544 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3545 Results.push_back(Elt: Tmp1);
3546 break;
3547 }
3548 case ISD::VECTOR_SPLICE: {
3549 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3550 break;
3551 }
3552 case ISD::EXTRACT_ELEMENT: {
3553 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3554 if (Node->getConstantOperandVal(Num: 1)) {
3555 // 1 -> Hi
3556 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3557 N2: DAG.getConstant(Val: OpTy.getSizeInBits() / 2, DL: dl,
3558 VT: TLI.getShiftAmountTy(
3559 LHSTy: Node->getOperand(Num: 0).getValueType(),
3560 DL: DAG.getDataLayout())));
3561 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3562 } else {
3563 // 0 -> Lo
3564 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3565 Operand: Node->getOperand(Num: 0));
3566 }
3567 Results.push_back(Elt: Tmp1);
3568 break;
3569 }
3570 case ISD::STACKSAVE:
3571 // Expand to CopyFromReg if the target set
3572 // StackPointerRegisterToSaveRestore.
3573 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3574 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3575 VT: Node->getValueType(ResNo: 0)));
3576 Results.push_back(Elt: Results[0].getValue(R: 1));
3577 } else {
3578 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3579 Results.push_back(Elt: Node->getOperand(Num: 0));
3580 }
3581 break;
3582 case ISD::STACKRESTORE:
3583 // Expand to CopyToReg if the target set
3584 // StackPointerRegisterToSaveRestore.
3585 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3586 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3587 N: Node->getOperand(Num: 1)));
3588 } else {
3589 Results.push_back(Elt: Node->getOperand(Num: 0));
3590 }
3591 break;
3592 case ISD::GET_DYNAMIC_AREA_OFFSET:
3593 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3594 Results.push_back(Elt: Results[0].getValue(R: 0));
3595 break;
3596 case ISD::FCOPYSIGN:
3597 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3598 break;
3599 case ISD::FNEG:
3600 Results.push_back(Elt: ExpandFNEG(Node));
3601 break;
3602 case ISD::FABS:
3603 Results.push_back(Elt: ExpandFABS(Node));
3604 break;
3605 case ISD::IS_FPCLASS: {
3606 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3607 if (SDValue Expanded =
3608 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3609 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3610 Results.push_back(Elt: Expanded);
3611 break;
3612 }
3613 case ISD::SMIN:
3614 case ISD::SMAX:
3615 case ISD::UMIN:
3616 case ISD::UMAX: {
3617 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3618 ISD::CondCode Pred;
3619 switch (Node->getOpcode()) {
3620 default: llvm_unreachable("How did we get here?");
3621 case ISD::SMAX: Pred = ISD::SETGT; break;
3622 case ISD::SMIN: Pred = ISD::SETLT; break;
3623 case ISD::UMAX: Pred = ISD::SETUGT; break;
3624 case ISD::UMIN: Pred = ISD::SETULT; break;
3625 }
3626 Tmp1 = Node->getOperand(Num: 0);
3627 Tmp2 = Node->getOperand(Num: 1);
3628 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3629 Results.push_back(Elt: Tmp1);
3630 break;
3631 }
3632 case ISD::FMINNUM:
3633 case ISD::FMAXNUM: {
3634 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3635 Results.push_back(Elt: Expanded);
3636 break;
3637 }
3638 case ISD::FMINIMUM:
3639 case ISD::FMAXIMUM: {
3640 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
3641 Results.push_back(Elt: Expanded);
3642 break;
3643 }
3644 case ISD::FSIN:
3645 case ISD::FCOS: {
3646 EVT VT = Node->getValueType(ResNo: 0);
3647 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3648 // fcos which share the same operand and both are used.
3649 if ((TLI.isOperationLegalOrCustom(Op: ISD::FSINCOS, VT) ||
3650 isSinCosLibcallAvailable(Node, TLI))
3651 && useSinCos(Node)) {
3652 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3653 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3654 if (Node->getOpcode() == ISD::FCOS)
3655 Tmp1 = Tmp1.getValue(R: 1);
3656 Results.push_back(Elt: Tmp1);
3657 }
3658 break;
3659 }
3660 case ISD::FLDEXP:
3661 case ISD::STRICT_FLDEXP: {
3662 EVT VT = Node->getValueType(ResNo: 0);
3663 RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT);
3664 // Use the LibCall instead, it is very likely faster
3665 // FIXME: Use separate LibCall action.
3666 if (TLI.getLibcallName(Call: LC))
3667 break;
3668
3669 if (SDValue Expanded = expandLdexp(Node)) {
3670 Results.push_back(Elt: Expanded);
3671 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3672 Results.push_back(Elt: Expanded.getValue(R: 1));
3673 }
3674
3675 break;
3676 }
3677 case ISD::FFREXP: {
3678 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0));
3679 // Use the LibCall instead, it is very likely faster
3680 // FIXME: Use separate LibCall action.
3681 if (TLI.getLibcallName(Call: LC))
3682 break;
3683
3684 if (SDValue Expanded = expandFrexp(Node)) {
3685 Results.push_back(Elt: Expanded);
3686 Results.push_back(Elt: Expanded.getValue(R: 1));
3687 }
3688 break;
3689 }
3690 case ISD::FMAD:
3691 llvm_unreachable("Illegal fmad should never be formed");
3692
3693 case ISD::FP16_TO_FP:
3694 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3695 // We can extend to types bigger than f32 in two steps without changing
3696 // the result. Since "f16 -> f32" is much more commonly available, give
3697 // CodeGen the option of emitting that before resorting to a libcall.
3698 SDValue Res =
3699 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
3700 Results.push_back(
3701 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
3702 }
3703 break;
3704 case ISD::STRICT_BF16_TO_FP:
3705 case ISD::STRICT_FP16_TO_FP:
3706 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3707 // We can extend to types bigger than f32 in two steps without changing
3708 // the result. Since "f16 -> f32" is much more commonly available, give
3709 // CodeGen the option of emitting that before resorting to a libcall.
3710 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
3711 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
3712 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
3713 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
3714 Ops: {Res.getValue(R: 1), Res});
3715 Results.push_back(Elt: Res);
3716 Results.push_back(Elt: Res.getValue(R: 1));
3717 }
3718 break;
3719 case ISD::FP_TO_FP16:
3720 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3721 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3722 SDValue Op = Node->getOperand(Num: 0);
3723 MVT SVT = Op.getSimpleValueType();
3724 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3725 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
3726 // Under fastmath, we can expand this node into a fround followed by
3727 // a float-half conversion.
3728 SDValue FloatVal =
3729 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3730 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3731 Results.push_back(
3732 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
3733 }
3734 }
3735 break;
3736 case ISD::ConstantFP: {
3737 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
3738 // Check to see if this FP immediate is already legal.
3739 // If this is a legal constant, turn it into a TargetConstantFP node.
3740 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
3741 ForCodeSize: DAG.shouldOptForSize()))
3742 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
3743 break;
3744 }
3745 case ISD::Constant: {
3746 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
3747 Results.push_back(Elt: ExpandConstant(CP));
3748 break;
3749 }
3750 case ISD::FSUB: {
3751 EVT VT = Node->getValueType(ResNo: 0);
3752 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
3753 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
3754 const SDNodeFlags Flags = Node->getFlags();
3755 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
3756 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
3757 Results.push_back(Elt: Tmp1);
3758 }
3759 break;
3760 }
3761 case ISD::SUB: {
3762 EVT VT = Node->getValueType(ResNo: 0);
3763 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3764 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3765 "Don't know how to expand this subtraction!");
3766 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
3767 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3768 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
3769 break;
3770 }
3771 case ISD::UREM:
3772 case ISD::SREM:
3773 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
3774 Results.push_back(Elt: Tmp1);
3775 break;
3776 case ISD::UDIV:
3777 case ISD::SDIV: {
3778 bool isSigned = Node->getOpcode() == ISD::SDIV;
3779 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3780 EVT VT = Node->getValueType(ResNo: 0);
3781 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
3782 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3783 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3784 N2: Node->getOperand(Num: 1));
3785 Results.push_back(Elt: Tmp1);
3786 }
3787 break;
3788 }
3789 case ISD::MULHU:
3790 case ISD::MULHS: {
3791 unsigned ExpandOpcode =
3792 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3793 EVT VT = Node->getValueType(ResNo: 0);
3794 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3795
3796 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3797 N2: Node->getOperand(Num: 1));
3798 Results.push_back(Elt: Tmp1.getValue(R: 1));
3799 break;
3800 }
3801 case ISD::UMUL_LOHI:
3802 case ISD::SMUL_LOHI: {
3803 SDValue LHS = Node->getOperand(Num: 0);
3804 SDValue RHS = Node->getOperand(Num: 1);
3805 MVT VT = LHS.getSimpleValueType();
3806 unsigned MULHOpcode =
3807 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3808
3809 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
3810 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
3811 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
3812 break;
3813 }
3814
3815 SmallVector<SDValue, 4> Halves;
3816 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(Context&: *DAG.getContext());
3817 assert(TLI.isTypeLegal(HalfType));
3818 if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
3819 HiLoVT: HalfType, DAG,
3820 Kind: TargetLowering::MulExpansionKind::Always)) {
3821 for (unsigned i = 0; i < 2; ++i) {
3822 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
3823 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
3824 SDValue Shift = DAG.getConstant(
3825 Val: HalfType.getScalarSizeInBits(), DL: dl,
3826 VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout()));
3827 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
3828 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
3829 }
3830 break;
3831 }
3832 break;
3833 }
3834 case ISD::MUL: {
3835 EVT VT = Node->getValueType(ResNo: 0);
3836 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3837 // See if multiply or divide can be lowered using two-result operations.
3838 // We just need the low half of the multiply; try both the signed
3839 // and unsigned forms. If the target supports both SMUL_LOHI and
3840 // UMUL_LOHI, form a preference by checking which forms of plain
3841 // MULH it supports.
3842 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
3843 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
3844 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
3845 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
3846 unsigned OpToUse = 0;
3847 if (HasSMUL_LOHI && !HasMULHS) {
3848 OpToUse = ISD::SMUL_LOHI;
3849 } else if (HasUMUL_LOHI && !HasMULHU) {
3850 OpToUse = ISD::UMUL_LOHI;
3851 } else if (HasSMUL_LOHI) {
3852 OpToUse = ISD::SMUL_LOHI;
3853 } else if (HasUMUL_LOHI) {
3854 OpToUse = ISD::UMUL_LOHI;
3855 }
3856 if (OpToUse) {
3857 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3858 N2: Node->getOperand(Num: 1)));
3859 break;
3860 }
3861
3862 SDValue Lo, Hi;
3863 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
3864 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
3865 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
3866 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
3867 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
3868 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
3869 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3870 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
3871 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
3872 SDValue Shift =
3873 DAG.getConstant(Val: HalfType.getSizeInBits(), DL: dl,
3874 VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout()));
3875 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
3876 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
3877 }
3878 break;
3879 }
3880 case ISD::FSHL:
3881 case ISD::FSHR:
3882 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
3883 Results.push_back(Elt: Expanded);
3884 break;
3885 case ISD::ROTL:
3886 case ISD::ROTR:
3887 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
3888 Results.push_back(Elt: Expanded);
3889 break;
3890 case ISD::SADDSAT:
3891 case ISD::UADDSAT:
3892 case ISD::SSUBSAT:
3893 case ISD::USUBSAT:
3894 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
3895 break;
3896 case ISD::SCMP:
3897 case ISD::UCMP:
3898 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
3899 break;
3900 case ISD::SSHLSAT:
3901 case ISD::USHLSAT:
3902 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
3903 break;
3904 case ISD::SMULFIX:
3905 case ISD::SMULFIXSAT:
3906 case ISD::UMULFIX:
3907 case ISD::UMULFIXSAT:
3908 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
3909 break;
3910 case ISD::SDIVFIX:
3911 case ISD::SDIVFIXSAT:
3912 case ISD::UDIVFIX:
3913 case ISD::UDIVFIXSAT:
3914 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
3915 LHS: Node->getOperand(Num: 0),
3916 RHS: Node->getOperand(Num: 1),
3917 Scale: Node->getConstantOperandVal(Num: 2),
3918 DAG)) {
3919 Results.push_back(Elt: V);
3920 break;
3921 }
3922 // FIXME: We might want to retry here with a wider type if we fail, if that
3923 // type is legal.
3924 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3925 // <= 128 (which is the case for all of the default Embedded-C types),
3926 // we will only get here with types and scales that we could always expand
3927 // if we were allowed to generate libcalls to division functions of illegal
3928 // type. But we cannot do that.
3929 llvm_unreachable("Cannot expand DIVFIX!");
3930 case ISD::UADDO_CARRY:
3931 case ISD::USUBO_CARRY: {
3932 SDValue LHS = Node->getOperand(Num: 0);
3933 SDValue RHS = Node->getOperand(Num: 1);
3934 SDValue Carry = Node->getOperand(Num: 2);
3935
3936 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3937
3938 // Initial add of the 2 operands.
3939 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3940 EVT VT = LHS.getValueType();
3941 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
3942
3943 // Initial check for overflow.
3944 EVT CarryType = Node->getValueType(ResNo: 1);
3945 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
3946 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3947 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
3948
3949 // Add of the sum and the carry.
3950 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3951 SDValue CarryExt =
3952 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
3953 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
3954
3955 // Second check for overflow. If we are adding, we can only overflow if the
3956 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3957 // If we are subtracting, we can only overflow if the initial sum is 0 and
3958 // the carry is set, resulting in a new sum of all 1s.
3959 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3960 SDValue Overflow2 =
3961 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
3962 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
3963 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
3964 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
3965
3966 SDValue ResultCarry =
3967 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
3968
3969 Results.push_back(Elt: Sum2);
3970 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
3971 break;
3972 }
3973 case ISD::SADDO:
3974 case ISD::SSUBO: {
3975 SDValue Result, Overflow;
3976 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3977 Results.push_back(Elt: Result);
3978 Results.push_back(Elt: Overflow);
3979 break;
3980 }
3981 case ISD::UADDO:
3982 case ISD::USUBO: {
3983 SDValue Result, Overflow;
3984 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3985 Results.push_back(Elt: Result);
3986 Results.push_back(Elt: Overflow);
3987 break;
3988 }
3989 case ISD::UMULO:
3990 case ISD::SMULO: {
3991 SDValue Result, Overflow;
3992 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3993 Results.push_back(Elt: Result);
3994 Results.push_back(Elt: Overflow);
3995 }
3996 break;
3997 }
3998 case ISD::BUILD_PAIR: {
3999 EVT PairTy = Node->getValueType(ResNo: 0);
4000 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4001 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4002 Tmp2 = DAG.getNode(
4003 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4004 N2: DAG.getConstant(Val: PairTy.getSizeInBits() / 2, DL: dl,
4005 VT: TLI.getShiftAmountTy(LHSTy: PairTy, DL: DAG.getDataLayout())));
4006 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4007 break;
4008 }
4009 case ISD::SELECT:
4010 Tmp1 = Node->getOperand(Num: 0);
4011 Tmp2 = Node->getOperand(Num: 1);
4012 Tmp3 = Node->getOperand(Num: 2);
4013 if (Tmp1.getOpcode() == ISD::SETCC) {
4014 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1),
4015 True: Tmp2, False: Tmp3,
4016 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get());
4017 } else {
4018 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1,
4019 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4020 True: Tmp2, False: Tmp3, Cond: ISD::SETNE);
4021 }
4022 Tmp1->setFlags(Node->getFlags());
4023 Results.push_back(Elt: Tmp1);
4024 break;
4025 case ISD::BR_JT: {
4026 SDValue Chain = Node->getOperand(Num: 0);
4027 SDValue Table = Node->getOperand(Num: 1);
4028 SDValue Index = Node->getOperand(Num: 2);
4029 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4030
4031 const DataLayout &TD = DAG.getDataLayout();
4032 EVT PTy = TLI.getPointerTy(DL: TD);
4033
4034 unsigned EntrySize =
4035 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4036
4037 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4038 // This transformation needs to be done here since otherwise the MIPS
4039 // backend will end up emitting a three instruction multiply sequence
4040 // instead of a single shift and MSP430 will call a runtime function.
4041 if (llvm::isPowerOf2_32(Value: EntrySize))
4042 Index = DAG.getNode(
4043 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4044 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4045 else
4046 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4047 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4048 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Index.getValueType(),
4049 N1: Index, N2: Table);
4050
4051 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4052 SDValue LD = DAG.getExtLoad(
4053 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4054 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4055 Addr = LD;
4056 if (TLI.isJumpTableRelative()) {
4057 // For PIC, the sequence is:
4058 // BRIND(load(Jumptable + index) + RelocBase)
4059 // RelocBase can be JumpTable, GOT or some sort of global base.
4060 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Addr,
4061 N2: TLI.getPICJumpTableRelocBase(Table, DAG));
4062 }
4063
4064 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4065 Results.push_back(Elt: Tmp1);
4066 break;
4067 }
4068 case ISD::BRCOND:
4069 // Expand brcond's setcc into its constituent parts and create a BR_CC
4070 // Node.
4071 Tmp1 = Node->getOperand(Num: 0);
4072 Tmp2 = Node->getOperand(Num: 1);
4073 if (Tmp2.getOpcode() == ISD::SETCC &&
4074 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4075 VT: Tmp2.getOperand(i: 0).getValueType())) {
4076 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4077 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4078 N5: Node->getOperand(Num: 2));
4079 } else {
4080 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4081 if (Tmp2.isUndef() ||
4082 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4083 Tmp3 = Tmp2;
4084 else
4085 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4086 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4087 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4088 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4089 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4090 N5: Node->getOperand(Num: 2));
4091 }
4092 Results.push_back(Elt: Tmp1);
4093 break;
4094 case ISD::SETCC:
4095 case ISD::VP_SETCC:
4096 case ISD::STRICT_FSETCC:
4097 case ISD::STRICT_FSETCCS: {
4098 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4099 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4100 Node->getOpcode() == ISD::STRICT_FSETCCS;
4101 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4102 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4103 unsigned Offset = IsStrict ? 1 : 0;
4104 Tmp1 = Node->getOperand(Num: 0 + Offset);
4105 Tmp2 = Node->getOperand(Num: 1 + Offset);
4106 Tmp3 = Node->getOperand(Num: 2 + Offset);
4107 SDValue Mask, EVL;
4108 if (IsVP) {
4109 Mask = Node->getOperand(Num: 3 + Offset);
4110 EVL = Node->getOperand(Num: 4 + Offset);
4111 }
4112 bool Legalized = TLI.LegalizeSetCCCondCode(
4113 DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl,
4114 Chain, IsSignaling);
4115
4116 if (Legalized) {
4117 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4118 // condition code, create a new SETCC node.
4119 if (Tmp3.getNode()) {
4120 if (IsStrict) {
4121 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4122 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4123 Chain = Tmp1.getValue(R: 1);
4124 } else if (IsVP) {
4125 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0),
4126 Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags());
4127 } else {
4128 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4129 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4130 }
4131 }
4132
4133 // If we expanded the SETCC by inverting the condition code, then wrap
4134 // the existing SETCC in a NOT to restore the intended condition.
4135 if (NeedInvert) {
4136 if (!IsVP)
4137 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4138 else
4139 Tmp1 =
4140 DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0));
4141 }
4142
4143 Results.push_back(Elt: Tmp1);
4144 if (IsStrict)
4145 Results.push_back(Elt: Chain);
4146
4147 break;
4148 }
4149
4150 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4151 // understand if this code is useful for strict nodes.
4152 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4153
4154 // Otherwise, SETCC for the given comparison type must be completely
4155 // illegal; expand it into a SELECT_CC.
4156 // FIXME: This drops the mask/evl for VP_SETCC.
4157 EVT VT = Node->getValueType(ResNo: 0);
4158 EVT Tmp1VT = Tmp1.getValueType();
4159 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4160 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4161 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3);
4162 Tmp1->setFlags(Node->getFlags());
4163 Results.push_back(Elt: Tmp1);
4164 break;
4165 }
4166 case ISD::SELECT_CC: {
4167 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4168 Tmp1 = Node->getOperand(Num: 0); // LHS
4169 Tmp2 = Node->getOperand(Num: 1); // RHS
4170 Tmp3 = Node->getOperand(Num: 2); // True
4171 Tmp4 = Node->getOperand(Num: 3); // False
4172 EVT VT = Node->getValueType(ResNo: 0);
4173 SDValue Chain;
4174 SDValue CC = Node->getOperand(Num: 4);
4175 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4176
4177 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4178 // If the condition code is legal, then we need to expand this
4179 // node using SETCC and SELECT.
4180 EVT CmpVT = Tmp1.getValueType();
4181 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4182 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4183 "expanded.");
4184 EVT CCVT = getSetCCResultType(VT: CmpVT);
4185 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4186 Results.push_back(
4187 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4188 break;
4189 }
4190
4191 // SELECT_CC is legal, so the condition code must not be.
4192 bool Legalized = false;
4193 // Try to legalize by inverting the condition. This is for targets that
4194 // might support an ordered version of a condition, but not the unordered
4195 // version (or vice versa).
4196 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4197 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4198 // Use the new condition code and swap true and false
4199 Legalized = true;
4200 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC);
4201 Tmp1->setFlags(Node->getFlags());
4202 } else {
4203 // If The inverse is not legal, then try to swap the arguments using
4204 // the inverse condition code.
4205 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4206 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4207 // The swapped inverse condition is legal, so swap true and false,
4208 // lhs and rhs.
4209 Legalized = true;
4210 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC);
4211 Tmp1->setFlags(Node->getFlags());
4212 }
4213 }
4214
4215 if (!Legalized) {
4216 Legalized = TLI.LegalizeSetCCCondCode(
4217 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4218 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4219
4220 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4221
4222 // If we expanded the SETCC by inverting the condition code, then swap
4223 // the True/False operands to match.
4224 if (NeedInvert)
4225 std::swap(a&: Tmp3, b&: Tmp4);
4226
4227 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4228 // condition code, create a new SELECT_CC node.
4229 if (CC.getNode()) {
4230 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
4231 N1: Tmp1, N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC);
4232 } else {
4233 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4234 CC = DAG.getCondCode(Cond: ISD::SETNE);
4235 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4236 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC);
4237 }
4238 Tmp1->setFlags(Node->getFlags());
4239 }
4240 Results.push_back(Elt: Tmp1);
4241 break;
4242 }
4243 case ISD::BR_CC: {
4244 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4245 SDValue Chain;
4246 Tmp1 = Node->getOperand(Num: 0); // Chain
4247 Tmp2 = Node->getOperand(Num: 2); // LHS
4248 Tmp3 = Node->getOperand(Num: 3); // RHS
4249 Tmp4 = Node->getOperand(Num: 1); // CC
4250
4251 bool Legalized = TLI.LegalizeSetCCCondCode(
4252 DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4,
4253 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4254 (void)Legalized;
4255 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4256
4257 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4258 // node.
4259 if (Tmp4.getNode()) {
4260 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4261
4262 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4263 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4264 } else {
4265 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4266 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4267 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4268 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4269 }
4270 Results.push_back(Elt: Tmp1);
4271 break;
4272 }
4273 case ISD::BUILD_VECTOR:
4274 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4275 break;
4276 case ISD::SPLAT_VECTOR:
4277 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4278 break;
4279 case ISD::SRA:
4280 case ISD::SRL:
4281 case ISD::SHL: {
4282 // Scalarize vector SRA/SRL/SHL.
4283 EVT VT = Node->getValueType(ResNo: 0);
4284 assert(VT.isVector() && "Unable to legalize non-vector shift");
4285 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4286 unsigned NumElem = VT.getVectorNumElements();
4287
4288 SmallVector<SDValue, 8> Scalars;
4289 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4290 SDValue Ex =
4291 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4292 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4293 SDValue Sh =
4294 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4295 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4296 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4297 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4298 }
4299
4300 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4301 Results.push_back(Elt: Result);
4302 break;
4303 }
4304 case ISD::VECREDUCE_FADD:
4305 case ISD::VECREDUCE_FMUL:
4306 case ISD::VECREDUCE_ADD:
4307 case ISD::VECREDUCE_MUL:
4308 case ISD::VECREDUCE_AND:
4309 case ISD::VECREDUCE_OR:
4310 case ISD::VECREDUCE_XOR:
4311 case ISD::VECREDUCE_SMAX:
4312 case ISD::VECREDUCE_SMIN:
4313 case ISD::VECREDUCE_UMAX:
4314 case ISD::VECREDUCE_UMIN:
4315 case ISD::VECREDUCE_FMAX:
4316 case ISD::VECREDUCE_FMIN:
4317 case ISD::VECREDUCE_FMAXIMUM:
4318 case ISD::VECREDUCE_FMINIMUM:
4319 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4320 break;
4321 case ISD::VP_CTTZ_ELTS:
4322 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4323 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4324 break;
4325 case ISD::CLEAR_CACHE:
4326 // The default expansion of llvm.clear_cache is simply a no-op for those
4327 // targets where it is not needed.
4328 Results.push_back(Elt: Node->getOperand(Num: 0));
4329 break;
4330 case ISD::GLOBAL_OFFSET_TABLE:
4331 case ISD::GlobalAddress:
4332 case ISD::GlobalTLSAddress:
4333 case ISD::ExternalSymbol:
4334 case ISD::ConstantPool:
4335 case ISD::JumpTable:
4336 case ISD::INTRINSIC_W_CHAIN:
4337 case ISD::INTRINSIC_WO_CHAIN:
4338 case ISD::INTRINSIC_VOID:
4339 // FIXME: Custom lowering for these operations shouldn't return null!
4340 // Return true so that we don't call ConvertNodeToLibcall which also won't
4341 // do anything.
4342 return true;
4343 }
4344
4345 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4346 // FIXME: We were asked to expand a strict floating-point operation,
4347 // but there is currently no expansion implemented that would preserve
4348 // the "strict" properties. For now, we just fall back to the non-strict
4349 // version if that is legal on the target. The actual mutation of the
4350 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4351 switch (Node->getOpcode()) {
4352 default:
4353 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4354 VT: Node->getValueType(ResNo: 0))
4355 == TargetLowering::Legal)
4356 return true;
4357 break;
4358 case ISD::STRICT_FSUB: {
4359 if (TLI.getStrictFPOperationAction(
4360 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4361 return true;
4362 if (TLI.getStrictFPOperationAction(
4363 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4364 break;
4365
4366 EVT VT = Node->getValueType(ResNo: 0);
4367 const SDNodeFlags Flags = Node->getFlags();
4368 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4369 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4370 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4371 Flags);
4372
4373 Results.push_back(Elt: Fadd);
4374 Results.push_back(Elt: Fadd.getValue(R: 1));
4375 break;
4376 }
4377 case ISD::STRICT_SINT_TO_FP:
4378 case ISD::STRICT_UINT_TO_FP:
4379 case ISD::STRICT_LRINT:
4380 case ISD::STRICT_LLRINT:
4381 case ISD::STRICT_LROUND:
4382 case ISD::STRICT_LLROUND:
4383 // These are registered by the operand type instead of the value
4384 // type. Reflect that here.
4385 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4386 VT: Node->getOperand(Num: 1).getValueType())
4387 == TargetLowering::Legal)
4388 return true;
4389 break;
4390 }
4391 }
4392
4393 // Replace the original node with the legalized result.
4394 if (Results.empty()) {
4395 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4396 return false;
4397 }
4398
4399 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4400 ReplaceNode(Old: Node, New: Results.data());
4401 return true;
4402}
4403
4404void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4405 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4406 SmallVector<SDValue, 8> Results;
4407 SDLoc dl(Node);
4408 // FIXME: Check flags on the node to see if we can use a finite call.
4409 unsigned Opc = Node->getOpcode();
4410 switch (Opc) {
4411 case ISD::ATOMIC_FENCE: {
4412 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4413 // FIXME: handle "fence singlethread" more efficiently.
4414 TargetLowering::ArgListTy Args;
4415
4416 TargetLowering::CallLoweringInfo CLI(DAG);
4417 CLI.setDebugLoc(dl)
4418 .setChain(Node->getOperand(Num: 0))
4419 .setLibCallee(
4420 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4421 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4422 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4423 ArgsList: std::move(Args));
4424
4425 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4426
4427 Results.push_back(Elt: CallResult.second);
4428 break;
4429 }
4430 // By default, atomic intrinsics are marked Legal and lowered. Targets
4431 // which don't support them directly, however, may want libcalls, in which
4432 // case they mark them Expand, and we get here.
4433 case ISD::ATOMIC_SWAP:
4434 case ISD::ATOMIC_LOAD_ADD:
4435 case ISD::ATOMIC_LOAD_SUB:
4436 case ISD::ATOMIC_LOAD_AND:
4437 case ISD::ATOMIC_LOAD_CLR:
4438 case ISD::ATOMIC_LOAD_OR:
4439 case ISD::ATOMIC_LOAD_XOR:
4440 case ISD::ATOMIC_LOAD_NAND:
4441 case ISD::ATOMIC_LOAD_MIN:
4442 case ISD::ATOMIC_LOAD_MAX:
4443 case ISD::ATOMIC_LOAD_UMIN:
4444 case ISD::ATOMIC_LOAD_UMAX:
4445 case ISD::ATOMIC_CMP_SWAP: {
4446 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4447 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4448 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4449 EVT RetVT = Node->getValueType(ResNo: 0);
4450 TargetLowering::MakeLibCallOptions CallOptions;
4451 SmallVector<SDValue, 4> Ops;
4452 if (TLI.getLibcallName(Call: LC)) {
4453 // If outline atomic available, prepare its arguments and expand.
4454 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4455 Ops.push_back(Elt: Node->getOperand(Num: 1));
4456
4457 } else {
4458 LC = RTLIB::getSYNC(Opc, VT);
4459 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4460 "Unexpected atomic op or value type!");
4461 // Arguments for expansion to sync libcall
4462 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4463 }
4464 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4465 Ops, CallOptions,
4466 dl: SDLoc(Node),
4467 Chain: Node->getOperand(Num: 0));
4468 Results.push_back(Elt: Tmp.first);
4469 Results.push_back(Elt: Tmp.second);
4470 break;
4471 }
4472 case ISD::TRAP: {
4473 // If this operation is not supported, lower it to 'abort()' call
4474 TargetLowering::ArgListTy Args;
4475 TargetLowering::CallLoweringInfo CLI(DAG);
4476 CLI.setDebugLoc(dl)
4477 .setChain(Node->getOperand(Num: 0))
4478 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4479 Target: DAG.getExternalSymbol(
4480 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4481 ArgsList: std::move(Args));
4482 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4483
4484 Results.push_back(Elt: CallResult.second);
4485 break;
4486 }
4487 case ISD::CLEAR_CACHE: {
4488 TargetLowering::MakeLibCallOptions CallOptions;
4489 SDValue InputChain = Node->getOperand(Num: 0);
4490 SDValue StartVal = Node->getOperand(Num: 1);
4491 SDValue EndVal = Node->getOperand(Num: 2);
4492 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4493 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
4494 dl: SDLoc(Node), Chain: InputChain);
4495 Results.push_back(Elt: Tmp.second);
4496 break;
4497 }
4498 case ISD::FMINNUM:
4499 case ISD::STRICT_FMINNUM:
4500 ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
4501 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
4502 Call_PPCF128: RTLIB::FMIN_PPCF128, Results);
4503 break;
4504 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4505 // libcall legalization for these nodes, but there is no default expasion for
4506 // these nodes either (see PR63267 for example).
4507 case ISD::FMAXNUM:
4508 case ISD::STRICT_FMAXNUM:
4509 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
4510 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
4511 Call_PPCF128: RTLIB::FMAX_PPCF128, Results);
4512 break;
4513 case ISD::FSQRT:
4514 case ISD::STRICT_FSQRT:
4515 ExpandFPLibCall(Node, Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
4516 Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
4517 Call_PPCF128: RTLIB::SQRT_PPCF128, Results);
4518 break;
4519 case ISD::FCBRT:
4520 ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64,
4521 Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128,
4522 Call_PPCF128: RTLIB::CBRT_PPCF128, Results);
4523 break;
4524 case ISD::FSIN:
4525 case ISD::STRICT_FSIN:
4526 ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
4527 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
4528 Call_PPCF128: RTLIB::SIN_PPCF128, Results);
4529 break;
4530 case ISD::FCOS:
4531 case ISD::STRICT_FCOS:
4532 ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
4533 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
4534 Call_PPCF128: RTLIB::COS_PPCF128, Results);
4535 break;
4536 case ISD::FTAN:
4537 case ISD::STRICT_FTAN:
4538 ExpandFPLibCall(Node, Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
4539 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128, Results);
4540 break;
4541 case ISD::FASIN:
4542 case ISD::STRICT_FASIN:
4543 ExpandFPLibCall(Node, Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
4544 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128, Results);
4545 break;
4546 case ISD::FACOS:
4547 case ISD::STRICT_FACOS:
4548 ExpandFPLibCall(Node, Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
4549 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128, Results);
4550 break;
4551 case ISD::FATAN:
4552 case ISD::STRICT_FATAN:
4553 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
4554 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128, Results);
4555 break;
4556 case ISD::FSINH:
4557 case ISD::STRICT_FSINH:
4558 ExpandFPLibCall(Node, Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
4559 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128, Results);
4560 break;
4561 case ISD::FCOSH:
4562 case ISD::STRICT_FCOSH:
4563 ExpandFPLibCall(Node, Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
4564 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128, Results);
4565 break;
4566 case ISD::FTANH:
4567 case ISD::STRICT_FTANH:
4568 ExpandFPLibCall(Node, Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
4569 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128, Results);
4570 break;
4571 case ISD::FSINCOS:
4572 // Expand into sincos libcall.
4573 ExpandSinCosLibCall(Node, Results);
4574 break;
4575 case ISD::FLOG:
4576 case ISD::STRICT_FLOG:
4577 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80,
4578 Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results);
4579 break;
4580 case ISD::FLOG2:
4581 case ISD::STRICT_FLOG2:
4582 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80,
4583 Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results);
4584 break;
4585 case ISD::FLOG10:
4586 case ISD::STRICT_FLOG10:
4587 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80,
4588 Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results);
4589 break;
4590 case ISD::FEXP:
4591 case ISD::STRICT_FEXP:
4592 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80,
4593 Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results);
4594 break;
4595 case ISD::FEXP2:
4596 case ISD::STRICT_FEXP2:
4597 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80,
4598 Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results);
4599 break;
4600 case ISD::FEXP10:
4601 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
4602 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results);
4603 break;
4604 case ISD::FTRUNC:
4605 case ISD::STRICT_FTRUNC:
4606 ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
4607 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
4608 Call_PPCF128: RTLIB::TRUNC_PPCF128, Results);
4609 break;
4610 case ISD::FFLOOR:
4611 case ISD::STRICT_FFLOOR:
4612 ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
4613 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
4614 Call_PPCF128: RTLIB::FLOOR_PPCF128, Results);
4615 break;
4616 case ISD::FCEIL:
4617 case ISD::STRICT_FCEIL:
4618 ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
4619 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
4620 Call_PPCF128: RTLIB::CEIL_PPCF128, Results);
4621 break;
4622 case ISD::FRINT:
4623 case ISD::STRICT_FRINT:
4624 ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
4625 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
4626 Call_PPCF128: RTLIB::RINT_PPCF128, Results);
4627 break;
4628 case ISD::FNEARBYINT:
4629 case ISD::STRICT_FNEARBYINT:
4630 ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32,
4631 Call_F64: RTLIB::NEARBYINT_F64,
4632 Call_F80: RTLIB::NEARBYINT_F80,
4633 Call_F128: RTLIB::NEARBYINT_F128,
4634 Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results);
4635 break;
4636 case ISD::FROUND:
4637 case ISD::STRICT_FROUND:
4638 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32,
4639 Call_F64: RTLIB::ROUND_F64,
4640 Call_F80: RTLIB::ROUND_F80,
4641 Call_F128: RTLIB::ROUND_F128,
4642 Call_PPCF128: RTLIB::ROUND_PPCF128, Results);
4643 break;
4644 case ISD::FROUNDEVEN:
4645 case ISD::STRICT_FROUNDEVEN:
4646 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32,
4647 Call_F64: RTLIB::ROUNDEVEN_F64,
4648 Call_F80: RTLIB::ROUNDEVEN_F80,
4649 Call_F128: RTLIB::ROUNDEVEN_F128,
4650 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results);
4651 break;
4652 case ISD::FLDEXP:
4653 case ISD::STRICT_FLDEXP:
4654 ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80,
4655 Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results);
4656 break;
4657 case ISD::FFREXP: {
4658 ExpandFrexpLibCall(Node, Results);
4659 break;
4660 }
4661 case ISD::FPOWI:
4662 case ISD::STRICT_FPOWI: {
4663 RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0));
4664 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4665 if (!TLI.getLibcallName(Call: LC)) {
4666 // Some targets don't have a powi libcall; use pow instead.
4667 if (Node->isStrictFPOpcode()) {
4668 SDValue Exponent =
4669 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
4670 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4671 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
4672 SDValue FPOW =
4673 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
4674 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4675 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
4676 Results.push_back(Elt: FPOW);
4677 Results.push_back(Elt: FPOW.getValue(R: 1));
4678 } else {
4679 SDValue Exponent =
4680 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
4681 Operand: Node->getOperand(Num: 1));
4682 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
4683 VT: Node->getValueType(ResNo: 0),
4684 N1: Node->getOperand(Num: 0), N2: Exponent));
4685 }
4686 break;
4687 }
4688 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4689 bool ExponentHasSizeOfInt =
4690 DAG.getLibInfo().getIntSize() ==
4691 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
4692 if (!ExponentHasSizeOfInt) {
4693 // If the exponent does not match with sizeof(int) a libcall to
4694 // RTLIB::POWI would use the wrong type for the argument.
4695 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
4696 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
4697 break;
4698 }
4699 ExpandFPLibCall(Node, LC, Results);
4700 break;
4701 }
4702 case ISD::FPOW:
4703 case ISD::STRICT_FPOW:
4704 ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80,
4705 Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results);
4706 break;
4707 case ISD::LROUND:
4708 case ISD::STRICT_LROUND:
4709 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
4710 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
4711 Call_F128: RTLIB::LROUND_F128,
4712 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
4713 break;
4714 case ISD::LLROUND:
4715 case ISD::STRICT_LLROUND:
4716 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
4717 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
4718 Call_F128: RTLIB::LLROUND_F128,
4719 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
4720 break;
4721 case ISD::LRINT:
4722 case ISD::STRICT_LRINT:
4723 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
4724 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
4725 Call_F128: RTLIB::LRINT_F128,
4726 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
4727 break;
4728 case ISD::LLRINT:
4729 case ISD::STRICT_LLRINT:
4730 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
4731 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
4732 Call_F128: RTLIB::LLRINT_F128,
4733 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
4734 break;
4735 case ISD::FDIV:
4736 case ISD::STRICT_FDIV:
4737 ExpandFPLibCall(Node, Call_F32: RTLIB::DIV_F32, Call_F64: RTLIB::DIV_F64,
4738 Call_F80: RTLIB::DIV_F80, Call_F128: RTLIB::DIV_F128,
4739 Call_PPCF128: RTLIB::DIV_PPCF128, Results);
4740 break;
4741 case ISD::FREM:
4742 case ISD::STRICT_FREM:
4743 ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
4744 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
4745 Call_PPCF128: RTLIB::REM_PPCF128, Results);
4746 break;
4747 case ISD::FMA:
4748 case ISD::STRICT_FMA:
4749 ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64,
4750 Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128,
4751 Call_PPCF128: RTLIB::FMA_PPCF128, Results);
4752 break;
4753 case ISD::FADD:
4754 case ISD::STRICT_FADD:
4755 ExpandFPLibCall(Node, Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
4756 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
4757 Call_PPCF128: RTLIB::ADD_PPCF128, Results);
4758 break;
4759 case ISD::FMUL:
4760 case ISD::STRICT_FMUL:
4761 ExpandFPLibCall(Node, Call_F32: RTLIB::MUL_F32, Call_F64: RTLIB::MUL_F64,
4762 Call_F80: RTLIB::MUL_F80, Call_F128: RTLIB::MUL_F128,
4763 Call_PPCF128: RTLIB::MUL_PPCF128, Results);
4764 break;
4765 case ISD::FP16_TO_FP:
4766 if (Node->getValueType(ResNo: 0) == MVT::f32) {
4767 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
4768 }
4769 break;
4770 case ISD::STRICT_BF16_TO_FP:
4771 if (Node->getValueType(ResNo: 0) == MVT::f32) {
4772 TargetLowering::MakeLibCallOptions CallOptions;
4773 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4774 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
4775 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
4776 Results.push_back(Elt: Tmp.first);
4777 Results.push_back(Elt: Tmp.second);
4778 }
4779 break;
4780 case ISD::STRICT_FP16_TO_FP: {
4781 if (Node->getValueType(ResNo: 0) == MVT::f32) {
4782 TargetLowering::MakeLibCallOptions CallOptions;
4783 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4784 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
4785 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
4786 Results.push_back(Elt: Tmp.first);
4787 Results.push_back(Elt: Tmp.second);
4788 }
4789 break;
4790 }
4791 case ISD::FP_TO_FP16: {
4792 RTLIB::Libcall LC =
4793 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
4794 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4795 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
4796 break;
4797 }
4798 case ISD::FP_TO_BF16: {
4799 RTLIB::Libcall LC =
4800 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
4801 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4802 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
4803 break;
4804 }
4805 case ISD::STRICT_SINT_TO_FP:
4806 case ISD::STRICT_UINT_TO_FP:
4807 case ISD::SINT_TO_FP:
4808 case ISD::UINT_TO_FP: {
4809 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4810 bool IsStrict = Node->isStrictFPOpcode();
4811 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4812 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4813 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
4814 EVT RVT = Node->getValueType(ResNo: 0);
4815 EVT NVT = EVT();
4816 SDLoc dl(Node);
4817
4818 // Even if the input is legal, no libcall may exactly match, eg. we don't
4819 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4820 // eg: i13 -> fp. Then, look for an appropriate libcall.
4821 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4822 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4823 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4824 ++t) {
4825 NVT = (MVT::SimpleValueType)t;
4826 // The source needs to big enough to hold the operand.
4827 if (NVT.bitsGE(VT: SVT))
4828 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
4829 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
4830 }
4831 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4832
4833 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4834 // Sign/zero extend the argument if the libcall takes a larger type.
4835 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
4836 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
4837 TargetLowering::MakeLibCallOptions CallOptions;
4838 CallOptions.setSExt(Signed);
4839 std::pair<SDValue, SDValue> Tmp =
4840 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
4841 Results.push_back(Elt: Tmp.first);
4842 if (IsStrict)
4843 Results.push_back(Elt: Tmp.second);
4844 break;
4845 }
4846 case ISD::FP_TO_SINT:
4847 case ISD::FP_TO_UINT:
4848 case ISD::STRICT_FP_TO_SINT:
4849 case ISD::STRICT_FP_TO_UINT: {
4850 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4851 bool IsStrict = Node->isStrictFPOpcode();
4852 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4853 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4854
4855 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
4856 EVT SVT = Op.getValueType();
4857 EVT RVT = Node->getValueType(ResNo: 0);
4858 EVT NVT = EVT();
4859 SDLoc dl(Node);
4860
4861 // Even if the result is legal, no libcall may exactly match, eg. we don't
4862 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4863 // eg: fp -> i32. Then, look for an appropriate libcall.
4864 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4865 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4866 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4867 ++IntVT) {
4868 NVT = (MVT::SimpleValueType)IntVT;
4869 // The type needs to big enough to hold the result.
4870 if (NVT.bitsGE(VT: RVT))
4871 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
4872 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
4873 }
4874 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4875
4876 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4877 TargetLowering::MakeLibCallOptions CallOptions;
4878 std::pair<SDValue, SDValue> Tmp =
4879 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
4880
4881 // Truncate the result if the libcall returns a larger type.
4882 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
4883 if (IsStrict)
4884 Results.push_back(Elt: Tmp.second);
4885 break;
4886 }
4887
4888 case ISD::FP_ROUND:
4889 case ISD::STRICT_FP_ROUND: {
4890 // X = FP_ROUND(Y, TRUNC)
4891 // TRUNC is a flag, which is always an integer that is zero or one.
4892 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4893 // is known to not change the value of Y.
4894 // We can only expand it into libcall if the TRUNC is 0.
4895 bool IsStrict = Node->isStrictFPOpcode();
4896 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
4897 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4898 EVT VT = Node->getValueType(ResNo: 0);
4899 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4900 "Unable to expand as libcall if it is not normal rounding");
4901
4902 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
4903 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4904
4905 TargetLowering::MakeLibCallOptions CallOptions;
4906 std::pair<SDValue, SDValue> Tmp =
4907 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
4908 Results.push_back(Elt: Tmp.first);
4909 if (IsStrict)
4910 Results.push_back(Elt: Tmp.second);
4911 break;
4912 }
4913 case ISD::FP_EXTEND: {
4914 Results.push_back(
4915 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
4916 RetVT: Node->getValueType(ResNo: 0)),
4917 Node, isSigned: false).first);
4918 break;
4919 }
4920 case ISD::STRICT_FP_EXTEND:
4921 case ISD::STRICT_FP_TO_FP16:
4922 case ISD::STRICT_FP_TO_BF16: {
4923 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4924 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
4925 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
4926 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
4927 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
4928 else
4929 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
4930 RetVT: Node->getValueType(ResNo: 0));
4931
4932 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4933
4934 TargetLowering::MakeLibCallOptions CallOptions;
4935 std::pair<SDValue, SDValue> Tmp =
4936 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
4937 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
4938 Results.push_back(Elt: Tmp.first);
4939 Results.push_back(Elt: Tmp.second);
4940 break;
4941 }
4942 case ISD::FSUB:
4943 case ISD::STRICT_FSUB:
4944 ExpandFPLibCall(Node, Call_F32: RTLIB::SUB_F32, Call_F64: RTLIB::SUB_F64,
4945 Call_F80: RTLIB::SUB_F80, Call_F128: RTLIB::SUB_F128,
4946 Call_PPCF128: RTLIB::SUB_PPCF128, Results);
4947 break;
4948 case ISD::SREM:
4949 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
4950 Call_I8: RTLIB::SREM_I8,
4951 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
4952 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
4953 break;
4954 case ISD::UREM:
4955 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4956 Call_I8: RTLIB::UREM_I8,
4957 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
4958 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
4959 break;
4960 case ISD::SDIV:
4961 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
4962 Call_I8: RTLIB::SDIV_I8,
4963 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
4964 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
4965 break;
4966 case ISD::UDIV:
4967 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4968 Call_I8: RTLIB::UDIV_I8,
4969 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
4970 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
4971 break;
4972 case ISD::SDIVREM:
4973 case ISD::UDIVREM:
4974 // Expand into divrem libcall
4975 ExpandDivRemLibCall(Node, Results);
4976 break;
4977 case ISD::MUL:
4978 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4979 Call_I8: RTLIB::MUL_I8,
4980 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
4981 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
4982 break;
4983 case ISD::CTLZ_ZERO_UNDEF:
4984 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
4985 default:
4986 llvm_unreachable("LibCall explicitly requested, but not available");
4987 case MVT::i32:
4988 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I32, Node, isSigned: false).first);
4989 break;
4990 case MVT::i64:
4991 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I64, Node, isSigned: false).first);
4992 break;
4993 case MVT::i128:
4994 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I128, Node, isSigned: false).first);
4995 break;
4996 }
4997 break;
4998 case ISD::RESET_FPENV: {
4999 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5000 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5001 SDValue Ptr = DAG.getIntPtrConstant(Val: -1LL, DL: dl);
5002 SDValue Chain = Node->getOperand(Num: 0);
5003 Results.push_back(
5004 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
5005 break;
5006 }
5007 case ISD::GET_FPENV_MEM: {
5008 SDValue Chain = Node->getOperand(Num: 0);
5009 SDValue EnvPtr = Node->getOperand(Num: 1);
5010 Results.push_back(
5011 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5012 break;
5013 }
5014 case ISD::SET_FPENV_MEM: {
5015 SDValue Chain = Node->getOperand(Num: 0);
5016 SDValue EnvPtr = Node->getOperand(Num: 1);
5017 Results.push_back(
5018 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5019 break;
5020 }
5021 case ISD::GET_FPMODE: {
5022 // Call fegetmode, which saves control modes into a stack slot. Then load
5023 // the value to return from the stack.
5024 EVT ModeVT = Node->getValueType(ResNo: 0);
5025 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5026 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5027 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5028 InChain: Node->getOperand(Num: 0), DLoc: dl);
5029 SDValue LdInst = DAG.getLoad(
5030 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5031 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5032 Results.push_back(Elt: LdInst);
5033 Results.push_back(Elt: LdInst.getValue(R: 1));
5034 break;
5035 }
5036 case ISD::SET_FPMODE: {
5037 // Move control modes to stack slot and then call fesetmode with the pointer
5038 // to the slot as argument.
5039 SDValue Mode = Node->getOperand(Num: 1);
5040 EVT ModeVT = Mode.getValueType();
5041 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5042 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5043 SDValue StInst = DAG.getStore(
5044 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5045 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5046 Results.push_back(
5047 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
5048 break;
5049 }
5050 case ISD::RESET_FPMODE: {
5051 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5052 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5053 // target must provide custom lowering.
5054 const DataLayout &DL = DAG.getDataLayout();
5055 EVT PtrTy = TLI.getPointerTy(DL);
5056 SDValue Mode = DAG.getConstant(Val: -1LL, DL: dl, VT: PtrTy);
5057 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5058 InChain: Node->getOperand(Num: 0), DLoc: dl));
5059 break;
5060 }
5061 }
5062
5063 // Replace the original node with the legalized result.
5064 if (!Results.empty()) {
5065 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5066 ReplaceNode(Old: Node, New: Results.data());
5067 } else
5068 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5069}
5070
5071// Determine the vector type to use in place of an original scalar element when
5072// promoting equally sized vectors.
5073static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5074 MVT EltVT, MVT NewEltVT) {
5075 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5076 MVT MidVT = OldEltsPerNewElt == 1
5077 ? NewEltVT
5078 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5079 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5080 return MidVT;
5081}
5082
5083void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5084 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5085 SmallVector<SDValue, 8> Results;
5086 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5087 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5088 Node->getOpcode() == ISD::SINT_TO_FP ||
5089 Node->getOpcode() == ISD::SETCC ||
5090 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5091 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
5092 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5093 }
5094 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5095 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5096 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5097 Node->getOpcode() == ISD::STRICT_FSETCC ||
5098 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5099 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5100 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5101 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5102 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5103 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5104 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5105 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5106 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5107 if (Node->getOpcode() == ISD::BR_CC ||
5108 Node->getOpcode() == ISD::SELECT_CC)
5109 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5110 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5111 SDLoc dl(Node);
5112 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5113 switch (Node->getOpcode()) {
5114 case ISD::CTTZ:
5115 case ISD::CTTZ_ZERO_UNDEF:
5116 case ISD::CTLZ:
5117 case ISD::CTPOP: {
5118 // Zero extend the argument unless its cttz, then use any_extend.
5119 if (Node->getOpcode() == ISD::CTTZ ||
5120 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
5121 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5122 else
5123 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5124
5125 unsigned NewOpc = Node->getOpcode();
5126 if (NewOpc == ISD::CTTZ) {
5127 // The count is the same in the promoted type except if the original
5128 // value was zero. This can be handled by setting the bit just off
5129 // the top of the original type.
5130 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5131 BitNo: OVT.getSizeInBits());
5132 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5133 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5134 NewOpc = ISD::CTTZ_ZERO_UNDEF;
5135 }
5136 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
5137 // already the correct result.
5138 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5139 if (NewOpc == ISD::CTLZ) {
5140 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5141 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5142 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5143 OVT.getSizeInBits(), DL: dl, VT: NVT));
5144 }
5145 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5146 break;
5147 }
5148 case ISD::CTLZ_ZERO_UNDEF: {
5149 // We know that the argument is unlikely to be zero, hence we can take a
5150 // different approach as compared to ISD::CTLZ
5151
5152 // Any Extend the argument
5153 auto AnyExtendedNode =
5154 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5155
5156 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5157 auto ShiftConstant = DAG.getShiftAmountConstant(
5158 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5159 auto LeftShiftResult =
5160 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5161
5162 // Perform the larger operation
5163 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5164 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5165 break;
5166 }
5167 case ISD::BITREVERSE:
5168 case ISD::BSWAP: {
5169 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5170 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5171 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5172 Tmp1 = DAG.getNode(
5173 Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5174 N2: DAG.getConstant(Val: DiffBits, DL: dl,
5175 VT: TLI.getShiftAmountTy(LHSTy: NVT, DL: DAG.getDataLayout())));
5176
5177 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5178 break;
5179 }
5180 case ISD::FP_TO_UINT:
5181 case ISD::STRICT_FP_TO_UINT:
5182 case ISD::FP_TO_SINT:
5183 case ISD::STRICT_FP_TO_SINT:
5184 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5185 break;
5186 case ISD::FP_TO_UINT_SAT:
5187 case ISD::FP_TO_SINT_SAT:
5188 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5189 break;
5190 case ISD::UINT_TO_FP:
5191 case ISD::STRICT_UINT_TO_FP:
5192 case ISD::SINT_TO_FP:
5193 case ISD::STRICT_SINT_TO_FP:
5194 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5195 break;
5196 case ISD::VAARG: {
5197 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5198 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5199
5200 unsigned TruncOp;
5201 if (OVT.isVector()) {
5202 TruncOp = ISD::BITCAST;
5203 } else {
5204 assert(OVT.isInteger()
5205 && "VAARG promotion is supported only for vectors or integer types");
5206 TruncOp = ISD::TRUNCATE;
5207 }
5208
5209 // Perform the larger operation, then convert back
5210 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5211 Align: Node->getConstantOperandVal(Num: 3));
5212 Chain = Tmp1.getValue(R: 1);
5213
5214 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5215
5216 // Modified the chain result - switch anything that used the old chain to
5217 // use the new one.
5218 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5219 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5220 if (UpdatedNodes) {
5221 UpdatedNodes->insert(X: Tmp2.getNode());
5222 UpdatedNodes->insert(X: Chain.getNode());
5223 }
5224 ReplacedNode(N: Node);
5225 break;
5226 }
5227 case ISD::MUL:
5228 case ISD::SDIV:
5229 case ISD::SREM:
5230 case ISD::UDIV:
5231 case ISD::UREM:
5232 case ISD::SMIN:
5233 case ISD::SMAX:
5234 case ISD::UMIN:
5235 case ISD::UMAX:
5236 case ISD::AND:
5237 case ISD::OR:
5238 case ISD::XOR: {
5239 unsigned ExtOp, TruncOp;
5240 if (OVT.isVector()) {
5241 ExtOp = ISD::BITCAST;
5242 TruncOp = ISD::BITCAST;
5243 } else {
5244 assert(OVT.isInteger() && "Cannot promote logic operation");
5245
5246 switch (Node->getOpcode()) {
5247 default:
5248 ExtOp = ISD::ANY_EXTEND;
5249 break;
5250 case ISD::SDIV:
5251 case ISD::SREM:
5252 case ISD::SMIN:
5253 case ISD::SMAX:
5254 ExtOp = ISD::SIGN_EXTEND;
5255 break;
5256 case ISD::UDIV:
5257 case ISD::UREM:
5258 ExtOp = ISD::ZERO_EXTEND;
5259 break;
5260 case ISD::UMIN:
5261 case ISD::UMAX:
5262 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5263 ExtOp = ISD::SIGN_EXTEND;
5264 else
5265 ExtOp = ISD::ZERO_EXTEND;
5266 break;
5267 }
5268 TruncOp = ISD::TRUNCATE;
5269 }
5270 // Promote each of the values to the new type.
5271 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5272 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5273 // Perform the larger operation, then convert back
5274 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5275 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5276 break;
5277 }
5278 case ISD::UMUL_LOHI:
5279 case ISD::SMUL_LOHI: {
5280 // Promote to a multiply in a wider integer type.
5281 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5282 : ISD::SIGN_EXTEND;
5283 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5284 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5285 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5286
5287 auto &DL = DAG.getDataLayout();
5288 unsigned OriginalSize = OVT.getScalarSizeInBits();
5289 Tmp2 = DAG.getNode(
5290 Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5291 N2: DAG.getConstant(Val: OriginalSize, DL: dl, VT: TLI.getScalarShiftAmountTy(DL, NVT)));
5292 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5293 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5294 break;
5295 }
5296 case ISD::SELECT: {
5297 unsigned ExtOp, TruncOp;
5298 if (Node->getValueType(ResNo: 0).isVector() ||
5299 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5300 ExtOp = ISD::BITCAST;
5301 TruncOp = ISD::BITCAST;
5302 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5303 ExtOp = ISD::ANY_EXTEND;
5304 TruncOp = ISD::TRUNCATE;
5305 } else {
5306 ExtOp = ISD::FP_EXTEND;
5307 TruncOp = ISD::FP_ROUND;
5308 }
5309 Tmp1 = Node->getOperand(Num: 0);
5310 // Promote each of the values to the new type.
5311 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5312 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5313 // Perform the larger operation, then round down.
5314 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5315 Tmp1->setFlags(Node->getFlags());
5316 if (TruncOp != ISD::FP_ROUND)
5317 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5318 else
5319 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5320 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
5321 Results.push_back(Elt: Tmp1);
5322 break;
5323 }
5324 case ISD::VECTOR_SHUFFLE: {
5325 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5326
5327 // Cast the two input vectors.
5328 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5329 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5330
5331 // Convert the shuffle mask to the right # elements.
5332 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5333 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5334 Results.push_back(Elt: Tmp1);
5335 break;
5336 }
5337 case ISD::VECTOR_SPLICE: {
5338 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5339 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5340 Tmp3 = DAG.getNode(Opcode: ISD::VECTOR_SPLICE, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5341 N3: Node->getOperand(Num: 2));
5342 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5343 break;
5344 }
5345 case ISD::SELECT_CC: {
5346 SDValue Cond = Node->getOperand(Num: 4);
5347 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5348 // Type of the comparison operands.
5349 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5350 assert(CVT == OVT && "not handled");
5351
5352 unsigned ExtOp = ISD::FP_EXTEND;
5353 if (NVT.isInteger()) {
5354 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5355 }
5356
5357 // Promote the comparison operands, if needed.
5358 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5359 Tmp1 = Node->getOperand(Num: 0);
5360 Tmp2 = Node->getOperand(Num: 1);
5361 } else {
5362 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5363 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5364 }
5365 // Cast the true/false operands.
5366 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5367 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5368
5369 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5370 Flags: Node->getFlags());
5371
5372 // Cast the result back to the original type.
5373 if (ExtOp != ISD::FP_EXTEND)
5374 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5375 else
5376 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5377 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5378
5379 Results.push_back(Elt: Tmp1);
5380 break;
5381 }
5382 case ISD::SETCC:
5383 case ISD::STRICT_FSETCC:
5384 case ISD::STRICT_FSETCCS: {
5385 unsigned ExtOp = ISD::FP_EXTEND;
5386 if (NVT.isInteger()) {
5387 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5388 if (isSignedIntSetCC(Code: CCCode) ||
5389 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5390 ExtOp = ISD::SIGN_EXTEND;
5391 else
5392 ExtOp = ISD::ZERO_EXTEND;
5393 }
5394 if (Node->isStrictFPOpcode()) {
5395 SDValue InChain = Node->getOperand(Num: 0);
5396 std::tie(args&: Tmp1, args: std::ignore) =
5397 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5398 std::tie(args&: Tmp2, args: std::ignore) =
5399 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5400 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5401 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5402 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
5403 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5404 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5405 Flags: Node->getFlags()));
5406 Results.push_back(Elt: Results.back().getValue(R: 1));
5407 break;
5408 }
5409 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5410 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5411 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5412 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5413 break;
5414 }
5415 case ISD::BR_CC: {
5416 unsigned ExtOp = ISD::FP_EXTEND;
5417 if (NVT.isInteger()) {
5418 ISD::CondCode CCCode =
5419 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5420 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5421 }
5422 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5423 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5424 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5425 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5426 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5427 break;
5428 }
5429 case ISD::FADD:
5430 case ISD::FSUB:
5431 case ISD::FMUL:
5432 case ISD::FDIV:
5433 case ISD::FREM:
5434 case ISD::FMINNUM:
5435 case ISD::FMAXNUM:
5436 case ISD::FMINIMUM:
5437 case ISD::FMAXIMUM:
5438 case ISD::FPOW:
5439 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5440 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5441 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5442 Flags: Node->getFlags());
5443 Results.push_back(
5444 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5445 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5446 break;
5447 case ISD::STRICT_FADD:
5448 case ISD::STRICT_FSUB:
5449 case ISD::STRICT_FMUL:
5450 case ISD::STRICT_FDIV:
5451 case ISD::STRICT_FMINNUM:
5452 case ISD::STRICT_FMAXNUM:
5453 case ISD::STRICT_FREM:
5454 case ISD::STRICT_FPOW:
5455 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5456 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5457 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5458 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5459 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5460 N2: Tmp2.getValue(R: 1));
5461 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5462 Ops: {Tmp3, Tmp1, Tmp2});
5463 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5464 Ops: {Tmp1.getValue(R: 1), Tmp1, DAG.getIntPtrConstant(Val: 0, DL: dl)});
5465 Results.push_back(Elt: Tmp1);
5466 Results.push_back(Elt: Tmp1.getValue(R: 1));
5467 break;
5468 case ISD::FMA:
5469 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5470 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5471 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5472 Results.push_back(
5473 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5474 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5475 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5476 break;
5477 case ISD::STRICT_FMA:
5478 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5479 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5480 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5481 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5482 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5483 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
5484 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5485 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
5486 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5487 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
5488 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5489 Ops: {Tmp4.getValue(R: 1), Tmp4, DAG.getIntPtrConstant(Val: 0, DL: dl)});
5490 Results.push_back(Elt: Tmp4);
5491 Results.push_back(Elt: Tmp4.getValue(R: 1));
5492 break;
5493 case ISD::FCOPYSIGN:
5494 case ISD::FLDEXP:
5495 case ISD::FPOWI: {
5496 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5497 Tmp2 = Node->getOperand(Num: 1);
5498 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5499
5500 // fcopysign doesn't change anything but the sign bit, so
5501 // (fp_round (fcopysign (fpext a), b))
5502 // is as precise as
5503 // (fp_round (fpext a))
5504 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5505 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5506 Results.push_back(
5507 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5508 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5509 break;
5510 }
5511 case ISD::STRICT_FPOWI:
5512 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5513 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5514 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5515 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
5516 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5517 Ops: {Tmp2.getValue(R: 1), Tmp2, DAG.getIntPtrConstant(Val: 0, DL: dl)});
5518 Results.push_back(Elt: Tmp3);
5519 Results.push_back(Elt: Tmp3.getValue(R: 1));
5520 break;
5521 case ISD::FFREXP: {
5522 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5523 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5524
5525 Results.push_back(
5526 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5527 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5528
5529 Results.push_back(Elt: Tmp2.getValue(R: 1));
5530 break;
5531 }
5532 case ISD::FFLOOR:
5533 case ISD::FCEIL:
5534 case ISD::FRINT:
5535 case ISD::FNEARBYINT:
5536 case ISD::FROUND:
5537 case ISD::FROUNDEVEN:
5538 case ISD::FTRUNC:
5539 case ISD::FNEG:
5540 case ISD::FSQRT:
5541 case ISD::FSIN:
5542 case ISD::FCOS:
5543 case ISD::FTAN:
5544 case ISD::FASIN:
5545 case ISD::FACOS:
5546 case ISD::FATAN:
5547 case ISD::FSINH:
5548 case ISD::FCOSH:
5549 case ISD::FTANH:
5550 case ISD::FLOG:
5551 case ISD::FLOG2:
5552 case ISD::FLOG10:
5553 case ISD::FABS:
5554 case ISD::FEXP:
5555 case ISD::FEXP2:
5556 case ISD::FEXP10:
5557 case ISD::FCANONICALIZE:
5558 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5559 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5560 Results.push_back(
5561 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5562 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5563 break;
5564 case ISD::STRICT_FFLOOR:
5565 case ISD::STRICT_FCEIL:
5566 case ISD::STRICT_FRINT:
5567 case ISD::STRICT_FNEARBYINT:
5568 case ISD::STRICT_FROUND:
5569 case ISD::STRICT_FROUNDEVEN:
5570 case ISD::STRICT_FTRUNC:
5571 case ISD::STRICT_FSQRT:
5572 case ISD::STRICT_FSIN:
5573 case ISD::STRICT_FCOS:
5574 case ISD::STRICT_FTAN:
5575 case ISD::STRICT_FASIN:
5576 case ISD::STRICT_FACOS:
5577 case ISD::STRICT_FATAN:
5578 case ISD::STRICT_FSINH:
5579 case ISD::STRICT_FCOSH:
5580 case ISD::STRICT_FTANH:
5581 case ISD::STRICT_FLOG:
5582 case ISD::STRICT_FLOG2:
5583 case ISD::STRICT_FLOG10:
5584 case ISD::STRICT_FEXP:
5585 case ISD::STRICT_FEXP2:
5586 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5587 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5588 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5589 Ops: {Tmp1.getValue(R: 1), Tmp1});
5590 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5591 Ops: {Tmp2.getValue(R: 1), Tmp2, DAG.getIntPtrConstant(Val: 0, DL: dl)});
5592 Results.push_back(Elt: Tmp3);
5593 Results.push_back(Elt: Tmp3.getValue(R: 1));
5594 break;
5595 case ISD::BUILD_VECTOR: {
5596 MVT EltVT = OVT.getVectorElementType();
5597 MVT NewEltVT = NVT.getVectorElementType();
5598
5599 // Handle bitcasts to a different vector type with the same total bit size
5600 //
5601 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
5602 // =>
5603 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
5604
5605 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5606 "Invalid promote type for build_vector");
5607 assert(NewEltVT.bitsLE(EltVT) && "not handled");
5608
5609 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5610
5611 SmallVector<SDValue, 8> NewOps;
5612 for (const SDValue &Op : Node->op_values())
5613 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
5614
5615 SDLoc SL(Node);
5616 SDValue Concat =
5617 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
5618 DL: SL, VT: NVT, Ops: NewOps);
5619 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
5620 Results.push_back(Elt: CvtVec);
5621 break;
5622 }
5623 case ISD::EXTRACT_VECTOR_ELT: {
5624 MVT EltVT = OVT.getVectorElementType();
5625 MVT NewEltVT = NVT.getVectorElementType();
5626
5627 // Handle bitcasts to a different vector type with the same total bit size.
5628 //
5629 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
5630 // =>
5631 // v4i32:castx = bitcast x:v2i64
5632 //
5633 // i64 = bitcast
5634 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
5635 // (i32 (extract_vector_elt castx, (2 * y + 1)))
5636 //
5637
5638 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5639 "Invalid promote type for extract_vector_elt");
5640 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5641
5642 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5643 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5644
5645 SDValue Idx = Node->getOperand(Num: 1);
5646 EVT IdxVT = Idx.getValueType();
5647 SDLoc SL(Node);
5648 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
5649 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
5650
5651 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
5652
5653 SmallVector<SDValue, 8> NewOps;
5654 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5655 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
5656 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
5657
5658 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
5659 N1: CastVec, N2: TmpIdx);
5660 NewOps.push_back(Elt);
5661 }
5662
5663 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
5664 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
5665 break;
5666 }
5667 case ISD::INSERT_VECTOR_ELT: {
5668 MVT EltVT = OVT.getVectorElementType();
5669 MVT NewEltVT = NVT.getVectorElementType();
5670
5671 // Handle bitcasts to a different vector type with the same total bit size
5672 //
5673 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
5674 // =>
5675 // v4i32:castx = bitcast x:v2i64
5676 // v2i32:casty = bitcast y:i64
5677 //
5678 // v2i64 = bitcast
5679 // (v4i32 insert_vector_elt
5680 // (v4i32 insert_vector_elt v4i32:castx,
5681 // (extract_vector_elt casty, 0), 2 * z),
5682 // (extract_vector_elt casty, 1), (2 * z + 1))
5683
5684 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5685 "Invalid promote type for insert_vector_elt");
5686 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5687
5688 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5689 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5690
5691 SDValue Val = Node->getOperand(Num: 1);
5692 SDValue Idx = Node->getOperand(Num: 2);
5693 EVT IdxVT = Idx.getValueType();
5694 SDLoc SL(Node);
5695
5696 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
5697 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
5698
5699 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
5700 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
5701
5702 SDValue NewVec = CastVec;
5703 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5704 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
5705 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
5706
5707 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
5708 N1: CastVal, N2: IdxOffset);
5709
5710 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
5711 N1: NewVec, N2: Elt, N3: InEltIdx);
5712 }
5713
5714 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
5715 break;
5716 }
5717 case ISD::SCALAR_TO_VECTOR: {
5718 MVT EltVT = OVT.getVectorElementType();
5719 MVT NewEltVT = NVT.getVectorElementType();
5720
5721 // Handle bitcasts to different vector type with the same total bit size.
5722 //
5723 // e.g. v2i64 = scalar_to_vector x:i64
5724 // =>
5725 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
5726 //
5727
5728 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5729 SDValue Val = Node->getOperand(Num: 0);
5730 SDLoc SL(Node);
5731
5732 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
5733 SDValue Undef = DAG.getUNDEF(VT: MidVT);
5734
5735 SmallVector<SDValue, 8> NewElts;
5736 NewElts.push_back(Elt: CastVal);
5737 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5738 NewElts.push_back(Elt: Undef);
5739
5740 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
5741 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
5742 Results.push_back(Elt: CvtVec);
5743 break;
5744 }
5745 case ISD::ATOMIC_SWAP:
5746 case ISD::ATOMIC_STORE: {
5747 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
5748 SDLoc SL(Node);
5749 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
5750 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5751 "unexpected promotion type");
5752 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5753 "unexpected atomic_swap with illegal type");
5754
5755 SDValue Op0 = AM->getBasePtr();
5756 SDValue Op1 = CastVal;
5757
5758 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
5759 // but really it should merge with ISD::STORE.
5760 if (AM->getOpcode() == ISD::ATOMIC_STORE)
5761 std::swap(a&: Op0, b&: Op1);
5762
5763 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
5764 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
5765
5766 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
5767 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
5768 Results.push_back(Elt: NewAtomic.getValue(R: 1));
5769 } else
5770 Results.push_back(Elt: NewAtomic);
5771 break;
5772 }
5773 case ISD::ATOMIC_LOAD: {
5774 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
5775 SDLoc SL(Node);
5776 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5777 "unexpected promotion type");
5778 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5779 "unexpected atomic_load with illegal type");
5780
5781 SDValue NewAtomic =
5782 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
5783 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
5784 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
5785 Results.push_back(Elt: NewAtomic.getValue(R: 1));
5786 break;
5787 }
5788 case ISD::SPLAT_VECTOR: {
5789 SDValue Scalar = Node->getOperand(Num: 0);
5790 MVT ScalarType = Scalar.getSimpleValueType();
5791 MVT NewScalarType = NVT.getVectorElementType();
5792 if (ScalarType.isInteger()) {
5793 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
5794 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5795 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5796 break;
5797 }
5798 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
5799 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5800 Results.push_back(
5801 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5802 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5803 break;
5804 }
5805 case ISD::VP_REDUCE_FADD:
5806 case ISD::VP_REDUCE_FMUL:
5807 case ISD::VP_REDUCE_FMAX:
5808 case ISD::VP_REDUCE_FMIN:
5809 case ISD::VP_REDUCE_FMAXIMUM:
5810 case ISD::VP_REDUCE_FMINIMUM:
5811 case ISD::VP_REDUCE_SEQ_FADD:
5812 Results.push_back(Elt: PromoteReduction(Node));
5813 break;
5814 }
5815
5816 // Replace the original node with the legalized result.
5817 if (!Results.empty()) {
5818 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
5819 ReplaceNode(Old: Node, New: Results.data());
5820 } else
5821 LLVM_DEBUG(dbgs() << "Could not promote node\n");
5822}
5823
5824/// This is the entry point for the file.
5825void SelectionDAG::Legalize() {
5826 AssignTopologicalOrder();
5827
5828 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5829 // Use a delete listener to remove nodes which were deleted during
5830 // legalization from LegalizeNodes. This is needed to handle the situation
5831 // where a new node is allocated by the object pool to the same address of a
5832 // previously deleted node.
5833 DAGNodeDeletedListener DeleteListener(
5834 *this,
5835 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
5836
5837 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
5838
5839 // Visit all the nodes. We start in topological order, so that we see
5840 // nodes with their original operands intact. Legalization can produce
5841 // new nodes which may themselves need to be legalized. Iterate until all
5842 // nodes have been legalized.
5843 while (true) {
5844 bool AnyLegalized = false;
5845 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
5846 --NI;
5847
5848 SDNode *N = &*NI;
5849 if (N->use_empty() && N != getRoot().getNode()) {
5850 ++NI;
5851 DeleteNode(N);
5852 continue;
5853 }
5854
5855 if (LegalizedNodes.insert(Ptr: N).second) {
5856 AnyLegalized = true;
5857 Legalizer.LegalizeOp(Node: N);
5858
5859 if (N->use_empty() && N != getRoot().getNode()) {
5860 ++NI;
5861 DeleteNode(N);
5862 }
5863 }
5864 }
5865 if (!AnyLegalized)
5866 break;
5867
5868 }
5869
5870 // Remove dead nodes now.
5871 RemoveDeadNodes();
5872}
5873
5874bool SelectionDAG::LegalizeOp(SDNode *N,
5875 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
5876 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5877 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
5878
5879 // Directly insert the node in question, and legalize it. This will recurse
5880 // as needed through operands.
5881 LegalizedNodes.insert(Ptr: N);
5882 Legalizer.LegalizeOp(Node: N);
5883
5884 return LegalizedNodes.count(Ptr: N);
5885}
5886