1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/ConstantFolding.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/CodeGen/ISDOpcodes.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineJumpTableInfo.h"
27#include "llvm/CodeGen/MachineMemOperand.h"
28#include "llvm/CodeGen/RuntimeLibcallUtil.h"
29#include "llvm/CodeGen/SelectionDAG.h"
30#include "llvm/CodeGen/SelectionDAGNodes.h"
31#include "llvm/CodeGen/TargetFrameLowering.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetSubtargetInfo.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/CodeGenTypes/MachineValueType.h"
36#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/DerivedTypes.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/Metadata.h"
42#include "llvm/IR/Type.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/Compiler.h"
45#include "llvm/Support/Debug.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Support/MathExtras.h"
48#include "llvm/Support/raw_ostream.h"
49#include "llvm/Target/TargetMachine.h"
50#include "llvm/Target/TargetOptions.h"
51#include <cassert>
52#include <cstdint>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "legalizedag"
59
60namespace {
61
62/// Keeps track of state when getting the sign of a floating-point value as an
63/// integer.
64struct FloatSignAsInt {
65 EVT FloatVT;
66 SDValue Chain;
67 SDValue FloatPtr;
68 SDValue IntPtr;
69 MachinePointerInfo IntPointerInfo;
70 MachinePointerInfo FloatPointerInfo;
71 SDValue IntValue;
72 APInt SignMask;
73 uint8_t SignBit;
74};
75
76//===----------------------------------------------------------------------===//
77/// This takes an arbitrary SelectionDAG as input and
78/// hacks on it until the target machine can handle it. This involves
79/// eliminating value sizes the machine cannot handle (promoting small sizes to
80/// large sizes or splitting up large values into small values) as well as
81/// eliminating operations the machine cannot handle.
82///
83/// This code also does a small amount of optimization and recognition of idioms
84/// as part of its processing. For example, if a target does not support a
85/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
86/// will attempt merge setcc and brc instructions into brcc's.
87class SelectionDAGLegalize {
88 const TargetMachine &TM;
89 const TargetLowering &TLI;
90 SelectionDAG &DAG;
91
92 /// The set of nodes which have already been legalized. We hold a
93 /// reference to it in order to update as necessary on node deletion.
94 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
95
96 /// A set of all the nodes updated during legalization.
97 SmallSetVector<SDNode *, 16> *UpdatedNodes;
98
99 EVT getSetCCResultType(EVT VT) const {
100 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
101 }
102
103 // Libcall insertion helpers.
104
105public:
106 SelectionDAGLegalize(SelectionDAG &DAG,
107 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
108 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112 /// Legalizes the given operation.
113 void LegalizeOp(SDNode *Node);
114
115private:
116 SDValue OptimizeFloatStore(StoreSDNode *ST);
117
118 void LegalizeLoadOps(SDNode *Node);
119 void LegalizeStoreOps(SDNode *Node);
120
121 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
122
123 /// Return a vector shuffle operation which
124 /// performs the same shuffe in terms of order or result bytes, but on a type
125 /// whose vector element type is narrower than the original shuffle type.
126 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
127 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
128 SDValue N1, SDValue N2,
129 ArrayRef<int> Mask) const;
130
131 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
132 TargetLowering::ArgListTy &&Args,
133 bool IsSigned, EVT RetVT);
134 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
135
136 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
137 SmallVectorImpl<SDValue> &Results);
138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
139 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
140 RTLIB::Libcall Call_F128,
141 RTLIB::Libcall Call_PPCF128,
142 SmallVectorImpl<SDValue> &Results);
143 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
144 RTLIB::Libcall Call_I8,
145 RTLIB::Libcall Call_I16,
146 RTLIB::Libcall Call_I32,
147 RTLIB::Libcall Call_I64,
148 RTLIB::Libcall Call_I128);
149 void ExpandArgFPLibCall(SDNode *Node,
150 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
151 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
152 RTLIB::Libcall Call_PPCF128,
153 SmallVectorImpl<SDValue> &Results);
154 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
155 RTLIB::Libcall CallI64,
156 RTLIB::Libcall CallI128);
157 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
158 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
159
160 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
161 const SDLoc &dl);
162 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
163 const SDLoc &dl, SDValue ChainIn);
164 SDValue ExpandBUILD_VECTOR(SDNode *Node);
165 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
166 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
167 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
168 SmallVectorImpl<SDValue> &Results);
169 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
170 SDValue Value) const;
171 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
172 SDValue NewIntValue) const;
173 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
174 SDValue ExpandFABS(SDNode *Node) const;
175 SDValue ExpandFNEG(SDNode *Node) const;
176 SDValue expandLdexp(SDNode *Node) const;
177 SDValue expandFrexp(SDNode *Node) const;
178
179 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
180 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
181 SmallVectorImpl<SDValue> &Results);
182 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
183 SmallVectorImpl<SDValue> &Results);
184 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
185
186 /// Implements vector reduce operation promotion.
187 ///
188 /// All vector operands are promoted to a vector type with larger element
189 /// type, and the start value is promoted to a larger scalar type. Then the
190 /// result is truncated back to the original scalar type.
191 SDValue PromoteReduction(SDNode *Node);
192
193 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
194
195 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
196 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
197 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
198 SDValue ExpandConcatVectors(SDNode *Node);
199
200 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
201 SDValue ExpandConstant(ConstantSDNode *CP);
202
203 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
204 bool ExpandNode(SDNode *Node);
205 void ConvertNodeToLibcall(SDNode *Node);
206 void PromoteNode(SDNode *Node);
207
208public:
209 // Node replacement helpers
210
211 void ReplacedNode(SDNode *N) {
212 LegalizedNodes.erase(Ptr: N);
213 if (UpdatedNodes)
214 UpdatedNodes->insert(X: N);
215 }
216
217 void ReplaceNode(SDNode *Old, SDNode *New) {
218 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
219 dbgs() << " with: "; New->dump(&DAG));
220
221 assert(Old->getNumValues() == New->getNumValues() &&
222 "Replacing one node with another that produces a different number "
223 "of values!");
224 DAG.ReplaceAllUsesWith(From: Old, To: New);
225 if (UpdatedNodes)
226 UpdatedNodes->insert(X: New);
227 ReplacedNode(N: Old);
228 }
229
230 void ReplaceNode(SDValue Old, SDValue New) {
231 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
232 dbgs() << " with: "; New->dump(&DAG));
233
234 DAG.ReplaceAllUsesWith(From: Old, To: New);
235 if (UpdatedNodes)
236 UpdatedNodes->insert(X: New.getNode());
237 ReplacedNode(N: Old.getNode());
238 }
239
240 void ReplaceNode(SDNode *Old, const SDValue *New) {
241 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
242
243 DAG.ReplaceAllUsesWith(From: Old, To: New);
244 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
245 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
246 New[i]->dump(&DAG));
247 if (UpdatedNodes)
248 UpdatedNodes->insert(X: New[i].getNode());
249 }
250 ReplacedNode(N: Old);
251 }
252
253 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
254 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
255 dbgs() << " with: "; New->dump(&DAG));
256
257 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
258 if (UpdatedNodes)
259 UpdatedNodes->insert(X: New.getNode());
260 ReplacedNode(N: Old.getNode());
261 }
262};
263
264} // end anonymous namespace
265
266// Helper function that generates an MMO that considers the alignment of the
267// stack, and the size of the stack object
268static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
269 MachineFunction &MF,
270 bool isObjectScalable) {
271 auto &MFI = MF.getFrameInfo();
272 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
273 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
274 LocationSize ObjectSize = isObjectScalable
275 ? LocationSize::beforeOrAfterPointer()
276 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
277 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
278 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
279}
280
281/// Return a vector shuffle operation which
282/// performs the same shuffle in terms of order or result bytes, but on a type
283/// whose vector element type is narrower than the original shuffle type.
284/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
285SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
286 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
287 ArrayRef<int> Mask) const {
288 unsigned NumMaskElts = VT.getVectorNumElements();
289 unsigned NumDestElts = NVT.getVectorNumElements();
290 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
291
292 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
293
294 if (NumEltsGrowth == 1)
295 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
296
297 SmallVector<int, 8> NewMask;
298 for (unsigned i = 0; i != NumMaskElts; ++i) {
299 int Idx = Mask[i];
300 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
301 if (Idx < 0)
302 NewMask.push_back(Elt: -1);
303 else
304 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
305 }
306 }
307 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
308 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
309 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
310}
311
312/// Expands the ConstantFP node to an integer constant or
313/// a load from the constant pool.
314SDValue
315SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
316 bool Extend = false;
317 SDLoc dl(CFP);
318
319 // If a FP immediate is precise when represented as a float and if the
320 // target can do an extending load from float to double, we put it into
321 // the constant pool as a float, even if it's is statically typed as a
322 // double. This shrinks FP constants and canonicalizes them for targets where
323 // an FP extending load is the same cost as a normal load (such as on the x87
324 // fp stack or PPC FP unit).
325 EVT VT = CFP->getValueType(ResNo: 0);
326 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
327 if (!UseCP) {
328 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
329 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
330 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
331 }
332
333 APFloat APF = CFP->getValueAPF();
334 EVT OrigVT = VT;
335 EVT SVT = VT;
336
337 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
338 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
339 if (!APF.isSignaling()) {
340 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
341 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
342 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
343 // Only do this if the target has a native EXTLOAD instruction from
344 // smaller type.
345 TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: OrigVT, MemVT: SVT) &&
346 TLI.ShouldShrinkFPConstant(OrigVT)) {
347 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
348 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
349 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
350 VT = SVT;
351 Extend = true;
352 }
353 }
354 }
355
356 SDValue CPIdx =
357 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
358 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
359 if (Extend) {
360 SDValue Result = DAG.getExtLoad(
361 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
362 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
363 Alignment);
364 return Result;
365 }
366 SDValue Result = DAG.getLoad(
367 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
368 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
369 return Result;
370}
371
372/// Expands the Constant node to a load from the constant pool.
373SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
374 SDLoc dl(CP);
375 EVT VT = CP->getValueType(ResNo: 0);
376 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
377 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
378 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
379 SDValue Result = DAG.getLoad(
380 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
381 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
382 return Result;
383}
384
385SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
386 SDValue Vec = Op.getOperand(i: 0);
387 SDValue Val = Op.getOperand(i: 1);
388 SDValue Idx = Op.getOperand(i: 2);
389 SDLoc dl(Op);
390
391 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
392 // SCALAR_TO_VECTOR requires that the type of the value being inserted
393 // match the element type of the vector being created, except for
394 // integers in which case the inserted value can be over width.
395 EVT EltVT = Vec.getValueType().getVectorElementType();
396 if (Val.getValueType() == EltVT ||
397 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
398 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
399 VT: Vec.getValueType(), Operand: Val);
400
401 unsigned NumElts = Vec.getValueType().getVectorNumElements();
402 // We generate a shuffle of InVec and ScVec, so the shuffle mask
403 // should be 0,1,2,3,4,5... with the appropriate element replaced with
404 // elt 0 of the RHS.
405 SmallVector<int, 8> ShufOps;
406 for (unsigned i = 0; i != NumElts; ++i)
407 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
408
409 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
410 }
411 }
412 return ExpandInsertToVectorThroughStack(Op);
413}
414
415SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
416 if (!ISD::isNormalStore(N: ST))
417 return SDValue();
418
419 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
420 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
421 // FIXME: move this to the DAG Combiner! Note that we can't regress due
422 // to phase ordering between legalized code and the dag combiner. This
423 // probably means that we need to integrate dag combiner and legalizer
424 // together.
425 // We generally can't do this one for long doubles.
426 SDValue Chain = ST->getChain();
427 SDValue Ptr = ST->getBasePtr();
428 SDValue Value = ST->getValue();
429 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
430 AAMDNodes AAInfo = ST->getAAInfo();
431 SDLoc dl(ST);
432
433 // Don't optimise TargetConstantFP
434 if (Value.getOpcode() == ISD::TargetConstantFP)
435 return SDValue();
436
437 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
438 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
439 TLI.isTypeLegal(VT: MVT::i32)) {
440 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
441 bitcastToAPInt().zextOrTrunc(width: 32),
442 DL: SDLoc(CFP), VT: MVT::i32);
443 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
444 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
445 }
446
447 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
448 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
449 // If this target supports 64-bit registers, do a single 64-bit store.
450 if (TLI.isTypeLegal(VT: MVT::i64)) {
451 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
452 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
453 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
454 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
455 }
456
457 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
458 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
459 // stores. If the target supports neither 32- nor 64-bits, this
460 // xform is certainly not worth it.
461 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
462 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
463 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
464 if (DAG.getDataLayout().isBigEndian())
465 std::swap(a&: Lo, b&: Hi);
466
467 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
468 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
469 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
470 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
471 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
472 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
473
474 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
475 }
476 }
477 }
478 return SDValue();
479}
480
481void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
482 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
483 SDValue Chain = ST->getChain();
484 SDValue Ptr = ST->getBasePtr();
485 SDLoc dl(Node);
486
487 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
488 AAMDNodes AAInfo = ST->getAAInfo();
489
490 if (!ST->isTruncatingStore()) {
491 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
492 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
493 ReplaceNode(Old: ST, New: OptStore);
494 return;
495 }
496
497 SDValue Value = ST->getValue();
498 MVT VT = Value.getSimpleValueType();
499 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
500 default: llvm_unreachable("This action is not supported yet!");
501 case TargetLowering::Legal: {
502 // If this is an unaligned store and the target doesn't support it,
503 // expand it.
504 EVT MemVT = ST->getMemoryVT();
505 const DataLayout &DL = DAG.getDataLayout();
506 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
507 MMO: *ST->getMemOperand())) {
508 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
509 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
510 ReplaceNode(Old: SDValue(ST, 0), New: Result);
511 } else
512 LLVM_DEBUG(dbgs() << "Legal store\n");
513 break;
514 }
515 case TargetLowering::Custom: {
516 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
517 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
518 if (Res && Res != SDValue(Node, 0))
519 ReplaceNode(Old: SDValue(Node, 0), New: Res);
520 return;
521 }
522 case TargetLowering::Promote: {
523 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
524 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
525 "Can only promote stores to same size type");
526 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
527 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
528 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
529 ReplaceNode(Old: SDValue(Node, 0), New: Result);
530 break;
531 }
532 }
533 return;
534 }
535
536 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
537 SDValue Value = ST->getValue();
538 EVT StVT = ST->getMemoryVT();
539 TypeSize StWidth = StVT.getSizeInBits();
540 TypeSize StSize = StVT.getStoreSizeInBits();
541 auto &DL = DAG.getDataLayout();
542
543 if (StWidth != StSize) {
544 // Promote to a byte-sized store with upper bits zero if not
545 // storing an integral number of bytes. For example, promote
546 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
547 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
548 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
549 SDValue Result =
550 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
551 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
552 ReplaceNode(Old: SDValue(Node, 0), New: Result);
553 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
554 // If not storing a power-of-2 number of bits, expand as two stores.
555 assert(!StVT.isVector() && "Unsupported truncstore!");
556 unsigned StWidthBits = StWidth.getFixedValue();
557 unsigned LogStWidth = Log2_32(Value: StWidthBits);
558 assert(LogStWidth < 32);
559 unsigned RoundWidth = 1 << LogStWidth;
560 assert(RoundWidth < StWidthBits);
561 unsigned ExtraWidth = StWidthBits - RoundWidth;
562 assert(ExtraWidth < RoundWidth);
563 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
564 "Store size not an integral number of bytes!");
565 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
566 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
567 SDValue Lo, Hi;
568 unsigned IncrementSize;
569
570 if (DL.isLittleEndian()) {
571 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
572 // Store the bottom RoundWidth bits.
573 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
574 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
575
576 // Store the remaining ExtraWidth bits.
577 IncrementSize = RoundWidth / 8;
578 Ptr =
579 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
580 Hi = DAG.getNode(
581 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
582 N2: DAG.getConstant(Val: RoundWidth, DL: dl,
583 VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL)));
584 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
585 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
586 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
587 } else {
588 // Big endian - avoid unaligned stores.
589 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
590 // Store the top RoundWidth bits.
591 Hi = DAG.getNode(
592 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
593 N2: DAG.getConstant(Val: ExtraWidth, DL: dl,
594 VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL)));
595 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
596 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
597
598 // Store the remaining ExtraWidth bits.
599 IncrementSize = RoundWidth / 8;
600 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
601 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
602 VT: Ptr.getValueType()));
603 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
604 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
605 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
606 }
607
608 // The order of the stores doesn't matter.
609 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
610 ReplaceNode(Old: SDValue(Node, 0), New: Result);
611 } else {
612 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT)) {
613 default: llvm_unreachable("This action is not supported yet!");
614 case TargetLowering::Legal: {
615 EVT MemVT = ST->getMemoryVT();
616 // If this is an unaligned store and the target doesn't support it,
617 // expand it.
618 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
619 MMO: *ST->getMemOperand())) {
620 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
621 ReplaceNode(Old: SDValue(ST, 0), New: Result);
622 }
623 break;
624 }
625 case TargetLowering::Custom: {
626 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
627 if (Res && Res != SDValue(Node, 0))
628 ReplaceNode(Old: SDValue(Node, 0), New: Res);
629 return;
630 }
631 case TargetLowering::Expand:
632 assert(!StVT.isVector() &&
633 "Vector Stores are handled in LegalizeVectorOps");
634
635 SDValue Result;
636
637 // TRUNCSTORE:i16 i32 -> STORE i16
638 if (TLI.isTypeLegal(VT: StVT)) {
639 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
640 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
641 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
642 } else {
643 // The in-memory type isn't legal. Truncate to the type it would promote
644 // to, and then do a truncstore.
645 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
646 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
647 Operand: Value);
648 Result = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
649 SVT: StVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
650 }
651
652 ReplaceNode(Old: SDValue(Node, 0), New: Result);
653 break;
654 }
655 }
656}
657
658void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
659 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
660 SDValue Chain = LD->getChain(); // The chain.
661 SDValue Ptr = LD->getBasePtr(); // The base pointer.
662 SDValue Value; // The value returned by the load op.
663 SDLoc dl(Node);
664
665 ISD::LoadExtType ExtType = LD->getExtensionType();
666 if (ExtType == ISD::NON_EXTLOAD) {
667 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
668 MVT VT = Node->getSimpleValueType(ResNo: 0);
669 SDValue RVal = SDValue(Node, 0);
670 SDValue RChain = SDValue(Node, 1);
671
672 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
673 default: llvm_unreachable("This action is not supported yet!");
674 case TargetLowering::Legal: {
675 EVT MemVT = LD->getMemoryVT();
676 const DataLayout &DL = DAG.getDataLayout();
677 // If this is an unaligned load and the target doesn't support it,
678 // expand it.
679 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
680 MMO: *LD->getMemOperand())) {
681 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
682 }
683 break;
684 }
685 case TargetLowering::Custom:
686 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
687 RVal = Res;
688 RChain = Res.getValue(R: 1);
689 }
690 break;
691
692 case TargetLowering::Promote: {
693 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
694 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
695 "Can only promote loads to same size type");
696
697 // If the range metadata type does not match the legalized memory
698 // operation type, remove the range metadata.
699 if (const MDNode *MD = LD->getRanges()) {
700 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
701 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
702 !NVT.isInteger())
703 LD->getMemOperand()->clearRanges();
704 }
705 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
706 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
707 RChain = Res.getValue(R: 1);
708 break;
709 }
710 }
711 if (RChain.getNode() != Node) {
712 assert(RVal.getNode() != Node && "Load must be completely replaced");
713 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
714 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
715 if (UpdatedNodes) {
716 UpdatedNodes->insert(X: RVal.getNode());
717 UpdatedNodes->insert(X: RChain.getNode());
718 }
719 ReplacedNode(N: Node);
720 }
721 return;
722 }
723
724 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
725 EVT SrcVT = LD->getMemoryVT();
726 TypeSize SrcWidth = SrcVT.getSizeInBits();
727 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
728 AAMDNodes AAInfo = LD->getAAInfo();
729
730 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
731 // Some targets pretend to have an i1 loading operation, and actually
732 // load an i8. This trick is correct for ZEXTLOAD because the top 7
733 // bits are guaranteed to be zero; it helps the optimizers understand
734 // that these bits are zero. It is also useful for EXTLOAD, since it
735 // tells the optimizers that those bits are undefined. It would be
736 // nice to have an effective generic way of getting these benefits...
737 // Until such a way is found, don't insist on promoting i1 here.
738 (SrcVT != MVT::i1 ||
739 TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1) ==
740 TargetLowering::Promote)) {
741 // Promote to a byte-sized load if not loading an integral number of
742 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
743 unsigned NewWidth = SrcVT.getStoreSizeInBits();
744 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
745 SDValue Ch;
746
747 // The extra bits are guaranteed to be zero, since we stored them that
748 // way. A zext load from NVT thus automatically gives zext from SrcVT.
749
750 ISD::LoadExtType NewExtType =
751 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
752
753 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
754 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
755 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
756
757 Ch = Result.getValue(R: 1); // The chain.
758
759 if (ExtType == ISD::SEXTLOAD)
760 // Having the top bits zero doesn't help when sign extending.
761 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
762 VT: Result.getValueType(),
763 N1: Result, N2: DAG.getValueType(SrcVT));
764 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
765 // All the top bits are guaranteed to be zero - inform the optimizers.
766 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
767 VT: Result.getValueType(), N1: Result,
768 N2: DAG.getValueType(SrcVT));
769
770 Value = Result;
771 Chain = Ch;
772 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
773 // If not loading a power-of-2 number of bits, expand as two loads.
774 assert(!SrcVT.isVector() && "Unsupported extload!");
775 unsigned SrcWidthBits = SrcWidth.getFixedValue();
776 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
777 assert(LogSrcWidth < 32);
778 unsigned RoundWidth = 1 << LogSrcWidth;
779 assert(RoundWidth < SrcWidthBits);
780 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
781 assert(ExtraWidth < RoundWidth);
782 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
783 "Load size not an integral number of bytes!");
784 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
785 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
786 SDValue Lo, Hi, Ch;
787 unsigned IncrementSize;
788 auto &DL = DAG.getDataLayout();
789
790 if (DL.isLittleEndian()) {
791 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
792 // Load the bottom RoundWidth bits.
793 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
794 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
795 MMOFlags, AAInfo);
796
797 // Load the remaining ExtraWidth bits.
798 IncrementSize = RoundWidth / 8;
799 Ptr =
800 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
801 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
802 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
803 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
804
805 // Build a factor node to remember that this load is independent of
806 // the other one.
807 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
808 N2: Hi.getValue(R: 1));
809
810 // Move the top bits to the right place.
811 Hi = DAG.getNode(
812 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
813 N2: DAG.getConstant(Val: RoundWidth, DL: dl,
814 VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL)));
815
816 // Join the hi and lo parts.
817 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
818 } else {
819 // Big endian - avoid unaligned loads.
820 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
821 // Load the top RoundWidth bits.
822 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
823 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
824 MMOFlags, AAInfo);
825
826 // Load the remaining ExtraWidth bits.
827 IncrementSize = RoundWidth / 8;
828 Ptr =
829 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
830 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
831 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
832 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
833
834 // Build a factor node to remember that this load is independent of
835 // the other one.
836 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
837 N2: Hi.getValue(R: 1));
838
839 // Move the top bits to the right place.
840 Hi = DAG.getNode(
841 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
842 N2: DAG.getConstant(Val: ExtraWidth, DL: dl,
843 VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL)));
844
845 // Join the hi and lo parts.
846 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
847 }
848
849 Chain = Ch;
850 } else {
851 bool isCustom = false;
852 switch (TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0),
853 MemVT: SrcVT.getSimpleVT())) {
854 default: llvm_unreachable("This action is not supported yet!");
855 case TargetLowering::Custom:
856 isCustom = true;
857 [[fallthrough]];
858 case TargetLowering::Legal:
859 Value = SDValue(Node, 0);
860 Chain = SDValue(Node, 1);
861
862 if (isCustom) {
863 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
864 Value = Res;
865 Chain = Res.getValue(R: 1);
866 }
867 } else {
868 // If this is an unaligned load and the target doesn't support it,
869 // expand it.
870 EVT MemVT = LD->getMemoryVT();
871 const DataLayout &DL = DAG.getDataLayout();
872 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
873 MMO: *LD->getMemOperand())) {
874 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
875 }
876 }
877 break;
878
879 case TargetLowering::Expand: {
880 EVT DestVT = Node->getValueType(ResNo: 0);
881 if (!TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SrcVT)) {
882 // If the source type is not legal, see if there is a legal extload to
883 // an intermediate type that we can then extend further.
884 EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT());
885 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
886 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
887 TLI.isLoadExtLegal(ExtType, ValVT: LoadVT, MemVT: SrcVT))) {
888 // If we are loading a legal type, this is a non-extload followed by a
889 // full extend.
890 ISD::LoadExtType MidExtType =
891 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
892
893 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
894 MemVT: SrcVT, MMO: LD->getMemOperand());
895 unsigned ExtendOp =
896 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
897 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
898 Chain = Load.getValue(R: 1);
899 break;
900 }
901
902 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
903 // normal undefined upper bits behavior to allow using an in-reg extend
904 // with the illegal FP type, so load as an integer and do the
905 // from-integer conversion.
906 EVT SVT = SrcVT.getScalarType();
907 if (SVT == MVT::f16 || SVT == MVT::bf16) {
908 EVT ISrcVT = SrcVT.changeTypeToInteger();
909 EVT IDestVT = DestVT.changeTypeToInteger();
910 EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT());
911
912 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
913 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
914 Value =
915 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
916 DL: dl, VT: DestVT, Operand: Result);
917 Chain = Result.getValue(R: 1);
918 break;
919 }
920 }
921
922 assert(!SrcVT.isVector() &&
923 "Vector Loads are handled in LegalizeVectorOps");
924
925 // FIXME: This does not work for vectors on most targets. Sign-
926 // and zero-extend operations are currently folded into extending
927 // loads, whether they are legal or not, and then we end up here
928 // without any support for legalizing them.
929 assert(ExtType != ISD::EXTLOAD &&
930 "EXTLOAD should always be supported!");
931 // Turn the unsupported load into an EXTLOAD followed by an
932 // explicit zero/sign extend inreg.
933 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
934 VT: Node->getValueType(ResNo: 0),
935 Chain, Ptr, MemVT: SrcVT,
936 MMO: LD->getMemOperand());
937 SDValue ValRes;
938 if (ExtType == ISD::SEXTLOAD)
939 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
940 VT: Result.getValueType(),
941 N1: Result, N2: DAG.getValueType(SrcVT));
942 else
943 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
944 Value = ValRes;
945 Chain = Result.getValue(R: 1);
946 break;
947 }
948 }
949 }
950
951 // Since loads produce two values, make sure to remember that we legalized
952 // both of them.
953 if (Chain.getNode() != Node) {
954 assert(Value.getNode() != Node && "Load must be completely replaced");
955 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
956 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
957 if (UpdatedNodes) {
958 UpdatedNodes->insert(X: Value.getNode());
959 UpdatedNodes->insert(X: Chain.getNode());
960 }
961 ReplacedNode(N: Node);
962 }
963}
964
965/// Return a legal replacement for the given operation, with all legal operands.
966void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
967 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
968
969 // Allow illegal target nodes and illegal registers.
970 if (Node->getOpcode() == ISD::TargetConstant ||
971 Node->getOpcode() == ISD::Register)
972 return;
973
974#ifndef NDEBUG
975 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
976 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
977 TargetLowering::TypeLegal &&
978 "Unexpected illegal type!");
979
980 for (const SDValue &Op : Node->op_values())
981 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
982 TargetLowering::TypeLegal ||
983 Op.getOpcode() == ISD::TargetConstant ||
984 Op.getOpcode() == ISD::Register) &&
985 "Unexpected illegal type!");
986#endif
987
988 // Figure out the correct action; the way to query this varies by opcode
989 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
990 bool SimpleFinishLegalizing = true;
991 switch (Node->getOpcode()) {
992 // TODO: Currently, POISON is being lowered to UNDEF here. However, there is
993 // an open concern that this transformation may not be ideal, as targets
994 // should ideally handle POISON directly. Changing this behavior would require
995 // adding support for POISON in TableGen, which is a large change.
996 // Additionally, many existing test cases rely on the current behavior (e.g.,
997 // llvm/test/CodeGen/PowerPC/vec_shuffle.ll). A broader discussion and
998 // incremental changes might be needed to properly
999 // support POISON without breaking existing targets and tests.
1000 case ISD::POISON: {
1001 SDValue UndefNode = DAG.getUNDEF(VT: Node->getValueType(ResNo: 0));
1002 ReplaceNode(Old: Node, New: UndefNode.getNode());
1003 break;
1004 }
1005 case ISD::INTRINSIC_W_CHAIN:
1006 case ISD::INTRINSIC_WO_CHAIN:
1007 case ISD::INTRINSIC_VOID:
1008 case ISD::STACKSAVE:
1009 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1010 break;
1011 case ISD::GET_DYNAMIC_AREA_OFFSET:
1012 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1013 VT: Node->getValueType(ResNo: 0));
1014 break;
1015 case ISD::VAARG:
1016 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1017 VT: Node->getValueType(ResNo: 0));
1018 if (Action != TargetLowering::Promote)
1019 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1020 break;
1021 case ISD::SET_FPENV:
1022 case ISD::SET_FPMODE:
1023 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1024 VT: Node->getOperand(Num: 1).getValueType());
1025 break;
1026 case ISD::FP_TO_FP16:
1027 case ISD::FP_TO_BF16:
1028 case ISD::SINT_TO_FP:
1029 case ISD::UINT_TO_FP:
1030 case ISD::EXTRACT_VECTOR_ELT:
1031 case ISD::LROUND:
1032 case ISD::LLROUND:
1033 case ISD::LRINT:
1034 case ISD::LLRINT:
1035 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1036 VT: Node->getOperand(Num: 0).getValueType());
1037 break;
1038 case ISD::STRICT_FP_TO_FP16:
1039 case ISD::STRICT_FP_TO_BF16:
1040 case ISD::STRICT_SINT_TO_FP:
1041 case ISD::STRICT_UINT_TO_FP:
1042 case ISD::STRICT_LRINT:
1043 case ISD::STRICT_LLRINT:
1044 case ISD::STRICT_LROUND:
1045 case ISD::STRICT_LLROUND:
1046 // These pseudo-ops are the same as the other STRICT_ ops except
1047 // they are registered with setOperationAction() using the input type
1048 // instead of the output type.
1049 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1050 VT: Node->getOperand(Num: 1).getValueType());
1051 break;
1052 case ISD::SIGN_EXTEND_INREG: {
1053 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1054 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1055 break;
1056 }
1057 case ISD::ATOMIC_STORE:
1058 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1059 VT: Node->getOperand(Num: 1).getValueType());
1060 break;
1061 case ISD::SELECT_CC:
1062 case ISD::STRICT_FSETCC:
1063 case ISD::STRICT_FSETCCS:
1064 case ISD::SETCC:
1065 case ISD::SETCCCARRY:
1066 case ISD::VP_SETCC:
1067 case ISD::BR_CC: {
1068 unsigned Opc = Node->getOpcode();
1069 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1070 : Opc == ISD::STRICT_FSETCC ? 3
1071 : Opc == ISD::STRICT_FSETCCS ? 3
1072 : Opc == ISD::SETCCCARRY ? 3
1073 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1074 : 1;
1075 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1076 : Opc == ISD::STRICT_FSETCC ? 1
1077 : Opc == ISD::STRICT_FSETCCS ? 1
1078 : 0;
1079 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1080 ISD::CondCode CCCode =
1081 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1082 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1083 if (Action == TargetLowering::Legal) {
1084 if (Node->getOpcode() == ISD::SELECT_CC)
1085 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1086 VT: Node->getValueType(ResNo: 0));
1087 else
1088 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1089 }
1090 break;
1091 }
1092 case ISD::LOAD:
1093 case ISD::STORE:
1094 // FIXME: Model these properly. LOAD and STORE are complicated, and
1095 // STORE expects the unlegalized operand in some cases.
1096 SimpleFinishLegalizing = false;
1097 break;
1098 case ISD::CALLSEQ_START:
1099 case ISD::CALLSEQ_END:
1100 // FIXME: This shouldn't be necessary. These nodes have special properties
1101 // dealing with the recursive nature of legalization. Removing this
1102 // special case should be done as part of making LegalizeDAG non-recursive.
1103 SimpleFinishLegalizing = false;
1104 break;
1105 case ISD::EXTRACT_ELEMENT:
1106 case ISD::GET_ROUNDING:
1107 case ISD::MERGE_VALUES:
1108 case ISD::EH_RETURN:
1109 case ISD::FRAME_TO_ARGS_OFFSET:
1110 case ISD::EH_DWARF_CFA:
1111 case ISD::EH_SJLJ_SETJMP:
1112 case ISD::EH_SJLJ_LONGJMP:
1113 case ISD::EH_SJLJ_SETUP_DISPATCH:
1114 // These operations lie about being legal: when they claim to be legal,
1115 // they should actually be expanded.
1116 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1117 if (Action == TargetLowering::Legal)
1118 Action = TargetLowering::Expand;
1119 break;
1120 case ISD::INIT_TRAMPOLINE:
1121 case ISD::ADJUST_TRAMPOLINE:
1122 case ISD::FRAMEADDR:
1123 case ISD::RETURNADDR:
1124 case ISD::ADDROFRETURNADDR:
1125 case ISD::SPONENTRY:
1126 // These operations lie about being legal: when they claim to be legal,
1127 // they should actually be custom-lowered.
1128 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1129 if (Action == TargetLowering::Legal)
1130 Action = TargetLowering::Custom;
1131 break;
1132 case ISD::CLEAR_CACHE:
1133 // This operation is typically going to be LibCall unless the target wants
1134 // something differrent.
1135 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1136 break;
1137 case ISD::READCYCLECOUNTER:
1138 case ISD::READSTEADYCOUNTER:
1139 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1140 // legalization might have expanded that to several smaller types.
1141 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1142 break;
1143 case ISD::READ_REGISTER:
1144 case ISD::WRITE_REGISTER:
1145 // Named register is legal in the DAG, but blocked by register name
1146 // selection if not implemented by target (to chose the correct register)
1147 // They'll be converted to Copy(To/From)Reg.
1148 Action = TargetLowering::Legal;
1149 break;
1150 case ISD::UBSANTRAP:
1151 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1152 if (Action == TargetLowering::Expand) {
1153 // replace ISD::UBSANTRAP with ISD::TRAP
1154 SDValue NewVal;
1155 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1156 N: Node->getOperand(Num: 0));
1157 ReplaceNode(Old: Node, New: NewVal.getNode());
1158 LegalizeOp(Node: NewVal.getNode());
1159 return;
1160 }
1161 break;
1162 case ISD::DEBUGTRAP:
1163 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1164 if (Action == TargetLowering::Expand) {
1165 // replace ISD::DEBUGTRAP with ISD::TRAP
1166 SDValue NewVal;
1167 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1168 N: Node->getOperand(Num: 0));
1169 ReplaceNode(Old: Node, New: NewVal.getNode());
1170 LegalizeOp(Node: NewVal.getNode());
1171 return;
1172 }
1173 break;
1174 case ISD::SADDSAT:
1175 case ISD::UADDSAT:
1176 case ISD::SSUBSAT:
1177 case ISD::USUBSAT:
1178 case ISD::SSHLSAT:
1179 case ISD::USHLSAT:
1180 case ISD::SCMP:
1181 case ISD::UCMP:
1182 case ISD::FP_TO_SINT_SAT:
1183 case ISD::FP_TO_UINT_SAT:
1184 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1185 break;
1186 case ISD::SMULFIX:
1187 case ISD::SMULFIXSAT:
1188 case ISD::UMULFIX:
1189 case ISD::UMULFIXSAT:
1190 case ISD::SDIVFIX:
1191 case ISD::SDIVFIXSAT:
1192 case ISD::UDIVFIX:
1193 case ISD::UDIVFIXSAT: {
1194 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1195 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1196 VT: Node->getValueType(ResNo: 0), Scale);
1197 break;
1198 }
1199 case ISD::MSCATTER:
1200 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1201 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1202 break;
1203 case ISD::MSTORE:
1204 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1205 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1206 break;
1207 case ISD::VP_SCATTER:
1208 Action = TLI.getOperationAction(
1209 Op: Node->getOpcode(),
1210 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1211 break;
1212 case ISD::VP_STORE:
1213 Action = TLI.getOperationAction(
1214 Op: Node->getOpcode(),
1215 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1216 break;
1217 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1218 Action = TLI.getOperationAction(
1219 Op: Node->getOpcode(),
1220 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1221 break;
1222 case ISD::VECREDUCE_FADD:
1223 case ISD::VECREDUCE_FMUL:
1224 case ISD::VECREDUCE_ADD:
1225 case ISD::VECREDUCE_MUL:
1226 case ISD::VECREDUCE_AND:
1227 case ISD::VECREDUCE_OR:
1228 case ISD::VECREDUCE_XOR:
1229 case ISD::VECREDUCE_SMAX:
1230 case ISD::VECREDUCE_SMIN:
1231 case ISD::VECREDUCE_UMAX:
1232 case ISD::VECREDUCE_UMIN:
1233 case ISD::VECREDUCE_FMAX:
1234 case ISD::VECREDUCE_FMIN:
1235 case ISD::VECREDUCE_FMAXIMUM:
1236 case ISD::VECREDUCE_FMINIMUM:
1237 case ISD::IS_FPCLASS:
1238 Action = TLI.getOperationAction(
1239 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1240 break;
1241 case ISD::VECREDUCE_SEQ_FADD:
1242 case ISD::VECREDUCE_SEQ_FMUL:
1243 case ISD::VP_REDUCE_FADD:
1244 case ISD::VP_REDUCE_FMUL:
1245 case ISD::VP_REDUCE_ADD:
1246 case ISD::VP_REDUCE_MUL:
1247 case ISD::VP_REDUCE_AND:
1248 case ISD::VP_REDUCE_OR:
1249 case ISD::VP_REDUCE_XOR:
1250 case ISD::VP_REDUCE_SMAX:
1251 case ISD::VP_REDUCE_SMIN:
1252 case ISD::VP_REDUCE_UMAX:
1253 case ISD::VP_REDUCE_UMIN:
1254 case ISD::VP_REDUCE_FMAX:
1255 case ISD::VP_REDUCE_FMIN:
1256 case ISD::VP_REDUCE_FMAXIMUM:
1257 case ISD::VP_REDUCE_FMINIMUM:
1258 case ISD::VP_REDUCE_SEQ_FADD:
1259 case ISD::VP_REDUCE_SEQ_FMUL:
1260 Action = TLI.getOperationAction(
1261 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1262 break;
1263 case ISD::VP_CTTZ_ELTS:
1264 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1265 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1266 VT: Node->getOperand(Num: 0).getValueType());
1267 break;
1268 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1269 Action = TLI.getOperationAction(
1270 Op: Node->getOpcode(),
1271 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1272 break;
1273 default:
1274 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1275 Action = TLI.getCustomOperationAction(Op&: *Node);
1276 } else {
1277 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1278 }
1279 break;
1280 }
1281
1282 if (SimpleFinishLegalizing) {
1283 SDNode *NewNode = Node;
1284 switch (Node->getOpcode()) {
1285 default: break;
1286 case ISD::SHL:
1287 case ISD::SRL:
1288 case ISD::SRA:
1289 case ISD::ROTL:
1290 case ISD::ROTR: {
1291 // Legalizing shifts/rotates requires adjusting the shift amount
1292 // to the appropriate width.
1293 SDValue Op0 = Node->getOperand(Num: 0);
1294 SDValue Op1 = Node->getOperand(Num: 1);
1295 if (!Op1.getValueType().isVector()) {
1296 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1297 // The getShiftAmountOperand() may create a new operand node or
1298 // return the existing one. If new operand is created we need
1299 // to update the parent node.
1300 // Do not try to legalize SAO here! It will be automatically legalized
1301 // in the next round.
1302 if (SAO != Op1)
1303 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1304 }
1305 }
1306 break;
1307 case ISD::FSHL:
1308 case ISD::FSHR:
1309 case ISD::SRL_PARTS:
1310 case ISD::SRA_PARTS:
1311 case ISD::SHL_PARTS: {
1312 // Legalizing shifts/rotates requires adjusting the shift amount
1313 // to the appropriate width.
1314 SDValue Op0 = Node->getOperand(Num: 0);
1315 SDValue Op1 = Node->getOperand(Num: 1);
1316 SDValue Op2 = Node->getOperand(Num: 2);
1317 if (!Op2.getValueType().isVector()) {
1318 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1319 // The getShiftAmountOperand() may create a new operand node or
1320 // return the existing one. If new operand is created we need
1321 // to update the parent node.
1322 if (SAO != Op2)
1323 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1324 }
1325 break;
1326 }
1327 }
1328
1329 if (NewNode != Node) {
1330 ReplaceNode(Old: Node, New: NewNode);
1331 Node = NewNode;
1332 }
1333 switch (Action) {
1334 case TargetLowering::Legal:
1335 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1336 return;
1337 case TargetLowering::Custom:
1338 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1339 // FIXME: The handling for custom lowering with multiple results is
1340 // a complete mess.
1341 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1342 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1343 return;
1344
1345 if (Node->getNumValues() == 1) {
1346 // Verify the new types match the original. Glue is waived because
1347 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1348 assert((Res.getValueType() == Node->getValueType(0) ||
1349 Node->getValueType(0) == MVT::Glue) &&
1350 "Type mismatch for custom legalized operation");
1351 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1352 // We can just directly replace this node with the lowered value.
1353 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1354 return;
1355 }
1356
1357 SmallVector<SDValue, 8> ResultVals;
1358 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1359 // Verify the new types match the original. Glue is waived because
1360 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1361 assert((Res->getValueType(i) == Node->getValueType(i) ||
1362 Node->getValueType(i) == MVT::Glue) &&
1363 "Type mismatch for custom legalized operation");
1364 ResultVals.push_back(Elt: Res.getValue(R: i));
1365 }
1366 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1367 ReplaceNode(Old: Node, New: ResultVals.data());
1368 return;
1369 }
1370 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1371 [[fallthrough]];
1372 case TargetLowering::Expand:
1373 if (ExpandNode(Node))
1374 return;
1375 [[fallthrough]];
1376 case TargetLowering::LibCall:
1377 ConvertNodeToLibcall(Node);
1378 return;
1379 case TargetLowering::Promote:
1380 PromoteNode(Node);
1381 return;
1382 }
1383 }
1384
1385 switch (Node->getOpcode()) {
1386 default:
1387#ifndef NDEBUG
1388 dbgs() << "NODE: ";
1389 Node->dump( &DAG);
1390 dbgs() << "\n";
1391#endif
1392 llvm_unreachable("Do not know how to legalize this operator!");
1393
1394 case ISD::CALLSEQ_START:
1395 case ISD::CALLSEQ_END:
1396 break;
1397 case ISD::LOAD:
1398 return LegalizeLoadOps(Node);
1399 case ISD::STORE:
1400 return LegalizeStoreOps(Node);
1401 }
1402}
1403
1404SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1405 SDValue Vec = Op.getOperand(i: 0);
1406 SDValue Idx = Op.getOperand(i: 1);
1407 SDLoc dl(Op);
1408
1409 // Before we generate a new store to a temporary stack slot, see if there is
1410 // already one that we can use. There often is because when we scalarize
1411 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1412 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1413 // the vector. If all are expanded here, we don't want one store per vector
1414 // element.
1415
1416 // Caches for hasPredecessorHelper
1417 SmallPtrSet<const SDNode *, 32> Visited;
1418 SmallVector<const SDNode *, 16> Worklist;
1419 Visited.insert(Ptr: Op.getNode());
1420 Worklist.push_back(Elt: Idx.getNode());
1421 SDValue StackPtr, Ch;
1422 for (SDNode *User : Vec.getNode()->users()) {
1423 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1424 if (ST->isIndexed() || ST->isTruncatingStore() ||
1425 ST->getValue() != Vec)
1426 continue;
1427
1428 // Make sure that nothing else could have stored into the destination of
1429 // this store.
1430 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1431 continue;
1432
1433 // If the index is dependent on the store we will introduce a cycle when
1434 // creating the load (the load uses the index, and by replacing the chain
1435 // we will make the index dependent on the load). Also, the store might be
1436 // dependent on the extractelement and introduce a cycle when creating
1437 // the load.
1438 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1439 ST->hasPredecessor(N: Op.getNode()))
1440 continue;
1441
1442 StackPtr = ST->getBasePtr();
1443 Ch = SDValue(ST, 0);
1444 break;
1445 }
1446 }
1447
1448 EVT VecVT = Vec.getValueType();
1449
1450 if (!Ch.getNode()) {
1451 // Store the value to a temporary stack slot, then LOAD the returned part.
1452 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1453 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1454 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1455 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1456 }
1457
1458 SDValue NewLoad;
1459 Align ElementAlignment =
1460 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1461 b: DAG.getDataLayout().getPrefTypeAlign(
1462 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1463
1464 if (Op.getValueType().isVector()) {
1465 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1466 SubVecVT: Op.getValueType(), Index: Idx);
1467 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1468 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1469 } else {
1470 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1471 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1472 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1473 Alignment: ElementAlignment);
1474 }
1475
1476 // Replace the chain going out of the store, by the one out of the load.
1477 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1478
1479 // We introduced a cycle though, so update the loads operands, making sure
1480 // to use the original store's chain as an incoming chain.
1481 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1482 NewLoadOperands[0] = Ch;
1483 NewLoad =
1484 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1485 return NewLoad;
1486}
1487
1488SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1489 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1490
1491 SDValue Vec = Op.getOperand(i: 0);
1492 SDValue Part = Op.getOperand(i: 1);
1493 SDValue Idx = Op.getOperand(i: 2);
1494 SDLoc dl(Op);
1495
1496 // Store the value to a temporary stack slot, then LOAD the returned part.
1497 EVT VecVT = Vec.getValueType();
1498 EVT PartVT = Part.getValueType();
1499 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1500 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1501 MachinePointerInfo PtrInfo =
1502 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1503
1504 // First store the whole vector.
1505 Align BaseVecAlignment =
1506 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1507 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1508 Alignment: BaseVecAlignment);
1509
1510 // Freeze the index so we don't poison the clamping code we're about to emit.
1511 Idx = DAG.getFreeze(V: Idx);
1512
1513 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1514 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1515
1516 // Then store the inserted part.
1517 if (PartVT.isVector()) {
1518 SDValue SubStackPtr =
1519 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1520
1521 // Store the subvector.
1522 Ch = DAG.getStore(
1523 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1524 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1525 Alignment: PartAlignment);
1526 } else {
1527 SDValue SubStackPtr =
1528 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1529
1530 // Store the scalar value.
1531 Ch = DAG.getTruncStore(
1532 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1533 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1534 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1535 }
1536
1537 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1538 "ElementAlignment does not match!");
1539
1540 // Finally, load the updated vector.
1541 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1542 Alignment: BaseVecAlignment);
1543}
1544
1545SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1546 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1547 SDLoc DL(Node);
1548 SmallVector<SDValue, 16> Ops;
1549 unsigned NumOperands = Node->getNumOperands();
1550 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1551 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1552 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1553 EVT ElementValueType = TLI.getTypeToTransformTo(
1554 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1555 for (unsigned I = 0; I < NumOperands; ++I) {
1556 SDValue SubOp = Node->getOperand(Num: I);
1557 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1558 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1559 N1: SubOp,
1560 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1561 }
1562 }
1563 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1564}
1565
1566SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1567 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1568 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1569 "Unexpected opcode!");
1570
1571 // We can't handle this case efficiently. Allocate a sufficiently
1572 // aligned object on the stack, store each operand into it, then load
1573 // the result as a vector.
1574 // Create the stack frame object.
1575 EVT VT = Node->getValueType(ResNo: 0);
1576 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1577 : Node->getOperand(Num: 0).getValueType();
1578 SDLoc dl(Node);
1579 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1580 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1581 MachinePointerInfo PtrInfo =
1582 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1583
1584 // Emit a store of each element to the stack slot.
1585 SmallVector<SDValue, 8> Stores;
1586 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1587 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1588
1589 // If the destination vector element type of a BUILD_VECTOR is narrower than
1590 // the source element type, only store the bits necessary.
1591 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1592 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1593
1594 // Store (in the right endianness) the elements to memory.
1595 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1596 // Ignore undef elements.
1597 if (Node->getOperand(Num: i).isUndef()) continue;
1598
1599 unsigned Offset = TypeByteSize*i;
1600
1601 SDValue Idx =
1602 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1603
1604 if (Truncate)
1605 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1606 Val: Node->getOperand(Num: i), Ptr: Idx,
1607 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1608 else
1609 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1610 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1611 }
1612
1613 SDValue StoreChain;
1614 if (!Stores.empty()) // Not all undef elements?
1615 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1616 else
1617 StoreChain = DAG.getEntryNode();
1618
1619 // Result is a load from the stack slot.
1620 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1621}
1622
1623/// Bitcast a floating-point value to an integer value. Only bitcast the part
1624/// containing the sign bit if the target has no integer value capable of
1625/// holding all bits of the floating-point value.
1626void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1627 const SDLoc &DL,
1628 SDValue Value) const {
1629 EVT FloatVT = Value.getValueType();
1630 unsigned NumBits = FloatVT.getScalarSizeInBits();
1631 State.FloatVT = FloatVT;
1632 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1633 // Convert to an integer of the same size.
1634 if (TLI.isTypeLegal(VT: IVT)) {
1635 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1636 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1637 State.SignBit = NumBits - 1;
1638 return;
1639 }
1640
1641 auto &DataLayout = DAG.getDataLayout();
1642 // Store the float to memory, then load the sign part out as an integer.
1643 MVT LoadTy = TLI.getRegisterType(VT: MVT::i8);
1644 // First create a temporary that is aligned for both the load and store.
1645 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1646 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1647 // Then store the float to it.
1648 State.FloatPtr = StackPtr;
1649 MachineFunction &MF = DAG.getMachineFunction();
1650 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1651 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1652 PtrInfo: State.FloatPointerInfo);
1653
1654 SDValue IntPtr;
1655 if (DataLayout.isBigEndian()) {
1656 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1657 // Load out a legal integer with the same sign bit as the float.
1658 IntPtr = StackPtr;
1659 State.IntPointerInfo = State.FloatPointerInfo;
1660 } else {
1661 // Advance the pointer so that the loaded byte will contain the sign bit.
1662 unsigned ByteOffset = (NumBits / 8) - 1;
1663 IntPtr =
1664 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1665 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1666 Offset: ByteOffset);
1667 }
1668
1669 State.IntPtr = IntPtr;
1670 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1671 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1672 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1673 State.SignBit = 7;
1674}
1675
1676/// Replace the integer value produced by getSignAsIntValue() with a new value
1677/// and cast the result back to a floating-point type.
1678SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1679 const SDLoc &DL,
1680 SDValue NewIntValue) const {
1681 if (!State.Chain)
1682 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1683
1684 // Override the part containing the sign bit in the value stored on the stack.
1685 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1686 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1687 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1688 PtrInfo: State.FloatPointerInfo);
1689}
1690
1691SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1692 SDLoc DL(Node);
1693 SDValue Mag = Node->getOperand(Num: 0);
1694 SDValue Sign = Node->getOperand(Num: 1);
1695
1696 // Get sign bit into an integer value.
1697 FloatSignAsInt SignAsInt;
1698 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1699
1700 EVT IntVT = SignAsInt.IntValue.getValueType();
1701 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1702 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1703 N2: SignMask);
1704
1705 // If FABS is legal transform
1706 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1707 EVT FloatVT = Mag.getValueType();
1708 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1709 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1710 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1711 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1712 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1713 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1714 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1715 }
1716
1717 // Transform Mag value to integer, and clear the sign bit.
1718 FloatSignAsInt MagAsInt;
1719 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1720 EVT MagVT = MagAsInt.IntValue.getValueType();
1721 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1722 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1723 N2: ClearSignMask);
1724
1725 // Get the signbit at the right position for MagAsInt.
1726 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1727 EVT ShiftVT = IntVT;
1728 if (SignBit.getScalarValueSizeInBits() <
1729 ClearedSign.getScalarValueSizeInBits()) {
1730 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1731 ShiftVT = MagVT;
1732 }
1733 if (ShiftAmount > 0) {
1734 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1735 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1736 } else if (ShiftAmount < 0) {
1737 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1738 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1739 }
1740 if (SignBit.getScalarValueSizeInBits() >
1741 ClearedSign.getScalarValueSizeInBits()) {
1742 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1743 }
1744
1745 // Store the part with the modified sign and convert back to float.
1746 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1747 Flags: SDNodeFlags::Disjoint);
1748
1749 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1750}
1751
1752SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1753 // Get the sign bit as an integer.
1754 SDLoc DL(Node);
1755 FloatSignAsInt SignAsInt;
1756 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1757 EVT IntVT = SignAsInt.IntValue.getValueType();
1758
1759 // Flip the sign.
1760 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1761 SDValue SignFlip =
1762 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1763
1764 // Convert back to float.
1765 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1766}
1767
1768SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1769 SDLoc DL(Node);
1770 SDValue Value = Node->getOperand(Num: 0);
1771
1772 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1773 EVT FloatVT = Value.getValueType();
1774 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1775 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1776 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1777 }
1778
1779 // Transform value to integer, clear the sign bit and transform back.
1780 FloatSignAsInt ValueAsInt;
1781 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1782 EVT IntVT = ValueAsInt.IntValue.getValueType();
1783 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1784 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1785 N2: ClearSignMask);
1786 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1787}
1788
1789void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1790 SmallVectorImpl<SDValue> &Results) {
1791 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1792 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1793 " not tell us which reg is the stack pointer!");
1794 SDLoc dl(Node);
1795 EVT VT = Node->getValueType(ResNo: 0);
1796 SDValue Tmp1 = SDValue(Node, 0);
1797 SDValue Tmp2 = SDValue(Node, 1);
1798 SDValue Tmp3 = Node->getOperand(Num: 2);
1799 SDValue Chain = Tmp1.getOperand(i: 0);
1800
1801 // Chain the dynamic stack allocation so that it doesn't modify the stack
1802 // pointer when other instructions are using the stack.
1803 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1804
1805 SDValue Size = Tmp2.getOperand(i: 1);
1806 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1807 Chain = SP.getValue(R: 1);
1808 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1809 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1810 unsigned Opc =
1811 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1812 ISD::ADD : ISD::SUB;
1813
1814 Align StackAlign = TFL->getStackAlign();
1815 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1816 if (Alignment > StackAlign)
1817 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1818 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1819 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1820
1821 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1822
1823 Results.push_back(Elt: Tmp1);
1824 Results.push_back(Elt: Tmp2);
1825}
1826
1827/// Emit a store/load combination to the stack. This stores
1828/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1829/// a load from the stack slot to DestVT, extending it if needed.
1830/// The resultant code need not be legal.
1831SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1832 EVT DestVT, const SDLoc &dl) {
1833 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1834}
1835
1836SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1837 EVT DestVT, const SDLoc &dl,
1838 SDValue Chain) {
1839 EVT SrcVT = SrcOp.getValueType();
1840 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1841 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1842
1843 // Don't convert with stack if the load/store is expensive.
1844 if ((SrcVT.bitsGT(VT: SlotVT) &&
1845 !TLI.isTruncStoreLegalOrCustom(ValVT: SrcOp.getValueType(), MemVT: SlotVT)) ||
1846 (SlotVT.bitsLT(VT: DestVT) &&
1847 !TLI.isLoadExtLegalOrCustom(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SlotVT)))
1848 return SDValue();
1849
1850 // Create the stack frame object.
1851 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1852 Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1853 SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1854
1855 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1856 int SPFI = StackPtrFI->getIndex();
1857 MachinePointerInfo PtrInfo =
1858 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1859
1860 // Emit a store to the stack slot. Use a truncstore if the input value is
1861 // later than DestVT.
1862 SDValue Store;
1863
1864 if (SrcVT.bitsGT(VT: SlotVT))
1865 Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1866 SVT: SlotVT, Alignment: SrcAlign);
1867 else {
1868 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1869 Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1870 }
1871
1872 // Result is a load from the stack slot.
1873 if (SlotVT.bitsEq(VT: DestVT))
1874 return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1875
1876 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1877 return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1878 Alignment: DestAlign);
1879}
1880
1881SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1882 SDLoc dl(Node);
1883 // Create a vector sized/aligned stack slot, store the value to element #0,
1884 // then load the whole vector back out.
1885 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1886
1887 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1888 int SPFI = StackPtrFI->getIndex();
1889
1890 SDValue Ch = DAG.getTruncStore(
1891 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1892 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1893 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1894 return DAG.getLoad(
1895 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1896 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1897}
1898
1899static bool
1900ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1901 const TargetLowering &TLI, SDValue &Res) {
1902 unsigned NumElems = Node->getNumOperands();
1903 SDLoc dl(Node);
1904 EVT VT = Node->getValueType(ResNo: 0);
1905
1906 // Try to group the scalars into pairs, shuffle the pairs together, then
1907 // shuffle the pairs of pairs together, etc. until the vector has
1908 // been built. This will work only if all of the necessary shuffle masks
1909 // are legal.
1910
1911 // We do this in two phases; first to check the legality of the shuffles,
1912 // and next, assuming that all shuffles are legal, to create the new nodes.
1913 for (int Phase = 0; Phase < 2; ++Phase) {
1914 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1915 NewIntermedVals;
1916 for (unsigned i = 0; i < NumElems; ++i) {
1917 SDValue V = Node->getOperand(Num: i);
1918 if (V.isUndef())
1919 continue;
1920
1921 SDValue Vec;
1922 if (Phase)
1923 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1924 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1925 }
1926
1927 while (IntermedVals.size() > 2) {
1928 NewIntermedVals.clear();
1929 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1930 // This vector and the next vector are shuffled together (simply to
1931 // append the one to the other).
1932 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1933
1934 SmallVector<int, 16> FinalIndices;
1935 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1936 IntermedVals[i+1].second.size());
1937
1938 int k = 0;
1939 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1940 ++j, ++k) {
1941 ShuffleVec[k] = j;
1942 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1943 }
1944 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1945 ++j, ++k) {
1946 ShuffleVec[k] = NumElems + j;
1947 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1948 }
1949
1950 SDValue Shuffle;
1951 if (Phase)
1952 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1953 N2: IntermedVals[i+1].first,
1954 Mask: ShuffleVec);
1955 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1956 return false;
1957 NewIntermedVals.push_back(
1958 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1959 }
1960
1961 // If we had an odd number of defined values, then append the last
1962 // element to the array of new vectors.
1963 if ((IntermedVals.size() & 1) != 0)
1964 NewIntermedVals.push_back(Elt: IntermedVals.back());
1965
1966 IntermedVals.swap(RHS&: NewIntermedVals);
1967 }
1968
1969 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1970 "Invalid number of intermediate vectors");
1971 SDValue Vec1 = IntermedVals[0].first;
1972 SDValue Vec2;
1973 if (IntermedVals.size() > 1)
1974 Vec2 = IntermedVals[1].first;
1975 else if (Phase)
1976 Vec2 = DAG.getUNDEF(VT);
1977
1978 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1979 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1980 ShuffleVec[IntermedVals[0].second[i]] = i;
1981 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1982 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1983
1984 if (Phase)
1985 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1986 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1987 return false;
1988 }
1989
1990 return true;
1991}
1992
1993/// Expand a BUILD_VECTOR node on targets that don't
1994/// support the operation, but do support the resultant vector type.
1995SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1996 unsigned NumElems = Node->getNumOperands();
1997 SDValue Value1, Value2;
1998 SDLoc dl(Node);
1999 EVT VT = Node->getValueType(ResNo: 0);
2000 EVT OpVT = Node->getOperand(Num: 0).getValueType();
2001 EVT EltVT = VT.getVectorElementType();
2002
2003 // If the only non-undef value is the low element, turn this into a
2004 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
2005 bool isOnlyLowElement = true;
2006 bool MoreThanTwoValues = false;
2007 bool isConstant = true;
2008 for (unsigned i = 0; i < NumElems; ++i) {
2009 SDValue V = Node->getOperand(Num: i);
2010 if (V.isUndef())
2011 continue;
2012 if (i > 0)
2013 isOnlyLowElement = false;
2014 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2015 isConstant = false;
2016
2017 if (!Value1.getNode()) {
2018 Value1 = V;
2019 } else if (!Value2.getNode()) {
2020 if (V != Value1)
2021 Value2 = V;
2022 } else if (V != Value1 && V != Value2) {
2023 MoreThanTwoValues = true;
2024 }
2025 }
2026
2027 if (!Value1.getNode())
2028 return DAG.getUNDEF(VT);
2029
2030 if (isOnlyLowElement)
2031 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2032
2033 // If all elements are constants, create a load from the constant pool.
2034 if (isConstant) {
2035 SmallVector<Constant*, 16> CV;
2036 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2037 if (ConstantFPSDNode *V =
2038 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2039 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2040 } else if (ConstantSDNode *V =
2041 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2042 if (OpVT==EltVT)
2043 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2044 else {
2045 // If OpVT and EltVT don't match, EltVT is not legal and the
2046 // element values have been promoted/truncated earlier. Undo this;
2047 // we don't want a v16i8 to become a v16i32 for example.
2048 const ConstantInt *CI = V->getConstantIntValue();
2049 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2050 V: CI->getZExtValue()));
2051 }
2052 } else {
2053 assert(Node->getOperand(i).isUndef());
2054 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2055 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2056 }
2057 }
2058 Constant *CP = ConstantVector::get(V: CV);
2059 SDValue CPIdx =
2060 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2061 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2062 return DAG.getLoad(
2063 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2064 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2065 Alignment);
2066 }
2067
2068 SmallSet<SDValue, 16> DefinedValues;
2069 for (unsigned i = 0; i < NumElems; ++i) {
2070 if (Node->getOperand(Num: i).isUndef())
2071 continue;
2072 DefinedValues.insert(V: Node->getOperand(Num: i));
2073 }
2074
2075 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2076 if (!MoreThanTwoValues) {
2077 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2078 for (unsigned i = 0; i < NumElems; ++i) {
2079 SDValue V = Node->getOperand(Num: i);
2080 if (V.isUndef())
2081 continue;
2082 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2083 }
2084 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2085 // Get the splatted value into the low element of a vector register.
2086 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2087 SDValue Vec2;
2088 if (Value2.getNode())
2089 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2090 else
2091 Vec2 = DAG.getUNDEF(VT);
2092
2093 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2094 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2095 }
2096 } else {
2097 SDValue Res;
2098 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2099 return Res;
2100 }
2101 }
2102
2103 // Otherwise, we can't handle this case efficiently.
2104 return ExpandVectorBuildThroughStack(Node);
2105}
2106
2107SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2108 SDLoc DL(Node);
2109 EVT VT = Node->getValueType(ResNo: 0);
2110 SDValue SplatVal = Node->getOperand(Num: 0);
2111
2112 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2113}
2114
2115// Expand a node into a call to a libcall, returning the value as the first
2116// result and the chain as the second. If the result value does not fit into a
2117// register, return the lo part and set the hi part to the by-reg argument in
2118// the first. If it does fit into a single register, return the result and
2119// leave the Hi part unset.
2120std::pair<SDValue, SDValue>
2121SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2122 TargetLowering::ArgListTy &&Args,
2123 bool IsSigned, EVT RetVT) {
2124 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2125 SDValue Callee;
2126 if (const char *LibcallName = TLI.getLibcallName(Call: LC))
2127 Callee = DAG.getExternalSymbol(Sym: LibcallName, VT: CodePtrTy);
2128 else {
2129 Callee = DAG.getUNDEF(VT: CodePtrTy);
2130 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2131 Node->getOperationName(G: &DAG));
2132 }
2133
2134 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2135
2136 // By default, the input chain to this libcall is the entry node of the
2137 // function. If the libcall is going to be emitted as a tail call then
2138 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2139 // node which is being folded has a non-entry input chain.
2140 SDValue InChain = DAG.getEntryNode();
2141
2142 // isTailCall may be true since the callee does not reference caller stack
2143 // frame. Check if it's in the right position and that the return types match.
2144 SDValue TCChain = InChain;
2145 const Function &F = DAG.getMachineFunction().getFunction();
2146 bool isTailCall =
2147 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2148 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2149 if (isTailCall)
2150 InChain = TCChain;
2151
2152 TargetLowering::CallLoweringInfo CLI(DAG);
2153 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2154 CLI.setDebugLoc(SDLoc(Node))
2155 .setChain(InChain)
2156 .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee,
2157 ArgsList: std::move(Args))
2158 .setTailCall(isTailCall)
2159 .setSExtResult(signExtend)
2160 .setZExtResult(!signExtend)
2161 .setIsPostTypeLegalization(true);
2162
2163 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2164
2165 if (!CallInfo.second.getNode()) {
2166 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2167 // It's a tailcall, return the chain (which is the DAG root).
2168 return {DAG.getRoot(), DAG.getRoot()};
2169 }
2170
2171 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2172 return CallInfo;
2173}
2174
2175std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2176 bool isSigned) {
2177 TargetLowering::ArgListTy Args;
2178 TargetLowering::ArgListEntry Entry;
2179 for (const SDValue &Op : Node->op_values()) {
2180 EVT ArgVT = Op.getValueType();
2181 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2182 Entry.Node = Op;
2183 Entry.Ty = ArgTy;
2184 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2185 Entry.IsZExt = !Entry.IsSExt;
2186 Args.push_back(x: Entry);
2187 }
2188
2189 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2190 RetVT: Node->getValueType(ResNo: 0));
2191}
2192
2193void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2194 RTLIB::Libcall LC,
2195 SmallVectorImpl<SDValue> &Results) {
2196 if (LC == RTLIB::UNKNOWN_LIBCALL)
2197 llvm_unreachable("Can't create an unknown libcall!");
2198
2199 if (Node->isStrictFPOpcode()) {
2200 EVT RetVT = Node->getValueType(ResNo: 0);
2201 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2202 TargetLowering::MakeLibCallOptions CallOptions;
2203 CallOptions.IsPostTypeLegalization = true;
2204 // FIXME: This doesn't support tail calls.
2205 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2206 Ops, CallOptions,
2207 dl: SDLoc(Node),
2208 Chain: Node->getOperand(Num: 0));
2209 Results.push_back(Elt: Tmp.first);
2210 Results.push_back(Elt: Tmp.second);
2211 } else {
2212 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2213 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2214 Results.push_back(Elt: Tmp);
2215 }
2216}
2217
2218/// Expand the node to a libcall based on the result type.
2219void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2220 RTLIB::Libcall Call_F32,
2221 RTLIB::Libcall Call_F64,
2222 RTLIB::Libcall Call_F80,
2223 RTLIB::Libcall Call_F128,
2224 RTLIB::Libcall Call_PPCF128,
2225 SmallVectorImpl<SDValue> &Results) {
2226 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0),
2227 Call_F32, Call_F64, Call_F80,
2228 Call_F128, Call_PPCF128);
2229 ExpandFPLibCall(Node, LC, Results);
2230}
2231
2232SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2233 RTLIB::Libcall Call_I8,
2234 RTLIB::Libcall Call_I16,
2235 RTLIB::Libcall Call_I32,
2236 RTLIB::Libcall Call_I64,
2237 RTLIB::Libcall Call_I128) {
2238 RTLIB::Libcall LC;
2239 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2240 default: llvm_unreachable("Unexpected request for libcall!");
2241 case MVT::i8: LC = Call_I8; break;
2242 case MVT::i16: LC = Call_I16; break;
2243 case MVT::i32: LC = Call_I32; break;
2244 case MVT::i64: LC = Call_I64; break;
2245 case MVT::i128: LC = Call_I128; break;
2246 }
2247 return ExpandLibCall(LC, Node, isSigned).first;
2248}
2249
2250/// Expand the node to a libcall based on first argument type (for instance
2251/// lround and its variant).
2252void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2253 RTLIB::Libcall Call_F32,
2254 RTLIB::Libcall Call_F64,
2255 RTLIB::Libcall Call_F80,
2256 RTLIB::Libcall Call_F128,
2257 RTLIB::Libcall Call_PPCF128,
2258 SmallVectorImpl<SDValue> &Results) {
2259 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2260 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2261 Call_F32, Call_F64, Call_F80,
2262 Call_F128, Call_PPCF128);
2263 ExpandFPLibCall(Node, LC, Results);
2264}
2265
2266SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2267 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2268 RTLIB::Libcall CallI128) {
2269 RTLIB::Libcall LC;
2270 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2271 default:
2272 llvm_unreachable("Unexpected request for libcall!");
2273 case MVT::i32:
2274 LC = CallI32;
2275 break;
2276 case MVT::i64:
2277 LC = CallI64;
2278 break;
2279 case MVT::i128:
2280 LC = CallI128;
2281 break;
2282 }
2283
2284 // Bit-counting libcalls have one unsigned argument and return `int`.
2285 // Note that `int` may be illegal on this target; ExpandLibCall will
2286 // take care of promoting it to a legal type.
2287 SDValue Op = Node->getOperand(Num: 0);
2288 EVT IntVT =
2289 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2290
2291 TargetLowering::ArgListEntry Arg;
2292 EVT ArgVT = Op.getValueType();
2293 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2294 Arg.Node = Op;
2295 Arg.Ty = ArgTy;
2296 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2297 Arg.IsZExt = !Arg.IsSExt;
2298
2299 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2300 /*IsSigned=*/true, RetVT: IntVT)
2301 .first;
2302
2303 // If ExpandLibCall created a tail call, the result was already
2304 // of the correct type. Otherwise, we need to sign extend it.
2305 if (Res.getValueType() != MVT::Other)
2306 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2307 return Res;
2308}
2309
2310/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2311void
2312SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2313 SmallVectorImpl<SDValue> &Results) {
2314 unsigned Opcode = Node->getOpcode();
2315 bool isSigned = Opcode == ISD::SDIVREM;
2316
2317 RTLIB::Libcall LC;
2318 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2319 default: llvm_unreachable("Unexpected request for libcall!");
2320 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2321 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2322 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2323 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2324 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2325 }
2326
2327 // The input chain to this libcall is the entry node of the function.
2328 // Legalizing the call will automatically add the previous call to the
2329 // dependence.
2330 SDValue InChain = DAG.getEntryNode();
2331
2332 EVT RetVT = Node->getValueType(ResNo: 0);
2333 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2334
2335 TargetLowering::ArgListTy Args;
2336 TargetLowering::ArgListEntry Entry;
2337 for (const SDValue &Op : Node->op_values()) {
2338 EVT ArgVT = Op.getValueType();
2339 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2340 Entry.Node = Op;
2341 Entry.Ty = ArgTy;
2342 Entry.IsSExt = isSigned;
2343 Entry.IsZExt = !isSigned;
2344 Args.push_back(x: Entry);
2345 }
2346
2347 // Also pass the return address of the remainder.
2348 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2349 Entry.Node = FIPtr;
2350 Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2351 Entry.IsSExt = isSigned;
2352 Entry.IsZExt = !isSigned;
2353 Args.push_back(x: Entry);
2354
2355 SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2356 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2357
2358 SDLoc dl(Node);
2359 TargetLowering::CallLoweringInfo CLI(DAG);
2360 CLI.setDebugLoc(dl)
2361 .setChain(InChain)
2362 .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee,
2363 ArgsList: std::move(Args))
2364 .setSExtResult(isSigned)
2365 .setZExtResult(!isSigned);
2366
2367 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2368
2369 // Remainder is loaded back from the stack frame.
2370 SDValue Rem =
2371 DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo: MachinePointerInfo());
2372 Results.push_back(Elt: CallInfo.first);
2373 Results.push_back(Elt: Rem);
2374}
2375
2376/// Return true if sincos libcall is available.
2377static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2378 RTLIB::Libcall LC = RTLIB::getSINCOS(RetVT: Node->getSimpleValueType(ResNo: 0).SimpleTy);
2379 return TLI.getLibcallName(Call: LC) != nullptr;
2380}
2381
2382/// Only issue sincos libcall if both sin and cos are needed.
2383static bool useSinCos(SDNode *Node) {
2384 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2385 ? ISD::FCOS : ISD::FSIN;
2386
2387 SDValue Op0 = Node->getOperand(Num: 0);
2388 for (const SDNode *User : Op0.getNode()->users()) {
2389 if (User == Node)
2390 continue;
2391 // The other user might have been turned into sincos already.
2392 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2393 return true;
2394 }
2395 return false;
2396}
2397
2398SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2399 SDLoc dl(Node);
2400 EVT VT = Node->getValueType(ResNo: 0);
2401 SDValue X = Node->getOperand(Num: 0);
2402 SDValue N = Node->getOperand(Num: 1);
2403 EVT ExpVT = N.getValueType();
2404 EVT AsIntVT = VT.changeTypeToInteger();
2405 if (AsIntVT == EVT()) // TODO: How to handle f80?
2406 return SDValue();
2407
2408 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2409 return SDValue();
2410
2411 SDNodeFlags NSW;
2412 NSW.setNoSignedWrap(true);
2413 SDNodeFlags NUW_NSW;
2414 NUW_NSW.setNoUnsignedWrap(true);
2415 NUW_NSW.setNoSignedWrap(true);
2416
2417 EVT SetCCVT =
2418 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2419 const fltSemantics &FltSem = VT.getFltSemantics();
2420
2421 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2422 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2423 const int Precision = APFloat::semanticsPrecision(FltSem);
2424
2425 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2426 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2427
2428 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2429
2430 const APFloat One(FltSem, "1.0");
2431 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2432
2433 // Offset by precision to avoid denormal range.
2434 APFloat ScaleDownK =
2435 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2436
2437 // TODO: Should really introduce control flow and use a block for the >
2438 // MaxExp, < MinExp cases
2439
2440 // First, handle exponents Exp > MaxExp and scale down.
2441 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2442
2443 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2444 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2445 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2446 SDValue DecN1 =
2447 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2448
2449 SDValue ScaleUpTwice =
2450 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2451
2452 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2453 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2454 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2455
2456 SDValue SelectN_Big =
2457 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2458 SDValue SelectX_Big =
2459 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2460
2461 // Now handle exponents Exp < MinExp
2462 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2463
2464 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2465 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2466
2467 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2468
2469 SDValue ClampMinVal =
2470 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2471 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2472 SDValue IncN1 =
2473 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2474
2475 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2476 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2477 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2478
2479 SDValue ScaleDownTwice = DAG.getSetCC(
2480 DL: dl, VT: SetCCVT, LHS: N,
2481 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2482
2483 SDValue SelectN_Small =
2484 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2485 SDValue SelectX_Small =
2486 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2487
2488 // Now combine the two out of range exponent handling cases with the base
2489 // case.
2490 SDValue NewX = DAG.getNode(
2491 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2492 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2493
2494 SDValue NewN = DAG.getNode(
2495 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2496 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2497
2498 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2499
2500 SDValue ExponentShiftAmt =
2501 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2502 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2503
2504 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2505 N2: ExponentShiftAmt, Flags: NUW_NSW);
2506 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2507 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2508}
2509
2510SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2511 SDLoc dl(Node);
2512 SDValue Val = Node->getOperand(Num: 0);
2513 EVT VT = Val.getValueType();
2514 EVT ExpVT = Node->getValueType(ResNo: 1);
2515 EVT AsIntVT = VT.changeTypeToInteger();
2516 if (AsIntVT == EVT()) // TODO: How to handle f80?
2517 return SDValue();
2518
2519 const fltSemantics &FltSem = VT.getFltSemantics();
2520 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2521 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2522 const unsigned BitSize = VT.getScalarSizeInBits();
2523
2524 // TODO: Could introduce control flow and skip over the denormal handling.
2525
2526 // scale_up = fmul value, scalbn(1.0, precision + 1)
2527 // extracted_exp = (bitcast value to uint) >> precision - 1
2528 // biased_exp = extracted_exp + min_exp
2529 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2530 //
2531 // is_denormal = val < smallest_normalized
2532 // computed_fract = is_denormal ? scale_up : extracted_fract
2533 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2534 //
2535 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2536 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2537
2538 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2539 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2540 VT: AsIntVT);
2541
2542 SDValue SmallestNormalizedInt = DAG.getConstant(
2543 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2544 VT: AsIntVT);
2545
2546 // Masks out the exponent bits.
2547 SDValue ExpMask =
2548 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2549
2550 // Mask out the exponent part of the value.
2551 //
2552 // e.g, for f32 FractSignMaskVal = 0x807fffff
2553 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2554 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2555
2556 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2557 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2558
2559 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2560
2561 const APFloat One(FltSem, "1.0");
2562 // Scale a possible denormal input.
2563 // e.g., for f64, 0x1p+54
2564 APFloat ScaleUpKVal =
2565 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2566
2567 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2568 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2569
2570 EVT SetCCVT =
2571 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2572
2573 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2574
2575 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2576
2577 SDValue AddNegSmallestNormal =
2578 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2579 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2580 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2581
2582 SDValue IsDenormal =
2583 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2584
2585 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2586 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2587
2588 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2589 SDValue ScaledSelect =
2590 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2591
2592 SDValue ExpMaskScaled =
2593 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2594
2595 SDValue ScaledValue =
2596 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2597
2598 // Extract the exponent bits.
2599 SDValue ExponentShiftAmt =
2600 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2601 SDValue ShiftedExp =
2602 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2603 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2604
2605 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2606 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2607 SDValue DenormalExpBias =
2608 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2609
2610 SDValue MaskedFractAsInt =
2611 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2612 const APFloat Half(FltSem, "0.5");
2613 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2614 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2615 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2616
2617 SDValue ComputedExp =
2618 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2619
2620 SDValue Result0 =
2621 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2622
2623 SDValue Result1 =
2624 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2625
2626 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2627}
2628
2629/// This function is responsible for legalizing a
2630/// INT_TO_FP operation of the specified operand when the target requests that
2631/// we expand it. At this point, we know that the result and operand types are
2632/// legal for the target.
2633SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2634 SDValue &Chain) {
2635 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2636 Node->getOpcode() == ISD::SINT_TO_FP);
2637 EVT DestVT = Node->getValueType(ResNo: 0);
2638 SDLoc dl(Node);
2639 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2640 SDValue Op0 = Node->getOperand(Num: OpNo);
2641 EVT SrcVT = Op0.getValueType();
2642
2643 // TODO: Should any fast-math-flags be set for the created nodes?
2644 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2645 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2646 (DestVT.bitsLE(VT: MVT::f64) ||
2647 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2648 : ISD::FP_EXTEND,
2649 VT: DestVT))) {
2650 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2651 "expansion\n");
2652
2653 // Get the stack frame index of a 8 byte buffer.
2654 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2655
2656 SDValue Lo = Op0;
2657 // if signed map to unsigned space
2658 if (isSigned) {
2659 // Invert sign bit (signed to unsigned mapping).
2660 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2661 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2662 }
2663 // Initial hi portion of constructed double.
2664 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2665
2666 // If this a big endian target, swap the lo and high data.
2667 if (DAG.getDataLayout().isBigEndian())
2668 std::swap(a&: Lo, b&: Hi);
2669
2670 SDValue MemChain = DAG.getEntryNode();
2671
2672 // Store the lo of the constructed double.
2673 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2674 PtrInfo: MachinePointerInfo());
2675 // Store the hi of the constructed double.
2676 SDValue HiPtr =
2677 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2678 SDValue Store2 =
2679 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2680 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2681
2682 // load the constructed double
2683 SDValue Load =
2684 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2685 // FP constant to bias correct the final result
2686 SDValue Bias = DAG.getConstantFP(
2687 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2688 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2689 DL: dl, VT: MVT::f64);
2690 // Subtract the bias and get the final result.
2691 SDValue Sub;
2692 SDValue Result;
2693 if (Node->isStrictFPOpcode()) {
2694 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2695 Ops: {Node->getOperand(Num: 0), Load, Bias});
2696 Chain = Sub.getValue(R: 1);
2697 if (DestVT != Sub.getValueType()) {
2698 std::pair<SDValue, SDValue> ResultPair;
2699 ResultPair =
2700 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2701 Result = ResultPair.first;
2702 Chain = ResultPair.second;
2703 }
2704 else
2705 Result = Sub;
2706 } else {
2707 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2708 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2709 }
2710 return Result;
2711 }
2712
2713 if (isSigned)
2714 return SDValue();
2715
2716 // TODO: Generalize this for use with other types.
2717 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2718 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2719 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2720 // For unsigned conversions, convert them to signed conversions using the
2721 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2722 // should be valid for i32->f32 as well.
2723
2724 // More generally this transform should be valid if there are 3 more bits
2725 // in the integer type than the significand. Rounding uses the first bit
2726 // after the width of the significand and the OR of all bits after that. So
2727 // we need to be able to OR the shifted out bit into one of the bits that
2728 // participate in the OR.
2729
2730 // TODO: This really should be implemented using a branch rather than a
2731 // select. We happen to get lucky and machinesink does the right
2732 // thing most of the time. This would be a good candidate for a
2733 // pseudo-op, or, even better, for whole-function isel.
2734 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2735
2736 SDValue SignBitTest = DAG.getSetCC(
2737 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2738
2739 EVT ShiftVT = TLI.getShiftAmountTy(LHSTy: SrcVT, DL: DAG.getDataLayout());
2740 SDValue ShiftConst = DAG.getConstant(Val: 1, DL: dl, VT: ShiftVT);
2741 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2742 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2743 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2744 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2745
2746 SDValue Slow, Fast;
2747 if (Node->isStrictFPOpcode()) {
2748 // In strict mode, we must avoid spurious exceptions, and therefore
2749 // must make sure to only emit a single STRICT_SINT_TO_FP.
2750 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2751 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2752 Ops: { Node->getOperand(Num: 0), InCvt });
2753 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
2754 Ops: { Fast.getValue(R: 1), Fast, Fast });
2755 Chain = Slow.getValue(R: 1);
2756 // The STRICT_SINT_TO_FP inherits the exception mode from the
2757 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2758 // never raise any exception.
2759 SDNodeFlags Flags;
2760 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2761 Fast->setFlags(Flags);
2762 Flags.setNoFPExcept(true);
2763 Slow->setFlags(Flags);
2764 } else {
2765 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2766 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2767 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2768 }
2769
2770 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2771 }
2772
2773 // Don't expand it if there isn't cheap fadd.
2774 if (!TLI.isOperationLegalOrCustom(
2775 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2776 return SDValue();
2777
2778 // The following optimization is valid only if every value in SrcVT (when
2779 // treated as signed) is representable in DestVT. Check that the mantissa
2780 // size of DestVT is >= than the number of bits in SrcVT -1.
2781 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2782 SrcVT.getSizeInBits() - 1 &&
2783 "Cannot perform lossless SINT_TO_FP!");
2784
2785 SDValue Tmp1;
2786 if (Node->isStrictFPOpcode()) {
2787 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2788 Ops: { Node->getOperand(Num: 0), Op0 });
2789 } else
2790 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2791
2792 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2793 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2794 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2795 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2796 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2797 Cond: SignSet, LHS: Four, RHS: Zero);
2798
2799 // If the sign bit of the integer is set, the large number will be treated
2800 // as a negative number. To counteract this, the dynamic code adds an
2801 // offset depending on the data type.
2802 uint64_t FF;
2803 switch (SrcVT.getSimpleVT().SimpleTy) {
2804 default:
2805 return SDValue();
2806 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2807 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2808 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2809 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2810 }
2811 if (DAG.getDataLayout().isLittleEndian())
2812 FF <<= 32;
2813 Constant *FudgeFactor = ConstantInt::get(
2814 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2815
2816 SDValue CPIdx =
2817 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2818 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2819 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2820 Alignment = commonAlignment(A: Alignment, Offset: 4);
2821 SDValue FudgeInReg;
2822 if (DestVT == MVT::f32)
2823 FudgeInReg = DAG.getLoad(
2824 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2825 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2826 Alignment);
2827 else {
2828 SDValue Load = DAG.getExtLoad(
2829 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2830 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
2831 Alignment);
2832 HandleSDNode Handle(Load);
2833 LegalizeOp(Node: Load.getNode());
2834 FudgeInReg = Handle.getValue();
2835 }
2836
2837 if (Node->isStrictFPOpcode()) {
2838 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
2839 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
2840 Chain = Result.getValue(R: 1);
2841 return Result;
2842 }
2843
2844 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
2845}
2846
2847/// This function is responsible for legalizing a
2848/// *INT_TO_FP operation of the specified operand when the target requests that
2849/// we promote it. At this point, we know that the result and operand types are
2850/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2851/// operation that takes a larger input.
2852void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2853 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2854 bool IsStrict = N->isStrictFPOpcode();
2855 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2856 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2857 EVT DestVT = N->getValueType(ResNo: 0);
2858 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
2859 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2860 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2861
2862 // First step, figure out the appropriate *INT_TO_FP operation to use.
2863 EVT NewInTy = LegalOp.getValueType();
2864
2865 unsigned OpToUse = 0;
2866
2867 // Scan for the appropriate larger type to use.
2868 while (true) {
2869 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2870 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2871
2872 // If the target supports SINT_TO_FP of this type, use it.
2873 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
2874 OpToUse = SIntOp;
2875 break;
2876 }
2877 if (IsSigned)
2878 continue;
2879
2880 // If the target supports UINT_TO_FP of this type, use it.
2881 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
2882 OpToUse = UIntOp;
2883 break;
2884 }
2885
2886 // Otherwise, try a larger type.
2887 }
2888
2889 // Okay, we found the operation and type to use. Zero extend our input to the
2890 // desired type then run the operation on it.
2891 if (IsStrict) {
2892 SDValue Res =
2893 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
2894 Ops: {N->getOperand(Num: 0),
2895 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2896 DL: dl, VT: NewInTy, Operand: LegalOp)});
2897 Results.push_back(Elt: Res);
2898 Results.push_back(Elt: Res.getValue(R: 1));
2899 return;
2900 }
2901
2902 Results.push_back(
2903 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
2904 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2905 DL: dl, VT: NewInTy, Operand: LegalOp)));
2906}
2907
2908/// This function is responsible for legalizing a
2909/// FP_TO_*INT operation of the specified operand when the target requests that
2910/// we promote it. At this point, we know that the result and operand types are
2911/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2912/// operation that returns a larger result.
2913void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2914 SmallVectorImpl<SDValue> &Results) {
2915 bool IsStrict = N->isStrictFPOpcode();
2916 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2917 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2918 EVT DestVT = N->getValueType(ResNo: 0);
2919 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
2920 // First step, figure out the appropriate FP_TO*INT operation to use.
2921 EVT NewOutTy = DestVT;
2922
2923 unsigned OpToUse = 0;
2924
2925 // Scan for the appropriate larger type to use.
2926 while (true) {
2927 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2928 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2929
2930 // A larger signed type can hold all unsigned values of the requested type,
2931 // so using FP_TO_SINT is valid
2932 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2933 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
2934 break;
2935
2936 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2937 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2938 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
2939 break;
2940
2941 // Otherwise, try a larger type.
2942 }
2943
2944 // Okay, we found the operation and type to use.
2945 SDValue Operation;
2946 if (IsStrict) {
2947 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
2948 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
2949 } else
2950 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
2951
2952 // Truncate the result of the extended FP_TO_*INT operation to the desired
2953 // size.
2954 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
2955 Results.push_back(Elt: Trunc);
2956 if (IsStrict)
2957 Results.push_back(Elt: Operation.getValue(R: 1));
2958}
2959
2960/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2961/// the result and operand types are legal and there must be a legal
2962/// FP_TO_*INT_SAT operation for a larger result type.
2963SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2964 const SDLoc &dl) {
2965 unsigned Opcode = Node->getOpcode();
2966
2967 // Scan for the appropriate larger type to use.
2968 EVT NewOutTy = Node->getValueType(ResNo: 0);
2969 while (true) {
2970 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2971 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2972
2973 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
2974 break;
2975 }
2976
2977 // Saturation width is determined by second operand, so we don't have to
2978 // perform any fixup and can directly truncate the result.
2979 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
2980 N2: Node->getOperand(Num: 1));
2981 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
2982}
2983
2984/// Open code the operations for PARITY of the specified operation.
2985SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2986 EVT VT = Op.getValueType();
2987 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
2988 unsigned Sz = VT.getScalarSizeInBits();
2989
2990 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2991 SDValue Result;
2992 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
2993 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
2994 } else {
2995 Result = Op;
2996 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
2997 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
2998 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
2999 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3000 }
3001 }
3002
3003 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3004}
3005
3006SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3007 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3008 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3009 : Node->getOperand(Num: 0).getSimpleValueType();
3010 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3011 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3012 MVT NewScalarVT = NewVecVT.getVectorElementType();
3013
3014 SDLoc DL(Node);
3015 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3016
3017 // FIXME: Support integer.
3018 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3019 "Only FP promotion is supported");
3020
3021 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3022 if (Node->getOperand(Num: j).getValueType().isVector() &&
3023 !(IsVPOpcode &&
3024 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3025 // promote the vector operand.
3026 // FIXME: Support integer.
3027 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3028 "Only FP promotion is supported");
3029 Operands[j] =
3030 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3031 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3032 // promote the initial value.
3033 Operands[j] =
3034 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3035 } else {
3036 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3037 }
3038
3039 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3040 Flags: Node->getFlags());
3041
3042 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3043 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3044 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3045}
3046
3047bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3048 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3049 SmallVector<SDValue, 8> Results;
3050 SDLoc dl(Node);
3051 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3052 bool NeedInvert;
3053 switch (Node->getOpcode()) {
3054 case ISD::ABS:
3055 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3056 Results.push_back(Elt: Tmp1);
3057 break;
3058 case ISD::ABDS:
3059 case ISD::ABDU:
3060 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3061 Results.push_back(Elt: Tmp1);
3062 break;
3063 case ISD::AVGCEILS:
3064 case ISD::AVGCEILU:
3065 case ISD::AVGFLOORS:
3066 case ISD::AVGFLOORU:
3067 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3068 Results.push_back(Elt: Tmp1);
3069 break;
3070 case ISD::CTPOP:
3071 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3072 Results.push_back(Elt: Tmp1);
3073 break;
3074 case ISD::CTLZ:
3075 case ISD::CTLZ_ZERO_UNDEF:
3076 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3077 Results.push_back(Elt: Tmp1);
3078 break;
3079 case ISD::CTTZ:
3080 case ISD::CTTZ_ZERO_UNDEF:
3081 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3082 Results.push_back(Elt: Tmp1);
3083 break;
3084 case ISD::BITREVERSE:
3085 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3086 Results.push_back(Elt: Tmp1);
3087 break;
3088 case ISD::BSWAP:
3089 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3090 Results.push_back(Elt: Tmp1);
3091 break;
3092 case ISD::PARITY:
3093 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3094 break;
3095 case ISD::FRAMEADDR:
3096 case ISD::RETURNADDR:
3097 case ISD::FRAME_TO_ARGS_OFFSET:
3098 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3099 break;
3100 case ISD::EH_DWARF_CFA: {
3101 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3102 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3103 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3104 VT: CfaArg.getValueType(),
3105 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3106 VT: CfaArg.getValueType()),
3107 N2: CfaArg);
3108 SDValue FA = DAG.getNode(
3109 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3110 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3111 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3112 N1: FA, N2: Offset));
3113 break;
3114 }
3115 case ISD::GET_ROUNDING:
3116 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3117 Results.push_back(Elt: Node->getOperand(Num: 0));
3118 break;
3119 case ISD::EH_RETURN:
3120 case ISD::EH_LABEL:
3121 case ISD::PREFETCH:
3122 case ISD::VAEND:
3123 case ISD::EH_SJLJ_LONGJMP:
3124 // If the target didn't expand these, there's nothing to do, so just
3125 // preserve the chain and be done.
3126 Results.push_back(Elt: Node->getOperand(Num: 0));
3127 break;
3128 case ISD::READCYCLECOUNTER:
3129 case ISD::READSTEADYCOUNTER:
3130 // If the target didn't expand this, just return 'zero' and preserve the
3131 // chain.
3132 Results.append(NumInputs: Node->getNumValues() - 1,
3133 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3134 Results.push_back(Elt: Node->getOperand(Num: 0));
3135 break;
3136 case ISD::EH_SJLJ_SETJMP:
3137 // If the target didn't expand this, just return 'zero' and preserve the
3138 // chain.
3139 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3140 Results.push_back(Elt: Node->getOperand(Num: 0));
3141 break;
3142 case ISD::ATOMIC_LOAD: {
3143 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3144 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3145 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3146 SDValue Swap = DAG.getAtomicCmpSwap(
3147 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3148 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3149 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3150 Results.push_back(Elt: Swap.getValue(R: 0));
3151 Results.push_back(Elt: Swap.getValue(R: 1));
3152 break;
3153 }
3154 case ISD::ATOMIC_STORE: {
3155 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3156 SDValue Swap = DAG.getAtomic(
3157 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3158 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3159 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3160 Results.push_back(Elt: Swap.getValue(R: 1));
3161 break;
3162 }
3163 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3164 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3165 // splits out the success value as a comparison. Expanding the resulting
3166 // ATOMIC_CMP_SWAP will produce a libcall.
3167 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3168 SDValue Res = DAG.getAtomicCmpSwap(
3169 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3170 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3171 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3172
3173 SDValue ExtRes = Res;
3174 SDValue LHS = Res;
3175 SDValue RHS = Node->getOperand(Num: 1);
3176
3177 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3178 EVT OuterType = Node->getValueType(ResNo: 0);
3179 switch (TLI.getExtendForAtomicOps()) {
3180 case ISD::SIGN_EXTEND:
3181 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3182 N2: DAG.getValueType(AtomicType));
3183 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3184 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3185 ExtRes = LHS;
3186 break;
3187 case ISD::ZERO_EXTEND:
3188 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3189 N2: DAG.getValueType(AtomicType));
3190 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3191 ExtRes = LHS;
3192 break;
3193 case ISD::ANY_EXTEND:
3194 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3195 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3196 break;
3197 default:
3198 llvm_unreachable("Invalid atomic op extension");
3199 }
3200
3201 SDValue Success =
3202 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3203
3204 Results.push_back(Elt: ExtRes.getValue(R: 0));
3205 Results.push_back(Elt: Success);
3206 Results.push_back(Elt: Res.getValue(R: 1));
3207 break;
3208 }
3209 case ISD::ATOMIC_LOAD_SUB: {
3210 SDLoc DL(Node);
3211 EVT VT = Node->getValueType(ResNo: 0);
3212 SDValue RHS = Node->getOperand(Num: 2);
3213 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3214 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3215 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3216 RHS = RHS->getOperand(Num: 0);
3217 SDValue NewRHS =
3218 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3219 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3220 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3221 Val: NewRHS, MMO: AN->getMemOperand());
3222 Results.push_back(Elt: Res);
3223 Results.push_back(Elt: Res.getValue(R: 1));
3224 break;
3225 }
3226 case ISD::DYNAMIC_STACKALLOC:
3227 ExpandDYNAMIC_STACKALLOC(Node, Results);
3228 break;
3229 case ISD::MERGE_VALUES:
3230 for (unsigned i = 0; i < Node->getNumValues(); i++)
3231 Results.push_back(Elt: Node->getOperand(Num: i));
3232 break;
3233 case ISD::POISON:
3234 case ISD::UNDEF: {
3235 EVT VT = Node->getValueType(ResNo: 0);
3236 if (VT.isInteger())
3237 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3238 else {
3239 assert(VT.isFloatingPoint() && "Unknown value type!");
3240 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3241 }
3242 break;
3243 }
3244 case ISD::STRICT_FP_ROUND:
3245 // When strict mode is enforced we can't do expansion because it
3246 // does not honor the "strict" properties. Only libcall is allowed.
3247 if (TLI.isStrictFPEnabled())
3248 break;
3249 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3250 // since this operation is more efficient than stack operation.
3251 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3252 VT: Node->getValueType(ResNo: 0))
3253 == TargetLowering::Legal)
3254 break;
3255 // We fall back to use stack operation when the FP_ROUND operation
3256 // isn't available.
3257 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3258 DestVT: Node->getValueType(ResNo: 0), dl,
3259 Chain: Node->getOperand(Num: 0)))) {
3260 ReplaceNode(Old: Node, New: Tmp1.getNode());
3261 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3262 return true;
3263 }
3264 break;
3265 case ISD::FP_ROUND: {
3266 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3267 Results.push_back(Elt: Tmp1);
3268 break;
3269 }
3270
3271 [[fallthrough]];
3272 }
3273 case ISD::BITCAST:
3274 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3275 DestVT: Node->getValueType(ResNo: 0), dl)))
3276 Results.push_back(Elt: Tmp1);
3277 break;
3278 case ISD::STRICT_FP_EXTEND:
3279 // When strict mode is enforced we can't do expansion because it
3280 // does not honor the "strict" properties. Only libcall is allowed.
3281 if (TLI.isStrictFPEnabled())
3282 break;
3283 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3284 // since this operation is more efficient than stack operation.
3285 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3286 VT: Node->getValueType(ResNo: 0))
3287 == TargetLowering::Legal)
3288 break;
3289 // We fall back to use stack operation when the FP_EXTEND operation
3290 // isn't available.
3291 if ((Tmp1 = EmitStackConvert(
3292 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3293 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3294 ReplaceNode(Old: Node, New: Tmp1.getNode());
3295 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3296 return true;
3297 }
3298 break;
3299 case ISD::FP_EXTEND: {
3300 SDValue Op = Node->getOperand(Num: 0);
3301 EVT SrcVT = Op.getValueType();
3302 EVT DstVT = Node->getValueType(ResNo: 0);
3303 if (SrcVT.getScalarType() == MVT::bf16) {
3304 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3305 break;
3306 }
3307
3308 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3309 Results.push_back(Elt: Tmp1);
3310 break;
3311 }
3312 case ISD::BF16_TO_FP: {
3313 // Always expand bf16 to f32 casts, they lower to ext + shift.
3314 //
3315 // Note that the operand of this code can be bf16 or an integer type in case
3316 // bf16 is not supported on the target and was softened.
3317 SDValue Op = Node->getOperand(Num: 0);
3318 if (Op.getValueType() == MVT::bf16) {
3319 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3320 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3321 } else {
3322 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3323 }
3324 Op = DAG.getNode(
3325 Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3326 N2: DAG.getConstant(Val: 16, DL: dl,
3327 VT: TLI.getShiftAmountTy(LHSTy: MVT::i32, DL: DAG.getDataLayout())));
3328 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3329 // Add fp_extend in case the output is bigger than f32.
3330 if (Node->getValueType(ResNo: 0) != MVT::f32)
3331 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3332 Results.push_back(Elt: Op);
3333 break;
3334 }
3335 case ISD::FP_TO_BF16: {
3336 SDValue Op = Node->getOperand(Num: 0);
3337 if (Op.getValueType() != MVT::f32)
3338 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3339 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3340 // Certain SNaNs will turn into infinities if we do a simple shift right.
3341 if (!DAG.isKnownNeverSNaN(Op)) {
3342 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3343 }
3344 Op = DAG.getNode(
3345 Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3346 N2: DAG.getConstant(Val: 16, DL: dl,
3347 VT: TLI.getShiftAmountTy(LHSTy: MVT::i32, DL: DAG.getDataLayout())));
3348 // The result of this node can be bf16 or an integer type in case bf16 is
3349 // not supported on the target and was softened to i16 for storage.
3350 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3351 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3352 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3353 } else {
3354 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3355 }
3356 Results.push_back(Elt: Op);
3357 break;
3358 }
3359 case ISD::SIGN_EXTEND_INREG: {
3360 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3361 EVT VT = Node->getValueType(ResNo: 0);
3362
3363 // An in-register sign-extend of a boolean is a negation:
3364 // 'true' (1) sign-extended is -1.
3365 // 'false' (0) sign-extended is 0.
3366 // However, we must mask the high bits of the source operand because the
3367 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3368
3369 // TODO: Do this for vectors too?
3370 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3371 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3372 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3373 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3374 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3375 Results.push_back(Elt: Neg);
3376 break;
3377 }
3378
3379 // NOTE: we could fall back on load/store here too for targets without
3380 // SRA. However, it is doubtful that any exist.
3381 EVT ShiftAmountTy = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3382 unsigned BitsDiff = VT.getScalarSizeInBits() -
3383 ExtraVT.getScalarSizeInBits();
3384 SDValue ShiftCst = DAG.getConstant(Val: BitsDiff, DL: dl, VT: ShiftAmountTy);
3385 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Node->getValueType(ResNo: 0),
3386 N1: Node->getOperand(Num: 0), N2: ShiftCst);
3387 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: ShiftCst);
3388 Results.push_back(Elt: Tmp1);
3389 break;
3390 }
3391 case ISD::UINT_TO_FP:
3392 case ISD::STRICT_UINT_TO_FP:
3393 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3394 Results.push_back(Elt: Tmp1);
3395 if (Node->isStrictFPOpcode())
3396 Results.push_back(Elt: Tmp2);
3397 break;
3398 }
3399 [[fallthrough]];
3400 case ISD::SINT_TO_FP:
3401 case ISD::STRICT_SINT_TO_FP:
3402 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3403 Results.push_back(Elt: Tmp1);
3404 if (Node->isStrictFPOpcode())
3405 Results.push_back(Elt: Tmp2);
3406 }
3407 break;
3408 case ISD::FP_TO_SINT:
3409 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3410 Results.push_back(Elt: Tmp1);
3411 break;
3412 case ISD::STRICT_FP_TO_SINT:
3413 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3414 ReplaceNode(Old: Node, New: Tmp1.getNode());
3415 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3416 return true;
3417 }
3418 break;
3419 case ISD::FP_TO_UINT:
3420 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3421 Results.push_back(Elt: Tmp1);
3422 break;
3423 case ISD::STRICT_FP_TO_UINT:
3424 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3425 // Relink the chain.
3426 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3427 // Replace the new UINT result.
3428 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3429 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3430 return true;
3431 }
3432 break;
3433 case ISD::FP_TO_SINT_SAT:
3434 case ISD::FP_TO_UINT_SAT:
3435 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3436 break;
3437 case ISD::LROUND:
3438 case ISD::LLROUND: {
3439 SDValue Arg = Node->getOperand(Num: 0);
3440 EVT ArgVT = Arg.getValueType();
3441 EVT ResVT = Node->getValueType(ResNo: 0);
3442 SDLoc dl(Node);
3443 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3444 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3445 break;
3446 }
3447 case ISD::VAARG:
3448 Results.push_back(Elt: DAG.expandVAArg(Node));
3449 Results.push_back(Elt: Results[0].getValue(R: 1));
3450 break;
3451 case ISD::VACOPY:
3452 Results.push_back(Elt: DAG.expandVACopy(Node));
3453 break;
3454 case ISD::EXTRACT_VECTOR_ELT:
3455 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3456 // This must be an access of the only element. Return it.
3457 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3458 Operand: Node->getOperand(Num: 0));
3459 else
3460 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3461 Results.push_back(Elt: Tmp1);
3462 break;
3463 case ISD::EXTRACT_SUBVECTOR:
3464 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3465 break;
3466 case ISD::INSERT_SUBVECTOR:
3467 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3468 break;
3469 case ISD::CONCAT_VECTORS:
3470 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3471 VectorValueType.isScalableVector() ||
3472 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3473 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3474 else
3475 Results.push_back(Elt: ExpandConcatVectors(Node));
3476 break;
3477 case ISD::SCALAR_TO_VECTOR:
3478 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3479 break;
3480 case ISD::INSERT_VECTOR_ELT:
3481 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3482 break;
3483 case ISD::VECTOR_SHUFFLE: {
3484 SmallVector<int, 32> NewMask;
3485 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3486
3487 EVT VT = Node->getValueType(ResNo: 0);
3488 EVT EltVT = VT.getVectorElementType();
3489 SDValue Op0 = Node->getOperand(Num: 0);
3490 SDValue Op1 = Node->getOperand(Num: 1);
3491 if (!TLI.isTypeLegal(VT: EltVT)) {
3492 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3493
3494 // BUILD_VECTOR operands are allowed to be wider than the element type.
3495 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3496 // it.
3497 if (NewEltVT.bitsLT(VT: EltVT)) {
3498 // Convert shuffle node.
3499 // If original node was v4i64 and the new EltVT is i32,
3500 // cast operands to v8i32 and re-build the mask.
3501
3502 // Calculate new VT, the size of the new VT should be equal to original.
3503 EVT NewVT =
3504 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3505 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3506 assert(NewVT.bitsEq(VT));
3507
3508 // cast operands to new VT
3509 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3510 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3511
3512 // Convert the shuffle mask
3513 unsigned int factor =
3514 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3515
3516 // EltVT gets smaller
3517 assert(factor > 0);
3518
3519 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3520 if (Mask[i] < 0) {
3521 for (unsigned fi = 0; fi < factor; ++fi)
3522 NewMask.push_back(Elt: Mask[i]);
3523 }
3524 else {
3525 for (unsigned fi = 0; fi < factor; ++fi)
3526 NewMask.push_back(Elt: Mask[i]*factor+fi);
3527 }
3528 }
3529 Mask = NewMask;
3530 VT = NewVT;
3531 }
3532 EltVT = NewEltVT;
3533 }
3534 unsigned NumElems = VT.getVectorNumElements();
3535 SmallVector<SDValue, 16> Ops;
3536 for (unsigned i = 0; i != NumElems; ++i) {
3537 if (Mask[i] < 0) {
3538 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3539 continue;
3540 }
3541 unsigned Idx = Mask[i];
3542 if (Idx < NumElems)
3543 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3544 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3545 else
3546 Ops.push_back(
3547 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3548 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3549 }
3550
3551 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3552 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3553 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3554 Results.push_back(Elt: Tmp1);
3555 break;
3556 }
3557 case ISD::VECTOR_SPLICE: {
3558 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3559 break;
3560 }
3561 case ISD::VECTOR_DEINTERLEAVE: {
3562 unsigned Factor = Node->getNumOperands();
3563 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
3564 break;
3565 SmallVector<SDValue, 8> Ops(Node->ops());
3566 EVT VecVT = Node->getValueType(ResNo: 0);
3567 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3568 // Deinterleave at Factor/2 so each result contains two factors interleaved:
3569 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
3570 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3571 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
3572 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3573 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
3574 Results.resize(N: Factor);
3575 // Deinterleave the 2 factors out:
3576 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
3577 for (unsigned I = 0; I < Factor / 2; I++) {
3578 SDValue Deinterleave =
3579 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3580 Ops: {L.getValue(R: I), R.getValue(R: I)});
3581 Results[I] = Deinterleave.getValue(R: 0);
3582 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
3583 }
3584 break;
3585 }
3586 case ISD::VECTOR_INTERLEAVE: {
3587 unsigned Factor = Node->getNumOperands();
3588 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
3589 break;
3590 EVT VecVT = Node->getValueType(ResNo: 0);
3591 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3592 SmallVector<SDValue, 8> LOps, ROps;
3593 // Interleave so we have 2 factors per result:
3594 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
3595 for (unsigned I = 0; I < Factor / 2; I++) {
3596 SDValue Interleave =
3597 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3598 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
3599 LOps.push_back(Elt: Interleave.getValue(R: 0));
3600 ROps.push_back(Elt: Interleave.getValue(R: 1));
3601 }
3602 // Interleave at Factor/2:
3603 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
3604 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: LOps);
3605 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: ROps);
3606 for (unsigned I = 0; I < Factor / 2; I++)
3607 Results.push_back(Elt: L.getValue(R: I));
3608 for (unsigned I = 0; I < Factor / 2; I++)
3609 Results.push_back(Elt: R.getValue(R: I));
3610 break;
3611 }
3612 case ISD::EXTRACT_ELEMENT: {
3613 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3614 if (Node->getConstantOperandVal(Num: 1)) {
3615 // 1 -> Hi
3616 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3617 N2: DAG.getConstant(Val: OpTy.getSizeInBits() / 2, DL: dl,
3618 VT: TLI.getShiftAmountTy(
3619 LHSTy: Node->getOperand(Num: 0).getValueType(),
3620 DL: DAG.getDataLayout())));
3621 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3622 } else {
3623 // 0 -> Lo
3624 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3625 Operand: Node->getOperand(Num: 0));
3626 }
3627 Results.push_back(Elt: Tmp1);
3628 break;
3629 }
3630 case ISD::STACKSAVE:
3631 // Expand to CopyFromReg if the target set
3632 // StackPointerRegisterToSaveRestore.
3633 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3634 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3635 VT: Node->getValueType(ResNo: 0)));
3636 Results.push_back(Elt: Results[0].getValue(R: 1));
3637 } else {
3638 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3639 Results.push_back(Elt: Node->getOperand(Num: 0));
3640 }
3641 break;
3642 case ISD::STACKRESTORE:
3643 // Expand to CopyToReg if the target set
3644 // StackPointerRegisterToSaveRestore.
3645 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3646 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3647 N: Node->getOperand(Num: 1)));
3648 } else {
3649 Results.push_back(Elt: Node->getOperand(Num: 0));
3650 }
3651 break;
3652 case ISD::GET_DYNAMIC_AREA_OFFSET:
3653 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3654 Results.push_back(Elt: Results[0].getValue(R: 0));
3655 break;
3656 case ISD::FCOPYSIGN:
3657 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3658 break;
3659 case ISD::FNEG:
3660 Results.push_back(Elt: ExpandFNEG(Node));
3661 break;
3662 case ISD::FABS:
3663 Results.push_back(Elt: ExpandFABS(Node));
3664 break;
3665 case ISD::IS_FPCLASS: {
3666 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3667 if (SDValue Expanded =
3668 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3669 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3670 Results.push_back(Elt: Expanded);
3671 break;
3672 }
3673 case ISD::SMIN:
3674 case ISD::SMAX:
3675 case ISD::UMIN:
3676 case ISD::UMAX: {
3677 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3678 ISD::CondCode Pred;
3679 switch (Node->getOpcode()) {
3680 default: llvm_unreachable("How did we get here?");
3681 case ISD::SMAX: Pred = ISD::SETGT; break;
3682 case ISD::SMIN: Pred = ISD::SETLT; break;
3683 case ISD::UMAX: Pred = ISD::SETUGT; break;
3684 case ISD::UMIN: Pred = ISD::SETULT; break;
3685 }
3686 Tmp1 = Node->getOperand(Num: 0);
3687 Tmp2 = Node->getOperand(Num: 1);
3688 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3689 Results.push_back(Elt: Tmp1);
3690 break;
3691 }
3692 case ISD::FMINNUM:
3693 case ISD::FMAXNUM: {
3694 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3695 Results.push_back(Elt: Expanded);
3696 break;
3697 }
3698 case ISD::FMINIMUM:
3699 case ISD::FMAXIMUM: {
3700 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
3701 Results.push_back(Elt: Expanded);
3702 break;
3703 }
3704 case ISD::FMINIMUMNUM:
3705 case ISD::FMAXIMUMNUM: {
3706 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
3707 break;
3708 }
3709 case ISD::FSIN:
3710 case ISD::FCOS: {
3711 EVT VT = Node->getValueType(ResNo: 0);
3712 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3713 // fcos which share the same operand and both are used.
3714 if ((TLI.isOperationLegalOrCustom(Op: ISD::FSINCOS, VT) ||
3715 isSinCosLibcallAvailable(Node, TLI))
3716 && useSinCos(Node)) {
3717 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3718 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3719 if (Node->getOpcode() == ISD::FCOS)
3720 Tmp1 = Tmp1.getValue(R: 1);
3721 Results.push_back(Elt: Tmp1);
3722 }
3723 break;
3724 }
3725 case ISD::FLDEXP:
3726 case ISD::STRICT_FLDEXP: {
3727 EVT VT = Node->getValueType(ResNo: 0);
3728 RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT);
3729 // Use the LibCall instead, it is very likely faster
3730 // FIXME: Use separate LibCall action.
3731 if (TLI.getLibcallName(Call: LC))
3732 break;
3733
3734 if (SDValue Expanded = expandLdexp(Node)) {
3735 Results.push_back(Elt: Expanded);
3736 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3737 Results.push_back(Elt: Expanded.getValue(R: 1));
3738 }
3739
3740 break;
3741 }
3742 case ISD::FFREXP: {
3743 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0));
3744 // Use the LibCall instead, it is very likely faster
3745 // FIXME: Use separate LibCall action.
3746 if (TLI.getLibcallName(Call: LC))
3747 break;
3748
3749 if (SDValue Expanded = expandFrexp(Node)) {
3750 Results.push_back(Elt: Expanded);
3751 Results.push_back(Elt: Expanded.getValue(R: 1));
3752 }
3753 break;
3754 }
3755 case ISD::FSINCOS: {
3756 if (isSinCosLibcallAvailable(Node, TLI))
3757 break;
3758 EVT VT = Node->getValueType(ResNo: 0);
3759 SDValue Op = Node->getOperand(Num: 0);
3760 SDNodeFlags Flags = Node->getFlags();
3761 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
3762 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
3763 Results.append(IL: {Tmp1, Tmp2});
3764 break;
3765 }
3766 case ISD::FMAD:
3767 llvm_unreachable("Illegal fmad should never be formed");
3768
3769 case ISD::FP16_TO_FP:
3770 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3771 // We can extend to types bigger than f32 in two steps without changing
3772 // the result. Since "f16 -> f32" is much more commonly available, give
3773 // CodeGen the option of emitting that before resorting to a libcall.
3774 SDValue Res =
3775 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
3776 Results.push_back(
3777 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
3778 }
3779 break;
3780 case ISD::STRICT_BF16_TO_FP:
3781 case ISD::STRICT_FP16_TO_FP:
3782 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3783 // We can extend to types bigger than f32 in two steps without changing
3784 // the result. Since "f16 -> f32" is much more commonly available, give
3785 // CodeGen the option of emitting that before resorting to a libcall.
3786 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
3787 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
3788 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
3789 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
3790 Ops: {Res.getValue(R: 1), Res});
3791 Results.push_back(Elt: Res);
3792 Results.push_back(Elt: Res.getValue(R: 1));
3793 }
3794 break;
3795 case ISD::FP_TO_FP16:
3796 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3797 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3798 SDValue Op = Node->getOperand(Num: 0);
3799 MVT SVT = Op.getSimpleValueType();
3800 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3801 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
3802 // Under fastmath, we can expand this node into a fround followed by
3803 // a float-half conversion.
3804 SDValue FloatVal =
3805 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3806 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3807 Results.push_back(
3808 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
3809 }
3810 }
3811 break;
3812 case ISD::ConstantFP: {
3813 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
3814 // Check to see if this FP immediate is already legal.
3815 // If this is a legal constant, turn it into a TargetConstantFP node.
3816 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
3817 ForCodeSize: DAG.shouldOptForSize()))
3818 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
3819 break;
3820 }
3821 case ISD::Constant: {
3822 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
3823 Results.push_back(Elt: ExpandConstant(CP));
3824 break;
3825 }
3826 case ISD::FSUB: {
3827 EVT VT = Node->getValueType(ResNo: 0);
3828 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
3829 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
3830 const SDNodeFlags Flags = Node->getFlags();
3831 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
3832 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
3833 Results.push_back(Elt: Tmp1);
3834 }
3835 break;
3836 }
3837 case ISD::SUB: {
3838 EVT VT = Node->getValueType(ResNo: 0);
3839 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3840 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3841 "Don't know how to expand this subtraction!");
3842 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
3843 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3844 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
3845 break;
3846 }
3847 case ISD::UREM:
3848 case ISD::SREM:
3849 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
3850 Results.push_back(Elt: Tmp1);
3851 break;
3852 case ISD::UDIV:
3853 case ISD::SDIV: {
3854 bool isSigned = Node->getOpcode() == ISD::SDIV;
3855 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3856 EVT VT = Node->getValueType(ResNo: 0);
3857 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
3858 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3859 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3860 N2: Node->getOperand(Num: 1));
3861 Results.push_back(Elt: Tmp1);
3862 }
3863 break;
3864 }
3865 case ISD::MULHU:
3866 case ISD::MULHS: {
3867 unsigned ExpandOpcode =
3868 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3869 EVT VT = Node->getValueType(ResNo: 0);
3870 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3871
3872 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3873 N2: Node->getOperand(Num: 1));
3874 Results.push_back(Elt: Tmp1.getValue(R: 1));
3875 break;
3876 }
3877 case ISD::UMUL_LOHI:
3878 case ISD::SMUL_LOHI: {
3879 SDValue LHS = Node->getOperand(Num: 0);
3880 SDValue RHS = Node->getOperand(Num: 1);
3881 MVT VT = LHS.getSimpleValueType();
3882 unsigned MULHOpcode =
3883 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3884
3885 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
3886 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
3887 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
3888 break;
3889 }
3890
3891 SmallVector<SDValue, 4> Halves;
3892 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(Context&: *DAG.getContext());
3893 assert(TLI.isTypeLegal(HalfType));
3894 if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
3895 HiLoVT: HalfType, DAG,
3896 Kind: TargetLowering::MulExpansionKind::Always)) {
3897 for (unsigned i = 0; i < 2; ++i) {
3898 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
3899 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
3900 SDValue Shift = DAG.getConstant(
3901 Val: HalfType.getScalarSizeInBits(), DL: dl,
3902 VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout()));
3903 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
3904 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
3905 }
3906 break;
3907 }
3908 break;
3909 }
3910 case ISD::MUL: {
3911 EVT VT = Node->getValueType(ResNo: 0);
3912 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3913 // See if multiply or divide can be lowered using two-result operations.
3914 // We just need the low half of the multiply; try both the signed
3915 // and unsigned forms. If the target supports both SMUL_LOHI and
3916 // UMUL_LOHI, form a preference by checking which forms of plain
3917 // MULH it supports.
3918 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
3919 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
3920 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
3921 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
3922 unsigned OpToUse = 0;
3923 if (HasSMUL_LOHI && !HasMULHS) {
3924 OpToUse = ISD::SMUL_LOHI;
3925 } else if (HasUMUL_LOHI && !HasMULHU) {
3926 OpToUse = ISD::UMUL_LOHI;
3927 } else if (HasSMUL_LOHI) {
3928 OpToUse = ISD::SMUL_LOHI;
3929 } else if (HasUMUL_LOHI) {
3930 OpToUse = ISD::UMUL_LOHI;
3931 }
3932 if (OpToUse) {
3933 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3934 N2: Node->getOperand(Num: 1)));
3935 break;
3936 }
3937
3938 SDValue Lo, Hi;
3939 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
3940 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
3941 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
3942 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
3943 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
3944 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
3945 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3946 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
3947 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
3948 SDValue Shift =
3949 DAG.getConstant(Val: HalfType.getSizeInBits(), DL: dl,
3950 VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout()));
3951 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
3952 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
3953 }
3954 break;
3955 }
3956 case ISD::FSHL:
3957 case ISD::FSHR:
3958 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
3959 Results.push_back(Elt: Expanded);
3960 break;
3961 case ISD::ROTL:
3962 case ISD::ROTR:
3963 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
3964 Results.push_back(Elt: Expanded);
3965 break;
3966 case ISD::SADDSAT:
3967 case ISD::UADDSAT:
3968 case ISD::SSUBSAT:
3969 case ISD::USUBSAT:
3970 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
3971 break;
3972 case ISD::SCMP:
3973 case ISD::UCMP:
3974 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
3975 break;
3976 case ISD::SSHLSAT:
3977 case ISD::USHLSAT:
3978 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
3979 break;
3980 case ISD::SMULFIX:
3981 case ISD::SMULFIXSAT:
3982 case ISD::UMULFIX:
3983 case ISD::UMULFIXSAT:
3984 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
3985 break;
3986 case ISD::SDIVFIX:
3987 case ISD::SDIVFIXSAT:
3988 case ISD::UDIVFIX:
3989 case ISD::UDIVFIXSAT:
3990 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
3991 LHS: Node->getOperand(Num: 0),
3992 RHS: Node->getOperand(Num: 1),
3993 Scale: Node->getConstantOperandVal(Num: 2),
3994 DAG)) {
3995 Results.push_back(Elt: V);
3996 break;
3997 }
3998 // FIXME: We might want to retry here with a wider type if we fail, if that
3999 // type is legal.
4000 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4001 // <= 128 (which is the case for all of the default Embedded-C types),
4002 // we will only get here with types and scales that we could always expand
4003 // if we were allowed to generate libcalls to division functions of illegal
4004 // type. But we cannot do that.
4005 llvm_unreachable("Cannot expand DIVFIX!");
4006 case ISD::UADDO_CARRY:
4007 case ISD::USUBO_CARRY: {
4008 SDValue LHS = Node->getOperand(Num: 0);
4009 SDValue RHS = Node->getOperand(Num: 1);
4010 SDValue Carry = Node->getOperand(Num: 2);
4011
4012 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4013
4014 // Initial add of the 2 operands.
4015 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4016 EVT VT = LHS.getValueType();
4017 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4018
4019 // Initial check for overflow.
4020 EVT CarryType = Node->getValueType(ResNo: 1);
4021 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4022 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4023 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4024
4025 // Add of the sum and the carry.
4026 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4027 SDValue CarryExt =
4028 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4029 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4030
4031 // Second check for overflow. If we are adding, we can only overflow if the
4032 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4033 // If we are subtracting, we can only overflow if the initial sum is 0 and
4034 // the carry is set, resulting in a new sum of all 1s.
4035 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4036 SDValue Overflow2 =
4037 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4038 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4039 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4040 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4041
4042 SDValue ResultCarry =
4043 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4044
4045 Results.push_back(Elt: Sum2);
4046 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4047 break;
4048 }
4049 case ISD::SADDO:
4050 case ISD::SSUBO: {
4051 SDValue Result, Overflow;
4052 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4053 Results.push_back(Elt: Result);
4054 Results.push_back(Elt: Overflow);
4055 break;
4056 }
4057 case ISD::UADDO:
4058 case ISD::USUBO: {
4059 SDValue Result, Overflow;
4060 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4061 Results.push_back(Elt: Result);
4062 Results.push_back(Elt: Overflow);
4063 break;
4064 }
4065 case ISD::UMULO:
4066 case ISD::SMULO: {
4067 SDValue Result, Overflow;
4068 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4069 Results.push_back(Elt: Result);
4070 Results.push_back(Elt: Overflow);
4071 }
4072 break;
4073 }
4074 case ISD::BUILD_PAIR: {
4075 EVT PairTy = Node->getValueType(ResNo: 0);
4076 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4077 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4078 Tmp2 = DAG.getNode(
4079 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4080 N2: DAG.getConstant(Val: PairTy.getSizeInBits() / 2, DL: dl,
4081 VT: TLI.getShiftAmountTy(LHSTy: PairTy, DL: DAG.getDataLayout())));
4082 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4083 break;
4084 }
4085 case ISD::SELECT:
4086 Tmp1 = Node->getOperand(Num: 0);
4087 Tmp2 = Node->getOperand(Num: 1);
4088 Tmp3 = Node->getOperand(Num: 2);
4089 if (Tmp1.getOpcode() == ISD::SETCC) {
4090 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1),
4091 True: Tmp2, False: Tmp3,
4092 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get());
4093 } else {
4094 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1,
4095 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4096 True: Tmp2, False: Tmp3, Cond: ISD::SETNE);
4097 }
4098 Tmp1->setFlags(Node->getFlags());
4099 Results.push_back(Elt: Tmp1);
4100 break;
4101 case ISD::BR_JT: {
4102 SDValue Chain = Node->getOperand(Num: 0);
4103 SDValue Table = Node->getOperand(Num: 1);
4104 SDValue Index = Node->getOperand(Num: 2);
4105 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4106
4107 const DataLayout &TD = DAG.getDataLayout();
4108 EVT PTy = TLI.getPointerTy(DL: TD);
4109
4110 unsigned EntrySize =
4111 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4112
4113 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4114 // This transformation needs to be done here since otherwise the MIPS
4115 // backend will end up emitting a three instruction multiply sequence
4116 // instead of a single shift and MSP430 will call a runtime function.
4117 if (llvm::isPowerOf2_32(Value: EntrySize))
4118 Index = DAG.getNode(
4119 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4120 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4121 else
4122 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4123 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4124 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4125
4126 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4127 SDValue LD = DAG.getExtLoad(
4128 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4129 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4130 Addr = LD;
4131 if (TLI.isJumpTableRelative()) {
4132 // For PIC, the sequence is:
4133 // BRIND(RelocBase + load(Jumptable + index))
4134 // RelocBase can be JumpTable, GOT or some sort of global base.
4135 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4136 Offset: Addr, DL: dl);
4137 }
4138
4139 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4140 Results.push_back(Elt: Tmp1);
4141 break;
4142 }
4143 case ISD::BRCOND:
4144 // Expand brcond's setcc into its constituent parts and create a BR_CC
4145 // Node.
4146 Tmp1 = Node->getOperand(Num: 0);
4147 Tmp2 = Node->getOperand(Num: 1);
4148 if (Tmp2.getOpcode() == ISD::SETCC &&
4149 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4150 VT: Tmp2.getOperand(i: 0).getValueType())) {
4151 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4152 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4153 N5: Node->getOperand(Num: 2));
4154 } else {
4155 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4156 if (Tmp2.isUndef() ||
4157 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4158 Tmp3 = Tmp2;
4159 else
4160 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4161 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4162 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4163 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4164 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4165 N5: Node->getOperand(Num: 2));
4166 }
4167 Results.push_back(Elt: Tmp1);
4168 break;
4169 case ISD::SETCC:
4170 case ISD::VP_SETCC:
4171 case ISD::STRICT_FSETCC:
4172 case ISD::STRICT_FSETCCS: {
4173 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4174 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4175 Node->getOpcode() == ISD::STRICT_FSETCCS;
4176 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4177 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4178 unsigned Offset = IsStrict ? 1 : 0;
4179 Tmp1 = Node->getOperand(Num: 0 + Offset);
4180 Tmp2 = Node->getOperand(Num: 1 + Offset);
4181 Tmp3 = Node->getOperand(Num: 2 + Offset);
4182 SDValue Mask, EVL;
4183 if (IsVP) {
4184 Mask = Node->getOperand(Num: 3 + Offset);
4185 EVL = Node->getOperand(Num: 4 + Offset);
4186 }
4187 bool Legalized = TLI.LegalizeSetCCCondCode(
4188 DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl,
4189 Chain, IsSignaling);
4190
4191 if (Legalized) {
4192 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4193 // condition code, create a new SETCC node.
4194 if (Tmp3.getNode()) {
4195 if (IsStrict) {
4196 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4197 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4198 Chain = Tmp1.getValue(R: 1);
4199 } else if (IsVP) {
4200 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0),
4201 Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags());
4202 } else {
4203 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4204 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4205 }
4206 }
4207
4208 // If we expanded the SETCC by inverting the condition code, then wrap
4209 // the existing SETCC in a NOT to restore the intended condition.
4210 if (NeedInvert) {
4211 if (!IsVP)
4212 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4213 else
4214 Tmp1 =
4215 DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0));
4216 }
4217
4218 Results.push_back(Elt: Tmp1);
4219 if (IsStrict)
4220 Results.push_back(Elt: Chain);
4221
4222 break;
4223 }
4224
4225 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4226 // understand if this code is useful for strict nodes.
4227 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4228
4229 // Otherwise, SETCC for the given comparison type must be completely
4230 // illegal; expand it into a SELECT_CC.
4231 // FIXME: This drops the mask/evl for VP_SETCC.
4232 EVT VT = Node->getValueType(ResNo: 0);
4233 EVT Tmp1VT = Tmp1.getValueType();
4234 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4235 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4236 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3);
4237 Tmp1->setFlags(Node->getFlags());
4238 Results.push_back(Elt: Tmp1);
4239 break;
4240 }
4241 case ISD::SELECT_CC: {
4242 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4243 Tmp1 = Node->getOperand(Num: 0); // LHS
4244 Tmp2 = Node->getOperand(Num: 1); // RHS
4245 Tmp3 = Node->getOperand(Num: 2); // True
4246 Tmp4 = Node->getOperand(Num: 3); // False
4247 EVT VT = Node->getValueType(ResNo: 0);
4248 SDValue Chain;
4249 SDValue CC = Node->getOperand(Num: 4);
4250 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4251
4252 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4253 // If the condition code is legal, then we need to expand this
4254 // node using SETCC and SELECT.
4255 EVT CmpVT = Tmp1.getValueType();
4256 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4257 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4258 "expanded.");
4259 EVT CCVT = getSetCCResultType(VT: CmpVT);
4260 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4261 Results.push_back(
4262 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4263 break;
4264 }
4265
4266 // SELECT_CC is legal, so the condition code must not be.
4267 bool Legalized = false;
4268 // Try to legalize by inverting the condition. This is for targets that
4269 // might support an ordered version of a condition, but not the unordered
4270 // version (or vice versa).
4271 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4272 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4273 // Use the new condition code and swap true and false
4274 Legalized = true;
4275 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC);
4276 Tmp1->setFlags(Node->getFlags());
4277 } else {
4278 // If The inverse is not legal, then try to swap the arguments using
4279 // the inverse condition code.
4280 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4281 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4282 // The swapped inverse condition is legal, so swap true and false,
4283 // lhs and rhs.
4284 Legalized = true;
4285 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC);
4286 Tmp1->setFlags(Node->getFlags());
4287 }
4288 }
4289
4290 if (!Legalized) {
4291 Legalized = TLI.LegalizeSetCCCondCode(
4292 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4293 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4294
4295 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4296
4297 // If we expanded the SETCC by inverting the condition code, then swap
4298 // the True/False operands to match.
4299 if (NeedInvert)
4300 std::swap(a&: Tmp3, b&: Tmp4);
4301
4302 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4303 // condition code, create a new SELECT_CC node.
4304 if (CC.getNode()) {
4305 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
4306 N1: Tmp1, N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC);
4307 } else {
4308 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4309 CC = DAG.getCondCode(Cond: ISD::SETNE);
4310 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4311 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC);
4312 }
4313 Tmp1->setFlags(Node->getFlags());
4314 }
4315 Results.push_back(Elt: Tmp1);
4316 break;
4317 }
4318 case ISD::BR_CC: {
4319 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4320 SDValue Chain;
4321 Tmp1 = Node->getOperand(Num: 0); // Chain
4322 Tmp2 = Node->getOperand(Num: 2); // LHS
4323 Tmp3 = Node->getOperand(Num: 3); // RHS
4324 Tmp4 = Node->getOperand(Num: 1); // CC
4325
4326 bool Legalized = TLI.LegalizeSetCCCondCode(
4327 DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4,
4328 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4329 (void)Legalized;
4330 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4331
4332 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4333 // node.
4334 if (Tmp4.getNode()) {
4335 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4336
4337 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4338 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4339 } else {
4340 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4341 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4342 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4343 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4344 }
4345 Results.push_back(Elt: Tmp1);
4346 break;
4347 }
4348 case ISD::BUILD_VECTOR:
4349 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4350 break;
4351 case ISD::SPLAT_VECTOR:
4352 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4353 break;
4354 case ISD::SRA:
4355 case ISD::SRL:
4356 case ISD::SHL: {
4357 // Scalarize vector SRA/SRL/SHL.
4358 EVT VT = Node->getValueType(ResNo: 0);
4359 assert(VT.isVector() && "Unable to legalize non-vector shift");
4360 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4361 unsigned NumElem = VT.getVectorNumElements();
4362
4363 SmallVector<SDValue, 8> Scalars;
4364 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4365 SDValue Ex =
4366 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4367 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4368 SDValue Sh =
4369 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4370 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4371 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4372 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4373 }
4374
4375 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4376 Results.push_back(Elt: Result);
4377 break;
4378 }
4379 case ISD::VECREDUCE_FADD:
4380 case ISD::VECREDUCE_FMUL:
4381 case ISD::VECREDUCE_ADD:
4382 case ISD::VECREDUCE_MUL:
4383 case ISD::VECREDUCE_AND:
4384 case ISD::VECREDUCE_OR:
4385 case ISD::VECREDUCE_XOR:
4386 case ISD::VECREDUCE_SMAX:
4387 case ISD::VECREDUCE_SMIN:
4388 case ISD::VECREDUCE_UMAX:
4389 case ISD::VECREDUCE_UMIN:
4390 case ISD::VECREDUCE_FMAX:
4391 case ISD::VECREDUCE_FMIN:
4392 case ISD::VECREDUCE_FMAXIMUM:
4393 case ISD::VECREDUCE_FMINIMUM:
4394 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4395 break;
4396 case ISD::VP_CTTZ_ELTS:
4397 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4398 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4399 break;
4400 case ISD::CLEAR_CACHE:
4401 // The default expansion of llvm.clear_cache is simply a no-op for those
4402 // targets where it is not needed.
4403 Results.push_back(Elt: Node->getOperand(Num: 0));
4404 break;
4405 case ISD::LRINT:
4406 case ISD::LLRINT: {
4407 SDValue Arg = Node->getOperand(Num: 0);
4408 EVT ArgVT = Arg.getValueType();
4409 EVT ResVT = Node->getValueType(ResNo: 0);
4410 SDLoc dl(Node);
4411 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL: dl, VT: ArgVT, Operand: Arg);
4412 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
4413 break;
4414 }
4415 case ISD::ADDRSPACECAST:
4416 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4417 break;
4418 case ISD::GLOBAL_OFFSET_TABLE:
4419 case ISD::GlobalAddress:
4420 case ISD::GlobalTLSAddress:
4421 case ISD::ExternalSymbol:
4422 case ISD::ConstantPool:
4423 case ISD::JumpTable:
4424 case ISD::INTRINSIC_W_CHAIN:
4425 case ISD::INTRINSIC_WO_CHAIN:
4426 case ISD::INTRINSIC_VOID:
4427 // FIXME: Custom lowering for these operations shouldn't return null!
4428 // Return true so that we don't call ConvertNodeToLibcall which also won't
4429 // do anything.
4430 return true;
4431 }
4432
4433 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4434 // FIXME: We were asked to expand a strict floating-point operation,
4435 // but there is currently no expansion implemented that would preserve
4436 // the "strict" properties. For now, we just fall back to the non-strict
4437 // version if that is legal on the target. The actual mutation of the
4438 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4439 switch (Node->getOpcode()) {
4440 default:
4441 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4442 VT: Node->getValueType(ResNo: 0))
4443 == TargetLowering::Legal)
4444 return true;
4445 break;
4446 case ISD::STRICT_FSUB: {
4447 if (TLI.getStrictFPOperationAction(
4448 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4449 return true;
4450 if (TLI.getStrictFPOperationAction(
4451 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4452 break;
4453
4454 EVT VT = Node->getValueType(ResNo: 0);
4455 const SDNodeFlags Flags = Node->getFlags();
4456 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4457 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4458 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4459 Flags);
4460
4461 Results.push_back(Elt: Fadd);
4462 Results.push_back(Elt: Fadd.getValue(R: 1));
4463 break;
4464 }
4465 case ISD::STRICT_SINT_TO_FP:
4466 case ISD::STRICT_UINT_TO_FP:
4467 case ISD::STRICT_LRINT:
4468 case ISD::STRICT_LLRINT:
4469 case ISD::STRICT_LROUND:
4470 case ISD::STRICT_LLROUND:
4471 // These are registered by the operand type instead of the value
4472 // type. Reflect that here.
4473 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4474 VT: Node->getOperand(Num: 1).getValueType())
4475 == TargetLowering::Legal)
4476 return true;
4477 break;
4478 }
4479 }
4480
4481 // Replace the original node with the legalized result.
4482 if (Results.empty()) {
4483 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4484 return false;
4485 }
4486
4487 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4488 ReplaceNode(Old: Node, New: Results.data());
4489 return true;
4490}
4491
4492void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4493 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4494 SmallVector<SDValue, 8> Results;
4495 SDLoc dl(Node);
4496 TargetLowering::MakeLibCallOptions CallOptions;
4497 CallOptions.IsPostTypeLegalization = true;
4498 // FIXME: Check flags on the node to see if we can use a finite call.
4499 unsigned Opc = Node->getOpcode();
4500 switch (Opc) {
4501 case ISD::ATOMIC_FENCE: {
4502 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4503 // FIXME: handle "fence singlethread" more efficiently.
4504 TargetLowering::ArgListTy Args;
4505
4506 TargetLowering::CallLoweringInfo CLI(DAG);
4507 CLI.setDebugLoc(dl)
4508 .setChain(Node->getOperand(Num: 0))
4509 .setLibCallee(
4510 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4511 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4512 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4513 ArgsList: std::move(Args));
4514
4515 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4516
4517 Results.push_back(Elt: CallResult.second);
4518 break;
4519 }
4520 // By default, atomic intrinsics are marked Legal and lowered. Targets
4521 // which don't support them directly, however, may want libcalls, in which
4522 // case they mark them Expand, and we get here.
4523 case ISD::ATOMIC_SWAP:
4524 case ISD::ATOMIC_LOAD_ADD:
4525 case ISD::ATOMIC_LOAD_SUB:
4526 case ISD::ATOMIC_LOAD_AND:
4527 case ISD::ATOMIC_LOAD_CLR:
4528 case ISD::ATOMIC_LOAD_OR:
4529 case ISD::ATOMIC_LOAD_XOR:
4530 case ISD::ATOMIC_LOAD_NAND:
4531 case ISD::ATOMIC_LOAD_MIN:
4532 case ISD::ATOMIC_LOAD_MAX:
4533 case ISD::ATOMIC_LOAD_UMIN:
4534 case ISD::ATOMIC_LOAD_UMAX:
4535 case ISD::ATOMIC_CMP_SWAP: {
4536 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4537 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4538 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4539 EVT RetVT = Node->getValueType(ResNo: 0);
4540 SmallVector<SDValue, 4> Ops;
4541 if (TLI.getLibcallName(Call: LC)) {
4542 // If outline atomic available, prepare its arguments and expand.
4543 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4544 Ops.push_back(Elt: Node->getOperand(Num: 1));
4545
4546 } else {
4547 LC = RTLIB::getSYNC(Opc, VT);
4548 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4549 "Unexpected atomic op or value type!");
4550 // Arguments for expansion to sync libcall
4551 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4552 }
4553 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4554 Ops, CallOptions,
4555 dl: SDLoc(Node),
4556 Chain: Node->getOperand(Num: 0));
4557 Results.push_back(Elt: Tmp.first);
4558 Results.push_back(Elt: Tmp.second);
4559 break;
4560 }
4561 case ISD::TRAP: {
4562 // If this operation is not supported, lower it to 'abort()' call
4563 TargetLowering::ArgListTy Args;
4564 TargetLowering::CallLoweringInfo CLI(DAG);
4565 CLI.setDebugLoc(dl)
4566 .setChain(Node->getOperand(Num: 0))
4567 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4568 Target: DAG.getExternalSymbol(
4569 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4570 ArgsList: std::move(Args));
4571 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4572
4573 Results.push_back(Elt: CallResult.second);
4574 break;
4575 }
4576 case ISD::CLEAR_CACHE: {
4577 SDValue InputChain = Node->getOperand(Num: 0);
4578 SDValue StartVal = Node->getOperand(Num: 1);
4579 SDValue EndVal = Node->getOperand(Num: 2);
4580 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4581 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
4582 dl: SDLoc(Node), Chain: InputChain);
4583 Results.push_back(Elt: Tmp.second);
4584 break;
4585 }
4586 case ISD::FMINNUM:
4587 case ISD::STRICT_FMINNUM:
4588 ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
4589 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
4590 Call_PPCF128: RTLIB::FMIN_PPCF128, Results);
4591 break;
4592 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4593 // libcall legalization for these nodes, but there is no default expasion for
4594 // these nodes either (see PR63267 for example).
4595 case ISD::FMAXNUM:
4596 case ISD::STRICT_FMAXNUM:
4597 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
4598 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
4599 Call_PPCF128: RTLIB::FMAX_PPCF128, Results);
4600 break;
4601 case ISD::FMINIMUMNUM:
4602 ExpandFPLibCall(Node, Call_F32: RTLIB::FMINIMUM_NUM_F32, Call_F64: RTLIB::FMINIMUM_NUM_F64,
4603 Call_F80: RTLIB::FMINIMUM_NUM_F80, Call_F128: RTLIB::FMINIMUM_NUM_F128,
4604 Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128, Results);
4605 break;
4606 case ISD::FMAXIMUMNUM:
4607 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAXIMUM_NUM_F32, Call_F64: RTLIB::FMAXIMUM_NUM_F64,
4608 Call_F80: RTLIB::FMAXIMUM_NUM_F80, Call_F128: RTLIB::FMAXIMUM_NUM_F128,
4609 Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128, Results);
4610 break;
4611 case ISD::FSQRT:
4612 case ISD::STRICT_FSQRT:
4613 ExpandFPLibCall(Node, Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
4614 Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
4615 Call_PPCF128: RTLIB::SQRT_PPCF128, Results);
4616 break;
4617 case ISD::FCBRT:
4618 ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64,
4619 Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128,
4620 Call_PPCF128: RTLIB::CBRT_PPCF128, Results);
4621 break;
4622 case ISD::FSIN:
4623 case ISD::STRICT_FSIN:
4624 ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
4625 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
4626 Call_PPCF128: RTLIB::SIN_PPCF128, Results);
4627 break;
4628 case ISD::FCOS:
4629 case ISD::STRICT_FCOS:
4630 ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
4631 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
4632 Call_PPCF128: RTLIB::COS_PPCF128, Results);
4633 break;
4634 case ISD::FTAN:
4635 case ISD::STRICT_FTAN:
4636 ExpandFPLibCall(Node, Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
4637 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128, Results);
4638 break;
4639 case ISD::FASIN:
4640 case ISD::STRICT_FASIN:
4641 ExpandFPLibCall(Node, Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
4642 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128, Results);
4643 break;
4644 case ISD::FACOS:
4645 case ISD::STRICT_FACOS:
4646 ExpandFPLibCall(Node, Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
4647 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128, Results);
4648 break;
4649 case ISD::FATAN:
4650 case ISD::STRICT_FATAN:
4651 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
4652 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128, Results);
4653 break;
4654 case ISD::FATAN2:
4655 case ISD::STRICT_FATAN2:
4656 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN2_F32, Call_F64: RTLIB::ATAN2_F64, Call_F80: RTLIB::ATAN2_F80,
4657 Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128, Results);
4658 break;
4659 case ISD::FSINH:
4660 case ISD::STRICT_FSINH:
4661 ExpandFPLibCall(Node, Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
4662 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128, Results);
4663 break;
4664 case ISD::FCOSH:
4665 case ISD::STRICT_FCOSH:
4666 ExpandFPLibCall(Node, Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
4667 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128, Results);
4668 break;
4669 case ISD::FTANH:
4670 case ISD::STRICT_FTANH:
4671 ExpandFPLibCall(Node, Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
4672 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128, Results);
4673 break;
4674 case ISD::FSINCOS:
4675 case ISD::FSINCOSPI: {
4676 EVT VT = Node->getValueType(ResNo: 0);
4677 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4678 ? RTLIB::getSINCOS(RetVT: VT)
4679 : RTLIB::getSINCOSPI(RetVT: VT);
4680 bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results);
4681 if (!Expanded)
4682 llvm_unreachable("Expected scalar FSINCOS[PI] to expand to libcall!");
4683 break;
4684 }
4685 case ISD::FLOG:
4686 case ISD::STRICT_FLOG:
4687 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80,
4688 Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results);
4689 break;
4690 case ISD::FLOG2:
4691 case ISD::STRICT_FLOG2:
4692 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80,
4693 Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results);
4694 break;
4695 case ISD::FLOG10:
4696 case ISD::STRICT_FLOG10:
4697 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80,
4698 Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results);
4699 break;
4700 case ISD::FEXP:
4701 case ISD::STRICT_FEXP:
4702 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80,
4703 Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results);
4704 break;
4705 case ISD::FEXP2:
4706 case ISD::STRICT_FEXP2:
4707 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80,
4708 Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results);
4709 break;
4710 case ISD::FEXP10:
4711 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
4712 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results);
4713 break;
4714 case ISD::FTRUNC:
4715 case ISD::STRICT_FTRUNC:
4716 ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
4717 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
4718 Call_PPCF128: RTLIB::TRUNC_PPCF128, Results);
4719 break;
4720 case ISD::FFLOOR:
4721 case ISD::STRICT_FFLOOR:
4722 ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
4723 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
4724 Call_PPCF128: RTLIB::FLOOR_PPCF128, Results);
4725 break;
4726 case ISD::FCEIL:
4727 case ISD::STRICT_FCEIL:
4728 ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
4729 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
4730 Call_PPCF128: RTLIB::CEIL_PPCF128, Results);
4731 break;
4732 case ISD::FRINT:
4733 case ISD::STRICT_FRINT:
4734 ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
4735 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
4736 Call_PPCF128: RTLIB::RINT_PPCF128, Results);
4737 break;
4738 case ISD::FNEARBYINT:
4739 case ISD::STRICT_FNEARBYINT:
4740 ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32,
4741 Call_F64: RTLIB::NEARBYINT_F64,
4742 Call_F80: RTLIB::NEARBYINT_F80,
4743 Call_F128: RTLIB::NEARBYINT_F128,
4744 Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results);
4745 break;
4746 case ISD::FROUND:
4747 case ISD::STRICT_FROUND:
4748 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32,
4749 Call_F64: RTLIB::ROUND_F64,
4750 Call_F80: RTLIB::ROUND_F80,
4751 Call_F128: RTLIB::ROUND_F128,
4752 Call_PPCF128: RTLIB::ROUND_PPCF128, Results);
4753 break;
4754 case ISD::FROUNDEVEN:
4755 case ISD::STRICT_FROUNDEVEN:
4756 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32,
4757 Call_F64: RTLIB::ROUNDEVEN_F64,
4758 Call_F80: RTLIB::ROUNDEVEN_F80,
4759 Call_F128: RTLIB::ROUNDEVEN_F128,
4760 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results);
4761 break;
4762 case ISD::FLDEXP:
4763 case ISD::STRICT_FLDEXP:
4764 ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80,
4765 Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results);
4766 break;
4767 case ISD::FMODF:
4768 case ISD::FFREXP: {
4769 EVT VT = Node->getValueType(ResNo: 0);
4770 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(RetVT: VT)
4771 : RTLIB::getFREXP(RetVT: VT);
4772 bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results,
4773 /*CallRetResNo=*/0);
4774 if (!Expanded)
4775 llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");
4776 break;
4777 }
4778 case ISD::FPOWI:
4779 case ISD::STRICT_FPOWI: {
4780 RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0));
4781 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4782 if (!TLI.getLibcallName(Call: LC)) {
4783 // Some targets don't have a powi libcall; use pow instead.
4784 if (Node->isStrictFPOpcode()) {
4785 SDValue Exponent =
4786 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
4787 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4788 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
4789 SDValue FPOW =
4790 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
4791 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4792 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
4793 Results.push_back(Elt: FPOW);
4794 Results.push_back(Elt: FPOW.getValue(R: 1));
4795 } else {
4796 SDValue Exponent =
4797 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
4798 Operand: Node->getOperand(Num: 1));
4799 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
4800 VT: Node->getValueType(ResNo: 0),
4801 N1: Node->getOperand(Num: 0), N2: Exponent));
4802 }
4803 break;
4804 }
4805 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4806 bool ExponentHasSizeOfInt =
4807 DAG.getLibInfo().getIntSize() ==
4808 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
4809 if (!ExponentHasSizeOfInt) {
4810 // If the exponent does not match with sizeof(int) a libcall to
4811 // RTLIB::POWI would use the wrong type for the argument.
4812 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
4813 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
4814 break;
4815 }
4816 ExpandFPLibCall(Node, LC, Results);
4817 break;
4818 }
4819 case ISD::FPOW:
4820 case ISD::STRICT_FPOW:
4821 ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80,
4822 Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results);
4823 break;
4824 case ISD::LROUND:
4825 case ISD::STRICT_LROUND:
4826 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
4827 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
4828 Call_F128: RTLIB::LROUND_F128,
4829 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
4830 break;
4831 case ISD::LLROUND:
4832 case ISD::STRICT_LLROUND:
4833 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
4834 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
4835 Call_F128: RTLIB::LLROUND_F128,
4836 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
4837 break;
4838 case ISD::LRINT:
4839 case ISD::STRICT_LRINT:
4840 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
4841 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
4842 Call_F128: RTLIB::LRINT_F128,
4843 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
4844 break;
4845 case ISD::LLRINT:
4846 case ISD::STRICT_LLRINT:
4847 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
4848 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
4849 Call_F128: RTLIB::LLRINT_F128,
4850 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
4851 break;
4852 case ISD::FDIV:
4853 case ISD::STRICT_FDIV:
4854 ExpandFPLibCall(Node, Call_F32: RTLIB::DIV_F32, Call_F64: RTLIB::DIV_F64,
4855 Call_F80: RTLIB::DIV_F80, Call_F128: RTLIB::DIV_F128,
4856 Call_PPCF128: RTLIB::DIV_PPCF128, Results);
4857 break;
4858 case ISD::FREM:
4859 case ISD::STRICT_FREM:
4860 ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
4861 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
4862 Call_PPCF128: RTLIB::REM_PPCF128, Results);
4863 break;
4864 case ISD::FMA:
4865 case ISD::STRICT_FMA:
4866 ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64,
4867 Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128,
4868 Call_PPCF128: RTLIB::FMA_PPCF128, Results);
4869 break;
4870 case ISD::FADD:
4871 case ISD::STRICT_FADD:
4872 ExpandFPLibCall(Node, Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
4873 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
4874 Call_PPCF128: RTLIB::ADD_PPCF128, Results);
4875 break;
4876 case ISD::FMUL:
4877 case ISD::STRICT_FMUL:
4878 ExpandFPLibCall(Node, Call_F32: RTLIB::MUL_F32, Call_F64: RTLIB::MUL_F64,
4879 Call_F80: RTLIB::MUL_F80, Call_F128: RTLIB::MUL_F128,
4880 Call_PPCF128: RTLIB::MUL_PPCF128, Results);
4881 break;
4882 case ISD::FP16_TO_FP:
4883 if (Node->getValueType(ResNo: 0) == MVT::f32) {
4884 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
4885 }
4886 break;
4887 case ISD::STRICT_BF16_TO_FP:
4888 if (Node->getValueType(ResNo: 0) == MVT::f32) {
4889 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4890 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
4891 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
4892 Results.push_back(Elt: Tmp.first);
4893 Results.push_back(Elt: Tmp.second);
4894 }
4895 break;
4896 case ISD::STRICT_FP16_TO_FP: {
4897 if (Node->getValueType(ResNo: 0) == MVT::f32) {
4898 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4899 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
4900 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
4901 Results.push_back(Elt: Tmp.first);
4902 Results.push_back(Elt: Tmp.second);
4903 }
4904 break;
4905 }
4906 case ISD::FP_TO_FP16: {
4907 RTLIB::Libcall LC =
4908 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
4909 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4910 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
4911 break;
4912 }
4913 case ISD::FP_TO_BF16: {
4914 RTLIB::Libcall LC =
4915 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
4916 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4917 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
4918 break;
4919 }
4920 case ISD::STRICT_SINT_TO_FP:
4921 case ISD::STRICT_UINT_TO_FP:
4922 case ISD::SINT_TO_FP:
4923 case ISD::UINT_TO_FP: {
4924 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4925 bool IsStrict = Node->isStrictFPOpcode();
4926 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4927 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4928 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
4929 EVT RVT = Node->getValueType(ResNo: 0);
4930 EVT NVT = EVT();
4931 SDLoc dl(Node);
4932
4933 // Even if the input is legal, no libcall may exactly match, eg. we don't
4934 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4935 // eg: i13 -> fp. Then, look for an appropriate libcall.
4936 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4937 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4938 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4939 ++t) {
4940 NVT = (MVT::SimpleValueType)t;
4941 // The source needs to big enough to hold the operand.
4942 if (NVT.bitsGE(VT: SVT))
4943 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
4944 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
4945 }
4946 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4947
4948 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4949 // Sign/zero extend the argument if the libcall takes a larger type.
4950 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
4951 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
4952 CallOptions.setIsSigned(Signed);
4953 std::pair<SDValue, SDValue> Tmp =
4954 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
4955 Results.push_back(Elt: Tmp.first);
4956 if (IsStrict)
4957 Results.push_back(Elt: Tmp.second);
4958 break;
4959 }
4960 case ISD::FP_TO_SINT:
4961 case ISD::FP_TO_UINT:
4962 case ISD::STRICT_FP_TO_SINT:
4963 case ISD::STRICT_FP_TO_UINT: {
4964 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4965 bool IsStrict = Node->isStrictFPOpcode();
4966 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4967 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4968
4969 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
4970 EVT SVT = Op.getValueType();
4971 EVT RVT = Node->getValueType(ResNo: 0);
4972 EVT NVT = EVT();
4973 SDLoc dl(Node);
4974
4975 // Even if the result is legal, no libcall may exactly match, eg. we don't
4976 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4977 // eg: fp -> i32. Then, look for an appropriate libcall.
4978 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4979 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4980 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4981 ++IntVT) {
4982 NVT = (MVT::SimpleValueType)IntVT;
4983 // The type needs to big enough to hold the result.
4984 if (NVT.bitsGE(VT: RVT))
4985 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
4986 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
4987 }
4988 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4989
4990 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4991 std::pair<SDValue, SDValue> Tmp =
4992 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
4993
4994 // Truncate the result if the libcall returns a larger type.
4995 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
4996 if (IsStrict)
4997 Results.push_back(Elt: Tmp.second);
4998 break;
4999 }
5000
5001 case ISD::FP_ROUND:
5002 case ISD::STRICT_FP_ROUND: {
5003 // X = FP_ROUND(Y, TRUNC)
5004 // TRUNC is a flag, which is always an integer that is zero or one.
5005 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5006 // is known to not change the value of Y.
5007 // We can only expand it into libcall if the TRUNC is 0.
5008 bool IsStrict = Node->isStrictFPOpcode();
5009 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5010 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5011 EVT VT = Node->getValueType(ResNo: 0);
5012 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5013 "Unable to expand as libcall if it is not normal rounding");
5014
5015 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5016 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5017
5018 std::pair<SDValue, SDValue> Tmp =
5019 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5020 Results.push_back(Elt: Tmp.first);
5021 if (IsStrict)
5022 Results.push_back(Elt: Tmp.second);
5023 break;
5024 }
5025 case ISD::FP_EXTEND: {
5026 Results.push_back(
5027 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5028 RetVT: Node->getValueType(ResNo: 0)),
5029 Node, isSigned: false).first);
5030 break;
5031 }
5032 case ISD::STRICT_FP_EXTEND:
5033 case ISD::STRICT_FP_TO_FP16:
5034 case ISD::STRICT_FP_TO_BF16: {
5035 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5036 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5037 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5038 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5039 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5040 else
5041 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5042 RetVT: Node->getValueType(ResNo: 0));
5043
5044 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5045
5046 std::pair<SDValue, SDValue> Tmp =
5047 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5048 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5049 Results.push_back(Elt: Tmp.first);
5050 Results.push_back(Elt: Tmp.second);
5051 break;
5052 }
5053 case ISD::FSUB:
5054 case ISD::STRICT_FSUB:
5055 ExpandFPLibCall(Node, Call_F32: RTLIB::SUB_F32, Call_F64: RTLIB::SUB_F64,
5056 Call_F80: RTLIB::SUB_F80, Call_F128: RTLIB::SUB_F128,
5057 Call_PPCF128: RTLIB::SUB_PPCF128, Results);
5058 break;
5059 case ISD::SREM:
5060 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5061 Call_I8: RTLIB::SREM_I8,
5062 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5063 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5064 break;
5065 case ISD::UREM:
5066 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5067 Call_I8: RTLIB::UREM_I8,
5068 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5069 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5070 break;
5071 case ISD::SDIV:
5072 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5073 Call_I8: RTLIB::SDIV_I8,
5074 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5075 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5076 break;
5077 case ISD::UDIV:
5078 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5079 Call_I8: RTLIB::UDIV_I8,
5080 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5081 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5082 break;
5083 case ISD::SDIVREM:
5084 case ISD::UDIVREM:
5085 // Expand into divrem libcall
5086 ExpandDivRemLibCall(Node, Results);
5087 break;
5088 case ISD::MUL:
5089 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5090 Call_I8: RTLIB::MUL_I8,
5091 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5092 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5093 break;
5094 case ISD::CTLZ_ZERO_UNDEF:
5095 Results.push_back(Elt: ExpandBitCountingLibCall(
5096 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5097 break;
5098 case ISD::CTPOP:
5099 Results.push_back(Elt: ExpandBitCountingLibCall(
5100 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5101 break;
5102 case ISD::RESET_FPENV: {
5103 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5104 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5105 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5106 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5107 SDValue Chain = Node->getOperand(Num: 0);
5108 Results.push_back(
5109 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
5110 break;
5111 }
5112 case ISD::GET_FPENV_MEM: {
5113 SDValue Chain = Node->getOperand(Num: 0);
5114 SDValue EnvPtr = Node->getOperand(Num: 1);
5115 Results.push_back(
5116 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5117 break;
5118 }
5119 case ISD::SET_FPENV_MEM: {
5120 SDValue Chain = Node->getOperand(Num: 0);
5121 SDValue EnvPtr = Node->getOperand(Num: 1);
5122 Results.push_back(
5123 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5124 break;
5125 }
5126 case ISD::GET_FPMODE: {
5127 // Call fegetmode, which saves control modes into a stack slot. Then load
5128 // the value to return from the stack.
5129 EVT ModeVT = Node->getValueType(ResNo: 0);
5130 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5131 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5132 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5133 InChain: Node->getOperand(Num: 0), DLoc: dl);
5134 SDValue LdInst = DAG.getLoad(
5135 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5136 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5137 Results.push_back(Elt: LdInst);
5138 Results.push_back(Elt: LdInst.getValue(R: 1));
5139 break;
5140 }
5141 case ISD::SET_FPMODE: {
5142 // Move control modes to stack slot and then call fesetmode with the pointer
5143 // to the slot as argument.
5144 SDValue Mode = Node->getOperand(Num: 1);
5145 EVT ModeVT = Mode.getValueType();
5146 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5147 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5148 SDValue StInst = DAG.getStore(
5149 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5150 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5151 Results.push_back(
5152 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
5153 break;
5154 }
5155 case ISD::RESET_FPMODE: {
5156 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5157 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5158 // target must provide custom lowering.
5159 const DataLayout &DL = DAG.getDataLayout();
5160 EVT PtrTy = TLI.getPointerTy(DL);
5161 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5162 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5163 InChain: Node->getOperand(Num: 0), DLoc: dl));
5164 break;
5165 }
5166 }
5167
5168 // Replace the original node with the legalized result.
5169 if (!Results.empty()) {
5170 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5171 ReplaceNode(Old: Node, New: Results.data());
5172 } else
5173 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5174}
5175
5176// Determine the vector type to use in place of an original scalar element when
5177// promoting equally sized vectors.
5178static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5179 MVT EltVT, MVT NewEltVT) {
5180 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5181 MVT MidVT = OldEltsPerNewElt == 1
5182 ? NewEltVT
5183 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5184 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5185 return MidVT;
5186}
5187
5188void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5189 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5190 SmallVector<SDValue, 8> Results;
5191 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5192 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5193 Node->getOpcode() == ISD::SINT_TO_FP ||
5194 Node->getOpcode() == ISD::SETCC ||
5195 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5196 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5197 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5198 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5199 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5200 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM) {
5201 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5202 }
5203 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5204 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5205 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5206 Node->getOpcode() == ISD::STRICT_FSETCC ||
5207 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5208 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5209 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5210 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5211 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5212 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5213 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5214 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5215 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5216 if (Node->getOpcode() == ISD::BR_CC ||
5217 Node->getOpcode() == ISD::SELECT_CC)
5218 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5219 // Preserve fast math flags
5220 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5221 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5222 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5223 SDLoc dl(Node);
5224 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5225 switch (Node->getOpcode()) {
5226 case ISD::CTTZ:
5227 case ISD::CTTZ_ZERO_UNDEF:
5228 case ISD::CTLZ:
5229 case ISD::CTPOP: {
5230 // Zero extend the argument unless its cttz, then use any_extend.
5231 if (Node->getOpcode() == ISD::CTTZ ||
5232 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
5233 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5234 else
5235 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5236
5237 unsigned NewOpc = Node->getOpcode();
5238 if (NewOpc == ISD::CTTZ) {
5239 // The count is the same in the promoted type except if the original
5240 // value was zero. This can be handled by setting the bit just off
5241 // the top of the original type.
5242 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5243 BitNo: OVT.getSizeInBits());
5244 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5245 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5246 NewOpc = ISD::CTTZ_ZERO_UNDEF;
5247 }
5248 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
5249 // already the correct result.
5250 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5251 if (NewOpc == ISD::CTLZ) {
5252 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5253 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5254 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5255 OVT.getSizeInBits(), DL: dl, VT: NVT));
5256 }
5257 Results.push_back(
5258 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5259 break;
5260 }
5261 case ISD::CTLZ_ZERO_UNDEF: {
5262 // We know that the argument is unlikely to be zero, hence we can take a
5263 // different approach as compared to ISD::CTLZ
5264
5265 // Any Extend the argument
5266 auto AnyExtendedNode =
5267 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5268
5269 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5270 auto ShiftConstant = DAG.getShiftAmountConstant(
5271 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5272 auto LeftShiftResult =
5273 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5274
5275 // Perform the larger operation
5276 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5277 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5278 break;
5279 }
5280 case ISD::BITREVERSE:
5281 case ISD::BSWAP: {
5282 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5283 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5284 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5285 Tmp1 = DAG.getNode(
5286 Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5287 N2: DAG.getConstant(Val: DiffBits, DL: dl,
5288 VT: TLI.getShiftAmountTy(LHSTy: NVT, DL: DAG.getDataLayout())));
5289
5290 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5291 break;
5292 }
5293 case ISD::FP_TO_UINT:
5294 case ISD::STRICT_FP_TO_UINT:
5295 case ISD::FP_TO_SINT:
5296 case ISD::STRICT_FP_TO_SINT:
5297 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5298 break;
5299 case ISD::FP_TO_UINT_SAT:
5300 case ISD::FP_TO_SINT_SAT:
5301 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5302 break;
5303 case ISD::UINT_TO_FP:
5304 case ISD::STRICT_UINT_TO_FP:
5305 case ISD::SINT_TO_FP:
5306 case ISD::STRICT_SINT_TO_FP:
5307 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5308 break;
5309 case ISD::VAARG: {
5310 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5311 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5312
5313 unsigned TruncOp;
5314 if (OVT.isVector()) {
5315 TruncOp = ISD::BITCAST;
5316 } else {
5317 assert(OVT.isInteger()
5318 && "VAARG promotion is supported only for vectors or integer types");
5319 TruncOp = ISD::TRUNCATE;
5320 }
5321
5322 // Perform the larger operation, then convert back
5323 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5324 Align: Node->getConstantOperandVal(Num: 3));
5325 Chain = Tmp1.getValue(R: 1);
5326
5327 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5328
5329 // Modified the chain result - switch anything that used the old chain to
5330 // use the new one.
5331 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5332 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5333 if (UpdatedNodes) {
5334 UpdatedNodes->insert(X: Tmp2.getNode());
5335 UpdatedNodes->insert(X: Chain.getNode());
5336 }
5337 ReplacedNode(N: Node);
5338 break;
5339 }
5340 case ISD::MUL:
5341 case ISD::SDIV:
5342 case ISD::SREM:
5343 case ISD::UDIV:
5344 case ISD::UREM:
5345 case ISD::SMIN:
5346 case ISD::SMAX:
5347 case ISD::UMIN:
5348 case ISD::UMAX:
5349 case ISD::AND:
5350 case ISD::OR:
5351 case ISD::XOR: {
5352 unsigned ExtOp, TruncOp;
5353 if (OVT.isVector()) {
5354 ExtOp = ISD::BITCAST;
5355 TruncOp = ISD::BITCAST;
5356 } else {
5357 assert(OVT.isInteger() && "Cannot promote logic operation");
5358
5359 switch (Node->getOpcode()) {
5360 default:
5361 ExtOp = ISD::ANY_EXTEND;
5362 break;
5363 case ISD::SDIV:
5364 case ISD::SREM:
5365 case ISD::SMIN:
5366 case ISD::SMAX:
5367 ExtOp = ISD::SIGN_EXTEND;
5368 break;
5369 case ISD::UDIV:
5370 case ISD::UREM:
5371 ExtOp = ISD::ZERO_EXTEND;
5372 break;
5373 case ISD::UMIN:
5374 case ISD::UMAX:
5375 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5376 ExtOp = ISD::SIGN_EXTEND;
5377 else
5378 ExtOp = ISD::ZERO_EXTEND;
5379 break;
5380 }
5381 TruncOp = ISD::TRUNCATE;
5382 }
5383 // Promote each of the values to the new type.
5384 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5385 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5386 // Perform the larger operation, then convert back
5387 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5388 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5389 break;
5390 }
5391 case ISD::UMUL_LOHI:
5392 case ISD::SMUL_LOHI: {
5393 // Promote to a multiply in a wider integer type.
5394 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5395 : ISD::SIGN_EXTEND;
5396 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5397 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5398 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5399
5400 auto &DL = DAG.getDataLayout();
5401 unsigned OriginalSize = OVT.getScalarSizeInBits();
5402 Tmp2 = DAG.getNode(
5403 Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5404 N2: DAG.getConstant(Val: OriginalSize, DL: dl, VT: TLI.getScalarShiftAmountTy(DL, NVT)));
5405 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5406 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5407 break;
5408 }
5409 case ISD::SELECT: {
5410 unsigned ExtOp, TruncOp;
5411 if (Node->getValueType(ResNo: 0).isVector() ||
5412 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5413 ExtOp = ISD::BITCAST;
5414 TruncOp = ISD::BITCAST;
5415 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5416 ExtOp = ISD::ANY_EXTEND;
5417 TruncOp = ISD::TRUNCATE;
5418 } else {
5419 ExtOp = ISD::FP_EXTEND;
5420 TruncOp = ISD::FP_ROUND;
5421 }
5422 Tmp1 = Node->getOperand(Num: 0);
5423 // Promote each of the values to the new type.
5424 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5425 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5426 // Perform the larger operation, then round down.
5427 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5428 if (TruncOp != ISD::FP_ROUND)
5429 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5430 else
5431 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5432 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5433 Results.push_back(Elt: Tmp1);
5434 break;
5435 }
5436 case ISD::VECTOR_SHUFFLE: {
5437 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5438
5439 // Cast the two input vectors.
5440 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5441 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5442
5443 // Convert the shuffle mask to the right # elements.
5444 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5445 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5446 Results.push_back(Elt: Tmp1);
5447 break;
5448 }
5449 case ISD::VECTOR_SPLICE: {
5450 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5451 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5452 Tmp3 = DAG.getNode(Opcode: ISD::VECTOR_SPLICE, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5453 N3: Node->getOperand(Num: 2));
5454 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5455 break;
5456 }
5457 case ISD::SELECT_CC: {
5458 SDValue Cond = Node->getOperand(Num: 4);
5459 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5460 // Type of the comparison operands.
5461 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5462 assert(CVT == OVT && "not handled");
5463
5464 unsigned ExtOp = ISD::FP_EXTEND;
5465 if (NVT.isInteger()) {
5466 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5467 }
5468
5469 // Promote the comparison operands, if needed.
5470 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5471 Tmp1 = Node->getOperand(Num: 0);
5472 Tmp2 = Node->getOperand(Num: 1);
5473 } else {
5474 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5475 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5476 }
5477 // Cast the true/false operands.
5478 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5479 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5480
5481 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5482 Flags: Node->getFlags());
5483
5484 // Cast the result back to the original type.
5485 if (ExtOp != ISD::FP_EXTEND)
5486 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5487 else
5488 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5489 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5490
5491 Results.push_back(Elt: Tmp1);
5492 break;
5493 }
5494 case ISD::SETCC:
5495 case ISD::STRICT_FSETCC:
5496 case ISD::STRICT_FSETCCS: {
5497 unsigned ExtOp = ISD::FP_EXTEND;
5498 if (NVT.isInteger()) {
5499 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5500 if (isSignedIntSetCC(Code: CCCode) ||
5501 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5502 ExtOp = ISD::SIGN_EXTEND;
5503 else
5504 ExtOp = ISD::ZERO_EXTEND;
5505 }
5506 if (Node->isStrictFPOpcode()) {
5507 SDValue InChain = Node->getOperand(Num: 0);
5508 std::tie(args&: Tmp1, args: std::ignore) =
5509 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5510 std::tie(args&: Tmp2, args: std::ignore) =
5511 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5512 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5513 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5514 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
5515 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5516 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5517 Flags: Node->getFlags()));
5518 Results.push_back(Elt: Results.back().getValue(R: 1));
5519 break;
5520 }
5521 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5522 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5523 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5524 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5525 break;
5526 }
5527 case ISD::BR_CC: {
5528 unsigned ExtOp = ISD::FP_EXTEND;
5529 if (NVT.isInteger()) {
5530 ISD::CondCode CCCode =
5531 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5532 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5533 }
5534 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5535 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5536 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5537 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5538 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5539 break;
5540 }
5541 case ISD::FADD:
5542 case ISD::FSUB:
5543 case ISD::FMUL:
5544 case ISD::FDIV:
5545 case ISD::FREM:
5546 case ISD::FMINNUM:
5547 case ISD::FMAXNUM:
5548 case ISD::FMINIMUM:
5549 case ISD::FMAXIMUM:
5550 case ISD::FMINIMUMNUM:
5551 case ISD::FMAXIMUMNUM:
5552 case ISD::FPOW:
5553 case ISD::FATAN2:
5554 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5555 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5556 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5557 Results.push_back(
5558 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5559 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5560 break;
5561
5562 case ISD::STRICT_FMINIMUM:
5563 case ISD::STRICT_FMAXIMUM: {
5564 SDValue InChain = Node->getOperand(Num: 0);
5565 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
5566 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5567 N2: Node->getOperand(Num: 1));
5568 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5569 N2: Node->getOperand(Num: 2));
5570 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
5571 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
5572 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
5573 N1: InChain, N2: Tmp3,
5574 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5575 Results.push_back(Elt: Tmp4);
5576 Results.push_back(Elt: Tmp4.getValue(R: 1));
5577 break;
5578 }
5579
5580 case ISD::STRICT_FADD:
5581 case ISD::STRICT_FSUB:
5582 case ISD::STRICT_FMUL:
5583 case ISD::STRICT_FDIV:
5584 case ISD::STRICT_FMINNUM:
5585 case ISD::STRICT_FMAXNUM:
5586 case ISD::STRICT_FREM:
5587 case ISD::STRICT_FPOW:
5588 case ISD::STRICT_FATAN2:
5589 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5590 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5591 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5592 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5593 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5594 N2: Tmp2.getValue(R: 1));
5595 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5596 Ops: {Tmp3, Tmp1, Tmp2});
5597 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5598 Ops: {Tmp1.getValue(R: 1), Tmp1,
5599 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5600 Results.push_back(Elt: Tmp1);
5601 Results.push_back(Elt: Tmp1.getValue(R: 1));
5602 break;
5603 case ISD::FMA:
5604 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5605 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5606 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5607 Results.push_back(
5608 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5609 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5610 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5611 break;
5612 case ISD::STRICT_FMA:
5613 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5614 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5615 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5616 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5617 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5618 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
5619 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5620 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
5621 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5622 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
5623 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5624 Ops: {Tmp4.getValue(R: 1), Tmp4,
5625 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5626 Results.push_back(Elt: Tmp4);
5627 Results.push_back(Elt: Tmp4.getValue(R: 1));
5628 break;
5629 case ISD::FCOPYSIGN:
5630 case ISD::FLDEXP:
5631 case ISD::FPOWI: {
5632 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5633 Tmp2 = Node->getOperand(Num: 1);
5634 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5635
5636 // fcopysign doesn't change anything but the sign bit, so
5637 // (fp_round (fcopysign (fpext a), b))
5638 // is as precise as
5639 // (fp_round (fpext a))
5640 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5641 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5642 Results.push_back(
5643 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5644 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5645 break;
5646 }
5647 case ISD::STRICT_FLDEXP: {
5648 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5649 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5650 Tmp2 = Node->getOperand(Num: 2);
5651 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
5652 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
5653 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5654 Ops: {Tmp3.getValue(R: 1), Tmp3,
5655 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5656 Results.push_back(Elt: Tmp4);
5657 Results.push_back(Elt: Tmp4.getValue(R: 1));
5658 break;
5659 }
5660 case ISD::STRICT_FPOWI:
5661 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5662 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5663 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5664 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
5665 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5666 Ops: {Tmp2.getValue(R: 1), Tmp2,
5667 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5668 Results.push_back(Elt: Tmp3);
5669 Results.push_back(Elt: Tmp3.getValue(R: 1));
5670 break;
5671 case ISD::FFREXP: {
5672 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5673 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5674
5675 Results.push_back(
5676 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5677 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5678
5679 Results.push_back(Elt: Tmp2.getValue(R: 1));
5680 break;
5681 }
5682 case ISD::FMODF:
5683 case ISD::FSINCOS:
5684 case ISD::FSINCOSPI: {
5685 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5686 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
5687 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
5688 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5689 Results.push_back(
5690 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
5691 break;
5692 }
5693 case ISD::FFLOOR:
5694 case ISD::FCEIL:
5695 case ISD::FRINT:
5696 case ISD::FNEARBYINT:
5697 case ISD::FROUND:
5698 case ISD::FROUNDEVEN:
5699 case ISD::FTRUNC:
5700 case ISD::FNEG:
5701 case ISD::FSQRT:
5702 case ISD::FSIN:
5703 case ISD::FCOS:
5704 case ISD::FTAN:
5705 case ISD::FASIN:
5706 case ISD::FACOS:
5707 case ISD::FATAN:
5708 case ISD::FSINH:
5709 case ISD::FCOSH:
5710 case ISD::FTANH:
5711 case ISD::FLOG:
5712 case ISD::FLOG2:
5713 case ISD::FLOG10:
5714 case ISD::FABS:
5715 case ISD::FEXP:
5716 case ISD::FEXP2:
5717 case ISD::FEXP10:
5718 case ISD::FCANONICALIZE:
5719 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5720 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5721 Results.push_back(
5722 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5723 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5724 break;
5725 case ISD::STRICT_FFLOOR:
5726 case ISD::STRICT_FCEIL:
5727 case ISD::STRICT_FRINT:
5728 case ISD::STRICT_FNEARBYINT:
5729 case ISD::STRICT_FROUND:
5730 case ISD::STRICT_FROUNDEVEN:
5731 case ISD::STRICT_FTRUNC:
5732 case ISD::STRICT_FSQRT:
5733 case ISD::STRICT_FSIN:
5734 case ISD::STRICT_FCOS:
5735 case ISD::STRICT_FTAN:
5736 case ISD::STRICT_FASIN:
5737 case ISD::STRICT_FACOS:
5738 case ISD::STRICT_FATAN:
5739 case ISD::STRICT_FSINH:
5740 case ISD::STRICT_FCOSH:
5741 case ISD::STRICT_FTANH:
5742 case ISD::STRICT_FLOG:
5743 case ISD::STRICT_FLOG2:
5744 case ISD::STRICT_FLOG10:
5745 case ISD::STRICT_FEXP:
5746 case ISD::STRICT_FEXP2:
5747 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5748 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5749 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5750 Ops: {Tmp1.getValue(R: 1), Tmp1});
5751 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5752 Ops: {Tmp2.getValue(R: 1), Tmp2,
5753 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5754 Results.push_back(Elt: Tmp3);
5755 Results.push_back(Elt: Tmp3.getValue(R: 1));
5756 break;
5757 case ISD::BUILD_VECTOR: {
5758 MVT EltVT = OVT.getVectorElementType();
5759 MVT NewEltVT = NVT.getVectorElementType();
5760
5761 // Handle bitcasts to a different vector type with the same total bit size
5762 //
5763 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
5764 // =>
5765 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
5766
5767 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5768 "Invalid promote type for build_vector");
5769 assert(NewEltVT.bitsLE(EltVT) && "not handled");
5770
5771 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5772
5773 SmallVector<SDValue, 8> NewOps;
5774 for (const SDValue &Op : Node->op_values())
5775 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
5776
5777 SDLoc SL(Node);
5778 SDValue Concat =
5779 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
5780 DL: SL, VT: NVT, Ops: NewOps);
5781 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
5782 Results.push_back(Elt: CvtVec);
5783 break;
5784 }
5785 case ISD::EXTRACT_VECTOR_ELT: {
5786 MVT EltVT = OVT.getVectorElementType();
5787 MVT NewEltVT = NVT.getVectorElementType();
5788
5789 // Handle bitcasts to a different vector type with the same total bit size.
5790 //
5791 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
5792 // =>
5793 // v4i32:castx = bitcast x:v2i64
5794 //
5795 // i64 = bitcast
5796 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
5797 // (i32 (extract_vector_elt castx, (2 * y + 1)))
5798 //
5799
5800 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5801 "Invalid promote type for extract_vector_elt");
5802 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5803
5804 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5805 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5806
5807 SDValue Idx = Node->getOperand(Num: 1);
5808 EVT IdxVT = Idx.getValueType();
5809 SDLoc SL(Node);
5810 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
5811 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
5812
5813 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
5814
5815 SmallVector<SDValue, 8> NewOps;
5816 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5817 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
5818 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
5819
5820 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
5821 N1: CastVec, N2: TmpIdx);
5822 NewOps.push_back(Elt);
5823 }
5824
5825 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
5826 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
5827 break;
5828 }
5829 case ISD::INSERT_VECTOR_ELT: {
5830 MVT EltVT = OVT.getVectorElementType();
5831 MVT NewEltVT = NVT.getVectorElementType();
5832
5833 // Handle bitcasts to a different vector type with the same total bit size
5834 //
5835 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
5836 // =>
5837 // v4i32:castx = bitcast x:v2i64
5838 // v2i32:casty = bitcast y:i64
5839 //
5840 // v2i64 = bitcast
5841 // (v4i32 insert_vector_elt
5842 // (v4i32 insert_vector_elt v4i32:castx,
5843 // (extract_vector_elt casty, 0), 2 * z),
5844 // (extract_vector_elt casty, 1), (2 * z + 1))
5845
5846 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5847 "Invalid promote type for insert_vector_elt");
5848 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5849
5850 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5851 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5852
5853 SDValue Val = Node->getOperand(Num: 1);
5854 SDValue Idx = Node->getOperand(Num: 2);
5855 EVT IdxVT = Idx.getValueType();
5856 SDLoc SL(Node);
5857
5858 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
5859 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
5860
5861 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
5862 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
5863
5864 SDValue NewVec = CastVec;
5865 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5866 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
5867 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
5868
5869 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
5870 N1: CastVal, N2: IdxOffset);
5871
5872 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
5873 N1: NewVec, N2: Elt, N3: InEltIdx);
5874 }
5875
5876 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
5877 break;
5878 }
5879 case ISD::SCALAR_TO_VECTOR: {
5880 MVT EltVT = OVT.getVectorElementType();
5881 MVT NewEltVT = NVT.getVectorElementType();
5882
5883 // Handle bitcasts to different vector type with the same total bit size.
5884 //
5885 // e.g. v2i64 = scalar_to_vector x:i64
5886 // =>
5887 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
5888 //
5889
5890 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5891 SDValue Val = Node->getOperand(Num: 0);
5892 SDLoc SL(Node);
5893
5894 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
5895 SDValue Undef = DAG.getUNDEF(VT: MidVT);
5896
5897 SmallVector<SDValue, 8> NewElts;
5898 NewElts.push_back(Elt: CastVal);
5899 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5900 NewElts.push_back(Elt: Undef);
5901
5902 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
5903 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
5904 Results.push_back(Elt: CvtVec);
5905 break;
5906 }
5907 case ISD::ATOMIC_SWAP:
5908 case ISD::ATOMIC_STORE: {
5909 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
5910 SDLoc SL(Node);
5911 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
5912 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5913 "unexpected promotion type");
5914 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5915 "unexpected atomic_swap with illegal type");
5916
5917 SDValue Op0 = AM->getBasePtr();
5918 SDValue Op1 = CastVal;
5919
5920 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
5921 // but really it should merge with ISD::STORE.
5922 if (AM->getOpcode() == ISD::ATOMIC_STORE)
5923 std::swap(a&: Op0, b&: Op1);
5924
5925 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
5926 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
5927
5928 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
5929 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
5930 Results.push_back(Elt: NewAtomic.getValue(R: 1));
5931 } else
5932 Results.push_back(Elt: NewAtomic);
5933 break;
5934 }
5935 case ISD::ATOMIC_LOAD: {
5936 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
5937 SDLoc SL(Node);
5938 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5939 "unexpected promotion type");
5940 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5941 "unexpected atomic_load with illegal type");
5942
5943 SDValue NewAtomic =
5944 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
5945 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
5946 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
5947 Results.push_back(Elt: NewAtomic.getValue(R: 1));
5948 break;
5949 }
5950 case ISD::SPLAT_VECTOR: {
5951 SDValue Scalar = Node->getOperand(Num: 0);
5952 MVT ScalarType = Scalar.getSimpleValueType();
5953 MVT NewScalarType = NVT.getVectorElementType();
5954 if (ScalarType.isInteger()) {
5955 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
5956 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5957 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5958 break;
5959 }
5960 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
5961 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5962 Results.push_back(
5963 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5964 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5965 break;
5966 }
5967 case ISD::VECREDUCE_FMAX:
5968 case ISD::VECREDUCE_FMIN:
5969 case ISD::VECREDUCE_FMAXIMUM:
5970 case ISD::VECREDUCE_FMINIMUM:
5971 case ISD::VP_REDUCE_FMAX:
5972 case ISD::VP_REDUCE_FMIN:
5973 case ISD::VP_REDUCE_FMAXIMUM:
5974 case ISD::VP_REDUCE_FMINIMUM:
5975 Results.push_back(Elt: PromoteReduction(Node));
5976 break;
5977 }
5978
5979 // Replace the original node with the legalized result.
5980 if (!Results.empty()) {
5981 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
5982 ReplaceNode(Old: Node, New: Results.data());
5983 } else
5984 LLVM_DEBUG(dbgs() << "Could not promote node\n");
5985}
5986
5987/// This is the entry point for the file.
5988void SelectionDAG::Legalize() {
5989 AssignTopologicalOrder();
5990
5991 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5992 // Use a delete listener to remove nodes which were deleted during
5993 // legalization from LegalizeNodes. This is needed to handle the situation
5994 // where a new node is allocated by the object pool to the same address of a
5995 // previously deleted node.
5996 DAGNodeDeletedListener DeleteListener(
5997 *this,
5998 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
5999
6000 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6001
6002 // Visit all the nodes. We start in topological order, so that we see
6003 // nodes with their original operands intact. Legalization can produce
6004 // new nodes which may themselves need to be legalized. Iterate until all
6005 // nodes have been legalized.
6006 while (true) {
6007 bool AnyLegalized = false;
6008 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6009 --NI;
6010
6011 SDNode *N = &*NI;
6012 if (N->use_empty() && N != getRoot().getNode()) {
6013 ++NI;
6014 DeleteNode(N);
6015 continue;
6016 }
6017
6018 if (LegalizedNodes.insert(Ptr: N).second) {
6019 AnyLegalized = true;
6020 Legalizer.LegalizeOp(Node: N);
6021
6022 if (N->use_empty() && N != getRoot().getNode()) {
6023 ++NI;
6024 DeleteNode(N);
6025 }
6026 }
6027 }
6028 if (!AnyLegalized)
6029 break;
6030
6031 }
6032
6033 // Remove dead nodes now.
6034 RemoveDeadNodes();
6035}
6036
6037bool SelectionDAG::LegalizeOp(SDNode *N,
6038 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6039 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6040 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6041
6042 // Directly insert the node in question, and legalize it. This will recurse
6043 // as needed through operands.
6044 LegalizedNodes.insert(Ptr: N);
6045 Legalizer.LegalizeOp(Node: N);
6046
6047 return LegalizedNodes.count(Ptr: N);
6048}
6049