1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/Analysis/AliasAnalysis.h"
24#include "llvm/Analysis/BranchProbabilityInfo.h"
25#include "llvm/Analysis/ConstantFolding.h"
26#include "llvm/Analysis/Loads.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/TargetTransformInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/Analysis/VectorUtils.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
34#include "llvm/CodeGen/CodeGenCommonISel.h"
35#include "llvm/CodeGen/FunctionLoweringInfo.h"
36#include "llvm/CodeGen/GCMetadata.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/MachineBasicBlock.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineInstrBuilder.h"
42#include "llvm/CodeGen/MachineInstrBundleIterator.h"
43#include "llvm/CodeGen/MachineMemOperand.h"
44#include "llvm/CodeGen/MachineModuleInfo.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/MachineRegisterInfo.h"
47#include "llvm/CodeGen/SelectionDAG.h"
48#include "llvm/CodeGen/SelectionDAGNodes.h"
49#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
50#include "llvm/CodeGen/StackMaps.h"
51#include "llvm/CodeGen/SwiftErrorValueTracking.h"
52#include "llvm/CodeGen/TargetFrameLowering.h"
53#include "llvm/CodeGen/TargetInstrInfo.h"
54#include "llvm/CodeGen/TargetOpcodes.h"
55#include "llvm/CodeGen/TargetRegisterInfo.h"
56#include "llvm/CodeGen/TargetSubtargetInfo.h"
57#include "llvm/CodeGen/WinEHFuncInfo.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
64#include "llvm/IR/ConstantRange.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
68#include "llvm/IR/DebugInfoMetadata.h"
69#include "llvm/IR/DerivedTypes.h"
70#include "llvm/IR/DiagnosticInfo.h"
71#include "llvm/IR/EHPersonalities.h"
72#include "llvm/IR/Function.h"
73#include "llvm/IR/GetElementPtrTypeIterator.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
76#include "llvm/IR/Instructions.h"
77#include "llvm/IR/IntrinsicInst.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
84#include "llvm/IR/Metadata.h"
85#include "llvm/IR/Module.h"
86#include "llvm/IR/Operator.h"
87#include "llvm/IR/PatternMatch.h"
88#include "llvm/IR/Statepoint.h"
89#include "llvm/IR/Type.h"
90#include "llvm/IR/User.h"
91#include "llvm/IR/Value.h"
92#include "llvm/MC/MCAsmInfo.h"
93#include "llvm/MC/MCContext.h"
94#include "llvm/Support/AtomicOrdering.h"
95#include "llvm/Support/Casting.h"
96#include "llvm/Support/CommandLine.h"
97#include "llvm/Support/Compiler.h"
98#include "llvm/Support/Debug.h"
99#include "llvm/Support/InstructionCost.h"
100#include "llvm/Support/MathExtras.h"
101#include "llvm/Support/raw_ostream.h"
102#include "llvm/Target/TargetMachine.h"
103#include "llvm/Target/TargetOptions.h"
104#include "llvm/TargetParser/Triple.h"
105#include "llvm/Transforms/Utils/Local.h"
106#include <cstddef>
107#include <limits>
108#include <optional>
109#include <tuple>
110
111using namespace llvm;
112using namespace PatternMatch;
113using namespace SwitchCG;
114
115#define DEBUG_TYPE "isel"
116
117/// LimitFloatPrecision - Generate low-precision inline sequences for
118/// some float libcalls (6, 8 or 12 bits).
119static unsigned LimitFloatPrecision;
120
121static cl::opt<bool>
122 InsertAssertAlign("insert-assert-align", cl::init(Val: true),
123 cl::desc("Insert the experimental `assertalign` node."),
124 cl::ReallyHidden);
125
126static cl::opt<unsigned, true>
127 LimitFPPrecision("limit-float-precision",
128 cl::desc("Generate low-precision inline sequences "
129 "for some float libcalls"),
130 cl::location(L&: LimitFloatPrecision), cl::Hidden,
131 cl::init(Val: 0));
132
133static cl::opt<unsigned> SwitchPeelThreshold(
134 "switch-peel-threshold", cl::Hidden, cl::init(Val: 66),
135 cl::desc("Set the case probability threshold for peeling the case from a "
136 "switch statement. A value greater than 100 will void this "
137 "optimization"));
138
139// Limit the width of DAG chains. This is important in general to prevent
140// DAG-based analysis from blowing up. For example, alias analysis and
141// load clustering may not complete in reasonable time. It is difficult to
142// recognize and avoid this situation within each individual analysis, and
143// future analyses are likely to have the same behavior. Limiting DAG width is
144// the safe approach and will be especially important with global DAGs.
145//
146// MaxParallelChains default is arbitrarily high to avoid affecting
147// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
148// sequence over this should have been converted to llvm.memcpy by the
149// frontend. It is easy to induce this behavior with .ll code such as:
150// %buffer = alloca [4096 x i8]
151// %data = load [4096 x i8]* %argPtr
152// store [4096 x i8] %data, [4096 x i8]* %buffer
153static const unsigned MaxParallelChains = 64;
154
155static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
156 const SDValue *Parts, unsigned NumParts,
157 MVT PartVT, EVT ValueVT, const Value *V,
158 SDValue InChain,
159 std::optional<CallingConv::ID> CC);
160
161/// getCopyFromParts - Create a value that contains the specified legal parts
162/// combined into the value they represent. If the parts combine to a type
163/// larger than ValueVT then AssertOp can be used to specify whether the extra
164/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
165/// (ISD::AssertSext).
166static SDValue
167getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
168 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
169 SDValue InChain,
170 std::optional<CallingConv::ID> CC = std::nullopt,
171 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
172 // Let the target assemble the parts if it wants to
173 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
174 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
175 PartVT, ValueVT, CC))
176 return Val;
177
178 if (ValueVT.isVector())
179 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
180 InChain, CC);
181
182 assert(NumParts > 0 && "No parts to assemble!");
183 SDValue Val = Parts[0];
184
185 if (NumParts > 1) {
186 // Assemble the value from multiple parts.
187 if (ValueVT.isInteger()) {
188 unsigned PartBits = PartVT.getSizeInBits();
189 unsigned ValueBits = ValueVT.getSizeInBits();
190
191 // Assemble the power of 2 part.
192 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
193 unsigned RoundBits = PartBits * RoundParts;
194 EVT RoundVT = RoundBits == ValueBits ?
195 ValueVT : EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits);
196 SDValue Lo, Hi;
197
198 EVT HalfVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits/2);
199
200 if (RoundParts > 2) {
201 Lo = getCopyFromParts(DAG, DL, Parts, NumParts: RoundParts / 2, PartVT, ValueVT: HalfVT, V,
202 InChain);
203 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts / 2, NumParts: RoundParts / 2,
204 PartVT, ValueVT: HalfVT, V, InChain);
205 } else {
206 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[0]);
207 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[1]);
208 }
209
210 if (DAG.getDataLayout().isBigEndian())
211 std::swap(a&: Lo, b&: Hi);
212
213 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: RoundVT, N1: Lo, N2: Hi);
214
215 if (RoundParts < NumParts) {
216 // Assemble the trailing non-power-of-2 part.
217 unsigned OddParts = NumParts - RoundParts;
218 EVT OddVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OddParts * PartBits);
219 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts, NumParts: OddParts, PartVT,
220 ValueVT: OddVT, V, InChain, CC);
221
222 // Combine the round and odd parts.
223 Lo = Val;
224 if (DAG.getDataLayout().isBigEndian())
225 std::swap(a&: Lo, b&: Hi);
226 EVT TotalVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
227 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: TotalVT, Operand: Hi);
228 Hi = DAG.getNode(
229 Opcode: ISD::SHL, DL, VT: TotalVT, N1: Hi,
230 N2: DAG.getShiftAmountConstant(Val: Lo.getValueSizeInBits(), VT: TotalVT, DL));
231 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: TotalVT, Operand: Lo);
232 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: TotalVT, N1: Lo, N2: Hi);
233 }
234 } else if (PartVT.isFloatingPoint()) {
235 // FP split into multiple FP parts (for ppcf128)
236 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
237 "Unexpected split");
238 SDValue Lo, Hi;
239 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EVT(MVT::f64), Operand: Parts[0]);
240 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EVT(MVT::f64), Operand: Parts[1]);
241 if (TLI.hasBigEndianPartOrdering(VT: ValueVT, DL: DAG.getDataLayout()))
242 std::swap(a&: Lo, b&: Hi);
243 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: ValueVT, N1: Lo, N2: Hi);
244 } else {
245 // FP split into integer parts (soft fp)
246 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
247 !PartVT.isVector() && "Unexpected split");
248 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
249 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, ValueVT: IntVT, V,
250 InChain, CC);
251 }
252 }
253
254 // There is now one part, held in Val. Correct it to match ValueVT.
255 // PartEVT is the type of the register class that holds the value.
256 // ValueVT is the type of the inline asm operation.
257 EVT PartEVT = Val.getValueType();
258
259 if (PartEVT == ValueVT)
260 return Val;
261
262 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
263 ValueVT.bitsLT(VT: PartEVT)) {
264 // For an FP value in an integer part, we need to truncate to the right
265 // width first.
266 PartEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
267 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: PartEVT, Operand: Val);
268 }
269
270 // Handle types that have the same size.
271 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
272 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
273
274 // Handle types with different sizes.
275 if (PartEVT.isInteger() && ValueVT.isInteger()) {
276 if (ValueVT.bitsLT(VT: PartEVT)) {
277 // For a truncate, see if we have any information to
278 // indicate whether the truncated bits will always be
279 // zero or sign-extension.
280 if (AssertOp)
281 Val = DAG.getNode(Opcode: *AssertOp, DL, VT: PartEVT, N1: Val,
282 N2: DAG.getValueType(ValueVT));
283 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
284 }
285 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
286 }
287
288 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
289 // FP_ROUND's are always exact here.
290 if (ValueVT.bitsLT(VT: Val.getValueType())) {
291
292 SDValue NoChange =
293 DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
294
295 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
296 Kind: llvm::Attribute::StrictFP)) {
297 return DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL,
298 VTList: DAG.getVTList(VT1: ValueVT, VT2: MVT::Other), N1: InChain, N2: Val,
299 N3: NoChange);
300 }
301
302 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ValueVT, N1: Val, N2: NoChange);
303 }
304
305 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: ValueVT, Operand: Val);
306 }
307
308 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
309 // then truncating.
310 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
311 ValueVT.bitsLT(VT: PartEVT)) {
312 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Val);
313 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
314 }
315
316 report_fatal_error(reason: "Unknown mismatch in getCopyFromParts!");
317}
318
319static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
320 const Twine &ErrMsg) {
321 const Instruction *I = dyn_cast_or_null<Instruction>(Val: V);
322 if (!I)
323 return Ctx.emitError(ErrorStr: ErrMsg);
324
325 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
326 if (CI->isInlineAsm()) {
327 return Ctx.diagnose(DI: DiagnosticInfoInlineAsm(
328 *CI, ErrMsg + ", possible invalid constraint for vector type"));
329 }
330
331 return Ctx.emitError(I, ErrorStr: ErrMsg);
332}
333
334/// getCopyFromPartsVector - Create a value that contains the specified legal
335/// parts combined into the value they represent. If the parts combine to a
336/// type larger than ValueVT then AssertOp can be used to specify whether the
337/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
338/// ValueVT (ISD::AssertSext).
339static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
340 const SDValue *Parts, unsigned NumParts,
341 MVT PartVT, EVT ValueVT, const Value *V,
342 SDValue InChain,
343 std::optional<CallingConv::ID> CallConv) {
344 assert(ValueVT.isVector() && "Not a vector value");
345 assert(NumParts > 0 && "No parts to assemble!");
346 const bool IsABIRegCopy = CallConv.has_value();
347
348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
349 SDValue Val = Parts[0];
350
351 // Handle a multi-element vector.
352 if (NumParts > 1) {
353 EVT IntermediateVT;
354 MVT RegisterVT;
355 unsigned NumIntermediates;
356 unsigned NumRegs;
357
358 if (IsABIRegCopy) {
359 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
360 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT,
361 NumIntermediates, RegisterVT);
362 } else {
363 NumRegs =
364 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
365 NumIntermediates, RegisterVT);
366 }
367
368 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
369 NumParts = NumRegs; // Silence a compiler warning.
370 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
371 assert(RegisterVT.getSizeInBits() ==
372 Parts[0].getSimpleValueType().getSizeInBits() &&
373 "Part type sizes don't match!");
374
375 // Assemble the parts into intermediate operands.
376 SmallVector<SDValue, 8> Ops(NumIntermediates);
377 if (NumIntermediates == NumParts) {
378 // If the register was not expanded, truncate or copy the value,
379 // as appropriate.
380 for (unsigned i = 0; i != NumParts; ++i)
381 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i], NumParts: 1, PartVT, ValueVT: IntermediateVT,
382 V, InChain, CC: CallConv);
383 } else if (NumParts > 0) {
384 // If the intermediate type was expanded, build the intermediate
385 // operands from the parts.
386 assert(NumParts % NumIntermediates == 0 &&
387 "Must expand into a divisible number of parts!");
388 unsigned Factor = NumParts / NumIntermediates;
389 for (unsigned i = 0; i != NumIntermediates; ++i)
390 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i * Factor], NumParts: Factor, PartVT,
391 ValueVT: IntermediateVT, V, InChain, CC: CallConv);
392 }
393
394 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
395 // intermediate operands.
396 EVT BuiltVectorTy =
397 IntermediateVT.isVector()
398 ? EVT::getVectorVT(
399 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(),
400 EC: IntermediateVT.getVectorElementCount() * NumParts)
401 : EVT::getVectorVT(Context&: *DAG.getContext(),
402 VT: IntermediateVT.getScalarType(),
403 NumElements: NumIntermediates);
404 Val = DAG.getNode(Opcode: IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
405 : ISD::BUILD_VECTOR,
406 DL, VT: BuiltVectorTy, Ops);
407 }
408
409 // There is now one part, held in Val. Correct it to match ValueVT.
410 EVT PartEVT = Val.getValueType();
411
412 if (PartEVT == ValueVT)
413 return Val;
414
415 if (PartEVT.isVector()) {
416 // Vector/Vector bitcast.
417 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
418 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
419
420 // If the parts vector has more elements than the value vector, then we
421 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
422 // Extract the elements we want.
423 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
424 assert((PartEVT.getVectorElementCount().getKnownMinValue() >
425 ValueVT.getVectorElementCount().getKnownMinValue()) &&
426 (PartEVT.getVectorElementCount().isScalable() ==
427 ValueVT.getVectorElementCount().isScalable()) &&
428 "Cannot narrow, it would be a lossy transformation");
429 PartEVT =
430 EVT::getVectorVT(Context&: *DAG.getContext(), VT: PartEVT.getVectorElementType(),
431 EC: ValueVT.getVectorElementCount());
432 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: PartEVT, N1: Val,
433 N2: DAG.getVectorIdxConstant(Val: 0, DL));
434 if (PartEVT == ValueVT)
435 return Val;
436 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
437 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
438
439 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
440 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
441 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
442 }
443
444 // Promoted vector extract
445 return DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueVT);
446 }
447
448 // Trivial bitcast if the types are the same size and the destination
449 // vector type is legal.
450 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
451 TLI.isTypeLegal(VT: ValueVT))
452 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
453
454 if (ValueVT.getVectorNumElements() != 1) {
455 // Certain ABIs require that vectors are passed as integers. For vectors
456 // are the same size, this is an obvious bitcast.
457 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
458 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
459 } else if (ValueVT.bitsLT(VT: PartEVT)) {
460 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
461 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
462 // Drop the extra bits.
463 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
464 return DAG.getBitcast(VT: ValueVT, V: Val);
465 }
466
467 diagnosePossiblyInvalidConstraint(
468 Ctx&: *DAG.getContext(), V, ErrMsg: "non-trivial scalar-to-vector conversion");
469 return DAG.getUNDEF(VT: ValueVT);
470 }
471
472 // Handle cases such as i8 -> <1 x i1>
473 EVT ValueSVT = ValueVT.getVectorElementType();
474 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
475 unsigned ValueSize = ValueSVT.getSizeInBits();
476 if (ValueSize == PartEVT.getSizeInBits()) {
477 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueSVT, Operand: Val);
478 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
479 // It's possible a scalar floating point type gets softened to integer and
480 // then promoted to a larger integer. If PartEVT is the larger integer
481 // we need to truncate it and then bitcast to the FP type.
482 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
483 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
484 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
485 Val = DAG.getBitcast(VT: ValueSVT, V: Val);
486 } else {
487 Val = ValueVT.isFloatingPoint()
488 ? DAG.getFPExtendOrRound(Op: Val, DL, VT: ValueSVT)
489 : DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueSVT);
490 }
491 }
492
493 return DAG.getBuildVector(VT: ValueVT, DL, Ops: Val);
494}
495
496static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
497 SDValue Val, SDValue *Parts, unsigned NumParts,
498 MVT PartVT, const Value *V,
499 std::optional<CallingConv::ID> CallConv);
500
501/// getCopyToParts - Create a series of nodes that contain the specified value
502/// split into legal parts. If the parts contain more bits than Val, then, for
503/// integers, ExtendKind can be used to specify how to generate the extra bits.
504static void
505getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
506 unsigned NumParts, MVT PartVT, const Value *V,
507 std::optional<CallingConv::ID> CallConv = std::nullopt,
508 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
509 // Let the target split the parts if it wants to
510 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
511 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
512 CC: CallConv))
513 return;
514 EVT ValueVT = Val.getValueType();
515
516 // Handle the vector case separately.
517 if (ValueVT.isVector())
518 return getCopyToPartsVector(DAG, dl: DL, Val, Parts, NumParts, PartVT, V,
519 CallConv);
520
521 unsigned OrigNumParts = NumParts;
522 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
523 "Copying to an illegal type!");
524
525 if (NumParts == 0)
526 return;
527
528 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
529 EVT PartEVT = PartVT;
530 if (PartEVT == ValueVT) {
531 assert(NumParts == 1 && "No-op copy with multiple parts!");
532 Parts[0] = Val;
533 return;
534 }
535
536 unsigned PartBits = PartVT.getSizeInBits();
537 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
538 // If the parts cover more bits than the value has, promote the value.
539 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
540 assert(NumParts == 1 && "Do not know what to promote to!");
541 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
542 } else {
543 if (ValueVT.isFloatingPoint()) {
544 // FP values need to be bitcast, then extended if they are being put
545 // into a larger container.
546 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
547 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
548 }
549 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
550 ValueVT.isInteger() &&
551 "Unknown mismatch!");
552 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
553 Val = DAG.getNode(Opcode: ExtendKind, DL, VT: ValueVT, Operand: Val);
554 if (PartVT == MVT::x86mmx)
555 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
556 }
557 } else if (PartBits == ValueVT.getSizeInBits()) {
558 // Different types of the same size.
559 assert(NumParts == 1 && PartEVT != ValueVT);
560 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
561 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
562 // If the parts cover less bits than value has, truncate the value.
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
567 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
570 }
571
572 // The value may have changed - recompute ValueVT.
573 ValueVT = Val.getValueType();
574 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
575 "Failed to tile the value with PartVT!");
576
577 if (NumParts == 1) {
578 if (PartEVT != ValueVT) {
579 diagnosePossiblyInvalidConstraint(Ctx&: *DAG.getContext(), V,
580 ErrMsg: "scalar-to-vector conversion failed");
581 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
582 }
583
584 Parts[0] = Val;
585 return;
586 }
587
588 // Expand the value into multiple parts.
589 if (NumParts & (NumParts - 1)) {
590 // The number of parts is not a power of 2. Split off and copy the tail.
591 assert(PartVT.isInteger() && ValueVT.isInteger() &&
592 "Do not know what to expand to!");
593 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
594 unsigned RoundBits = RoundParts * PartBits;
595 unsigned OddParts = NumParts - RoundParts;
596 SDValue OddVal = DAG.getNode(Opcode: ISD::SRL, DL, VT: ValueVT, N1: Val,
597 N2: DAG.getShiftAmountConstant(Val: RoundBits, VT: ValueVT, DL));
598
599 getCopyToParts(DAG, DL, Val: OddVal, Parts: Parts + RoundParts, NumParts: OddParts, PartVT, V,
600 CallConv);
601
602 if (DAG.getDataLayout().isBigEndian())
603 // The odd parts were reversed by getCopyToParts - unreverse them.
604 std::reverse(first: Parts + RoundParts, last: Parts + NumParts);
605
606 NumParts = RoundParts;
607 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
608 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
609 }
610
611 // The number of parts is a power of 2. Repeatedly bisect the value using
612 // EXTRACT_ELEMENT.
613 Parts[0] = DAG.getNode(Opcode: ISD::BITCAST, DL,
614 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
615 BitWidth: ValueVT.getSizeInBits()),
616 Operand: Val);
617
618 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
619 for (unsigned i = 0; i < NumParts; i += StepSize) {
620 unsigned ThisBits = StepSize * PartBits / 2;
621 EVT ThisVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ThisBits);
622 SDValue &Part0 = Parts[i];
623 SDValue &Part1 = Parts[i+StepSize/2];
624
625 Part1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
626 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 1, DL));
627 Part0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
628 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 0, DL));
629
630 if (ThisBits == PartBits && ThisVT != PartVT) {
631 Part0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part0);
632 Part1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part1);
633 }
634 }
635 }
636
637 if (DAG.getDataLayout().isBigEndian())
638 std::reverse(first: Parts, last: Parts + OrigNumParts);
639}
640
641static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
642 const SDLoc &DL, EVT PartVT) {
643 if (!PartVT.isVector())
644 return SDValue();
645
646 EVT ValueVT = Val.getValueType();
647 EVT PartEVT = PartVT.getVectorElementType();
648 EVT ValueEVT = ValueVT.getVectorElementType();
649 ElementCount PartNumElts = PartVT.getVectorElementCount();
650 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
651
652 // We only support widening vectors with equivalent element types and
653 // fixed/scalable properties. If a target needs to widen a fixed-length type
654 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
655 if (ElementCount::isKnownLE(LHS: PartNumElts, RHS: ValueNumElts) ||
656 PartNumElts.isScalable() != ValueNumElts.isScalable())
657 return SDValue();
658
659 // Have a try for bf16 because some targets share its ABI with fp16.
660 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
661 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
662 "Cannot widen to illegal type");
663 Val = DAG.getNode(
664 Opcode: ISD::BITCAST, DL,
665 VT: ValueVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::f16), Operand: Val);
666 } else if (PartEVT != ValueEVT) {
667 return SDValue();
668 }
669
670 // Widening a scalable vector to another scalable vector is done by inserting
671 // the vector into a larger undef one.
672 if (PartNumElts.isScalable())
673 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: PartVT, N1: DAG.getUNDEF(VT: PartVT),
674 N2: Val, N3: DAG.getVectorIdxConstant(Val: 0, DL));
675
676 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
677 // undef elements.
678 SmallVector<SDValue, 16> Ops;
679 DAG.ExtractVectorElements(Op: Val, Args&: Ops);
680 SDValue EltUndef = DAG.getUNDEF(VT: PartEVT);
681 Ops.append(NumInputs: (PartNumElts - ValueNumElts).getFixedValue(), Elt: EltUndef);
682
683 // FIXME: Use CONCAT for 2x -> 4x.
684 return DAG.getBuildVector(VT: PartVT, DL, Ops);
685}
686
687/// getCopyToPartsVector - Create a series of nodes that contain the specified
688/// value split into legal parts.
689static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
690 SDValue Val, SDValue *Parts, unsigned NumParts,
691 MVT PartVT, const Value *V,
692 std::optional<CallingConv::ID> CallConv) {
693 EVT ValueVT = Val.getValueType();
694 assert(ValueVT.isVector() && "Not a vector");
695 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
696 const bool IsABIRegCopy = CallConv.has_value();
697
698 if (NumParts == 1) {
699 EVT PartEVT = PartVT;
700 if (PartEVT == ValueVT) {
701 // Nothing to do.
702 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
703 // Bitconvert vector->vector case.
704 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
705 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
706 Val = Widened;
707 } else if (PartVT.isVector() &&
708 PartEVT.getVectorElementType().bitsGE(
709 VT: ValueVT.getVectorElementType()) &&
710 PartEVT.getVectorElementCount() ==
711 ValueVT.getVectorElementCount()) {
712
713 // Promoted vector extract
714 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
715 } else if (PartEVT.isVector() &&
716 PartEVT.getVectorElementType() !=
717 ValueVT.getVectorElementType() &&
718 TLI.getTypeAction(Context&: *DAG.getContext(), VT: ValueVT) ==
719 TargetLowering::TypeWidenVector) {
720 // Combination of widening and promotion.
721 EVT WidenVT =
722 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
723 EC: PartVT.getVectorElementCount());
724 SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: WidenVT);
725 Val = DAG.getAnyExtOrTrunc(Op: Widened, DL, VT: PartVT);
726 } else {
727 // Don't extract an integer from a float vector. This can happen if the
728 // FP type gets softened to integer and then promoted. The promotion
729 // prevents it from being picked up by the earlier bitcast case.
730 if (ValueVT.getVectorElementCount().isScalar() &&
731 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
732 // If we reach this condition and PartVT is FP, this means that
733 // ValueVT is also FP and both have a different size, otherwise we
734 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
735 // would be invalid since that would mean the smaller FP type has to
736 // be extended to the larger one.
737 if (PartVT.isFloatingPoint()) {
738 Val = DAG.getBitcast(VT: ValueVT.getScalarType(), V: Val);
739 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
740 } else
741 Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: PartVT, N1: Val,
742 N2: DAG.getVectorIdxConstant(Val: 0, DL));
743 } else {
744 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
745 assert(PartVT.getFixedSizeInBits() > ValueSize &&
746 "lossy conversion of vector to scalar type");
747 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
748 Val = DAG.getBitcast(VT: IntermediateType, V: Val);
749 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
750 }
751 }
752
753 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
754 Parts[0] = Val;
755 return;
756 }
757
758 // Handle a multi-element vector.
759 EVT IntermediateVT;
760 MVT RegisterVT;
761 unsigned NumIntermediates;
762 unsigned NumRegs;
763 if (IsABIRegCopy) {
764 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
765 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT, NumIntermediates,
766 RegisterVT);
767 } else {
768 NumRegs =
769 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
770 NumIntermediates, RegisterVT);
771 }
772
773 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
774 NumParts = NumRegs; // Silence a compiler warning.
775 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
776
777 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
778 "Mixing scalable and fixed vectors when copying in parts");
779
780 std::optional<ElementCount> DestEltCnt;
781
782 if (IntermediateVT.isVector())
783 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
784 else
785 DestEltCnt = ElementCount::getFixed(MinVal: NumIntermediates);
786
787 EVT BuiltVectorTy = EVT::getVectorVT(
788 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(), EC: *DestEltCnt);
789
790 if (ValueVT == BuiltVectorTy) {
791 // Nothing to do.
792 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
793 // Bitconvert vector->vector case.
794 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BuiltVectorTy, Operand: Val);
795 } else {
796 if (BuiltVectorTy.getVectorElementType().bitsGT(
797 VT: ValueVT.getVectorElementType())) {
798 // Integer promotion.
799 ValueVT = EVT::getVectorVT(Context&: *DAG.getContext(),
800 VT: BuiltVectorTy.getVectorElementType(),
801 EC: ValueVT.getVectorElementCount());
802 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
803 }
804
805 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: BuiltVectorTy)) {
806 Val = Widened;
807 }
808 }
809
810 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
811
812 // Split the vector into intermediate operands.
813 SmallVector<SDValue, 8> Ops(NumIntermediates);
814 for (unsigned i = 0; i != NumIntermediates; ++i) {
815 if (IntermediateVT.isVector()) {
816 // This does something sensible for scalable vectors - see the
817 // definition of EXTRACT_SUBVECTOR for further details.
818 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
819 Ops[i] =
820 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: IntermediateVT, N1: Val,
821 N2: DAG.getVectorIdxConstant(Val: i * IntermediateNumElts, DL));
822 } else {
823 Ops[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IntermediateVT, N1: Val,
824 N2: DAG.getVectorIdxConstant(Val: i, DL));
825 }
826 }
827
828 // Split the intermediate operands into legal parts.
829 if (NumParts == NumIntermediates) {
830 // If the register was not expanded, promote or copy the value,
831 // as appropriate.
832 for (unsigned i = 0; i != NumParts; ++i)
833 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i], NumParts: 1, PartVT, V, CallConv);
834 } else if (NumParts > 0) {
835 // If the intermediate type was expanded, split each the value into
836 // legal parts.
837 assert(NumIntermediates != 0 && "division by zero");
838 assert(NumParts % NumIntermediates == 0 &&
839 "Must expand into a divisible number of parts!");
840 unsigned Factor = NumParts / NumIntermediates;
841 for (unsigned i = 0; i != NumIntermediates; ++i)
842 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i * Factor], NumParts: Factor, PartVT, V,
843 CallConv);
844 }
845}
846
847static void failForInvalidBundles(const CallBase &I, StringRef Name,
848 ArrayRef<uint32_t> AllowedBundles) {
849 if (I.hasOperandBundlesOtherThan(IDs: AllowedBundles)) {
850 ListSeparator LS;
851 std::string Error;
852 raw_string_ostream OS(Error);
853 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
854 OperandBundleUse U = I.getOperandBundleAt(Index: i);
855 if (!is_contained(Range&: AllowedBundles, Element: U.getTagID()))
856 OS << LS << U.getTagName();
857 }
858 reportFatalUsageError(
859 reason: Twine("cannot lower ", Name)
860 .concat(Suffix: Twine(" with arbitrary operand bundles: ", Error)));
861 }
862}
863
864RegsForValue::RegsForValue(const SmallVector<Register, 4> &regs, MVT regvt,
865 EVT valuevt, std::optional<CallingConv::ID> CC)
866 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
867 RegCount(1, regs.size()), CallConv(CC) {}
868
869RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
870 const DataLayout &DL, Register Reg, Type *Ty,
871 std::optional<CallingConv::ID> CC) {
872 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
873
874 CallConv = CC;
875
876 for (EVT ValueVT : ValueVTs) {
877 unsigned NumRegs =
878 isABIMangled()
879 ? TLI.getNumRegistersForCallingConv(Context, CC: *CC, VT: ValueVT)
880 : TLI.getNumRegisters(Context, VT: ValueVT);
881 MVT RegisterVT =
882 isABIMangled()
883 ? TLI.getRegisterTypeForCallingConv(Context, CC: *CC, VT: ValueVT)
884 : TLI.getRegisterType(Context, VT: ValueVT);
885 for (unsigned i = 0; i != NumRegs; ++i)
886 Regs.push_back(Elt: Reg + i);
887 RegVTs.push_back(Elt: RegisterVT);
888 RegCount.push_back(Elt: NumRegs);
889 Reg = Reg.id() + NumRegs;
890 }
891}
892
893SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
894 FunctionLoweringInfo &FuncInfo,
895 const SDLoc &dl, SDValue &Chain,
896 SDValue *Glue, const Value *V) const {
897 // A Value with type {} or [0 x %t] needs no registers.
898 if (ValueVTs.empty())
899 return SDValue();
900
901 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
902
903 // Assemble the legal parts into the final values.
904 SmallVector<SDValue, 4> Values(ValueVTs.size());
905 SmallVector<SDValue, 8> Parts;
906 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
907 // Copy the legal parts from the registers.
908 EVT ValueVT = ValueVTs[Value];
909 unsigned NumRegs = RegCount[Value];
910 MVT RegisterVT = isABIMangled()
911 ? TLI.getRegisterTypeForCallingConv(
912 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
913 : RegVTs[Value];
914
915 Parts.resize(N: NumRegs);
916 for (unsigned i = 0; i != NumRegs; ++i) {
917 SDValue P;
918 if (!Glue) {
919 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT);
920 } else {
921 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT, Glue: *Glue);
922 *Glue = P.getValue(R: 2);
923 }
924
925 Chain = P.getValue(R: 1);
926 Parts[i] = P;
927
928 // If the source register was virtual and if we know something about it,
929 // add an assert node.
930 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
931 continue;
932
933 const FunctionLoweringInfo::LiveOutInfo *LOI =
934 FuncInfo.GetLiveOutRegInfo(Reg: Regs[Part+i]);
935 if (!LOI)
936 continue;
937
938 unsigned RegSize = RegisterVT.getScalarSizeInBits();
939 unsigned NumSignBits = LOI->NumSignBits;
940 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
941
942 if (NumZeroBits == RegSize) {
943 // The current value is a zero.
944 // Explicitly express that as it would be easier for
945 // optimizations to kick in.
946 Parts[i] = DAG.getConstant(Val: 0, DL: dl, VT: RegisterVT);
947 continue;
948 }
949
950 // FIXME: We capture more information than the dag can represent. For
951 // now, just use the tightest assertzext/assertsext possible.
952 bool isSExt;
953 EVT FromVT(MVT::Other);
954 if (NumZeroBits) {
955 FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
956 isSExt = false;
957 } else if (NumSignBits > 1) {
958 FromVT =
959 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumSignBits + 1);
960 isSExt = true;
961 } else {
962 continue;
963 }
964 // Add an assertion node.
965 assert(FromVT != MVT::Other);
966 Parts[i] = DAG.getNode(Opcode: isSExt ? ISD::AssertSext : ISD::AssertZext, DL: dl,
967 VT: RegisterVT, N1: P, N2: DAG.getValueType(FromVT));
968 }
969
970 Values[Value] = getCopyFromParts(DAG, DL: dl, Parts: Parts.begin(), NumParts: NumRegs,
971 PartVT: RegisterVT, ValueVT, V, InChain: Chain, CC: CallConv);
972 Part += NumRegs;
973 Parts.clear();
974 }
975
976 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values);
977}
978
979void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
980 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
981 const Value *V,
982 ISD::NodeType PreferredExtendType) const {
983 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
984 ISD::NodeType ExtendKind = PreferredExtendType;
985
986 // Get the list of the values's legal parts.
987 unsigned NumRegs = Regs.size();
988 SmallVector<SDValue, 8> Parts(NumRegs);
989 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
990 unsigned NumParts = RegCount[Value];
991
992 MVT RegisterVT = isABIMangled()
993 ? TLI.getRegisterTypeForCallingConv(
994 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
995 : RegVTs[Value];
996
997 if (ExtendKind == ISD::ANY_EXTEND)
998 if (TLI.isZExtFree(Val: peekThroughFreeze(V: Val), VT2: RegisterVT))
999 ExtendKind = ISD::ZERO_EXTEND;
1000
1001 getCopyToParts(DAG, DL: dl, Val: Val.getValue(R: Val.getResNo() + Value), Parts: &Parts[Part],
1002 NumParts, PartVT: RegisterVT, V, CallConv, ExtendKind);
1003 Part += NumParts;
1004 }
1005
1006 // Copy the parts into the registers.
1007 SmallVector<SDValue, 8> Chains(NumRegs);
1008 for (unsigned i = 0; i != NumRegs; ++i) {
1009 SDValue Part;
1010 if (!Glue) {
1011 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i]);
1012 } else {
1013 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i], Glue: *Glue);
1014 *Glue = Part.getValue(R: 1);
1015 }
1016
1017 Chains[i] = Part.getValue(R: 0);
1018 }
1019
1020 if (NumRegs == 1 || Glue)
1021 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1022 // flagged to it. That is the CopyToReg nodes and the user are considered
1023 // a single scheduling unit. If we create a TokenFactor and return it as
1024 // chain, then the TokenFactor is both a predecessor (operand) of the
1025 // user as well as a successor (the TF operands are flagged to the user).
1026 // c1, f1 = CopyToReg
1027 // c2, f2 = CopyToReg
1028 // c3 = TokenFactor c1, c2
1029 // ...
1030 // = op c3, ..., f2
1031 Chain = Chains[NumRegs-1];
1032 else
1033 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
1034}
1035
1036void RegsForValue::AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching,
1037 unsigned MatchingIdx, const SDLoc &dl,
1038 SelectionDAG &DAG,
1039 std::vector<SDValue> &Ops) const {
1040 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1041
1042 InlineAsm::Flag Flag(Code, Regs.size());
1043 if (HasMatching)
1044 Flag.setMatchingOp(MatchingIdx);
1045 else if (!Regs.empty() && Regs.front().isVirtual()) {
1046 // Put the register class of the virtual registers in the flag word. That
1047 // way, later passes can recompute register class constraints for inline
1048 // assembly as well as normal instructions.
1049 // Don't do this for tied operands that can use the regclass information
1050 // from the def.
1051 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
1052 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Regs.front());
1053 Flag.setRegClass(RC->getID());
1054 }
1055
1056 SDValue Res = DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32);
1057 Ops.push_back(x: Res);
1058
1059 if (Code == InlineAsm::Kind::Clobber) {
1060 // Clobbers should always have a 1:1 mapping with registers, and may
1061 // reference registers that have illegal (e.g. vector) types. Hence, we
1062 // shouldn't try to apply any sort of splitting logic to them.
1063 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1064 "No 1:1 mapping from clobbers to regs?");
1065 Register SP = TLI.getStackPointerRegisterToSaveRestore();
1066 (void)SP;
1067 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1068 Ops.push_back(x: DAG.getRegister(Reg: Regs[I], VT: RegVTs[I]));
1069 assert(
1070 (Regs[I] != SP ||
1071 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
1072 "If we clobbered the stack pointer, MFI should know about it.");
1073 }
1074 return;
1075 }
1076
1077 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1078 MVT RegisterVT = RegVTs[Value];
1079 unsigned NumRegs = TLI.getNumRegisters(Context&: *DAG.getContext(), VT: ValueVTs[Value],
1080 RegisterVT);
1081 for (unsigned i = 0; i != NumRegs; ++i) {
1082 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1083 Register TheReg = Regs[Reg++];
1084 Ops.push_back(x: DAG.getRegister(Reg: TheReg, VT: RegisterVT));
1085 }
1086 }
1087}
1088
1089SmallVector<std::pair<Register, TypeSize>, 4>
1090RegsForValue::getRegsAndSizes() const {
1091 SmallVector<std::pair<Register, TypeSize>, 4> OutVec;
1092 unsigned I = 0;
1093 for (auto CountAndVT : zip_first(t: RegCount, u: RegVTs)) {
1094 unsigned RegCount = std::get<0>(t&: CountAndVT);
1095 MVT RegisterVT = std::get<1>(t&: CountAndVT);
1096 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1097 for (unsigned E = I + RegCount; I != E; ++I)
1098 OutVec.push_back(Elt: std::make_pair(x: Regs[I], y&: RegisterSize));
1099 }
1100 return OutVec;
1101}
1102
1103void SelectionDAGBuilder::init(GCFunctionInfo *gfi, BatchAAResults *aa,
1104 AssumptionCache *ac, const TargetLibraryInfo *li,
1105 const TargetTransformInfo &TTI) {
1106 BatchAA = aa;
1107 AC = ac;
1108 GFI = gfi;
1109 LibInfo = li;
1110 Context = DAG.getContext();
1111 LPadToCallSiteMap.clear();
1112 this->TTI = &TTI;
1113 SL->init(tli: DAG.getTargetLoweringInfo(), tm: TM, dl: DAG.getDataLayout());
1114 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1115 M: *DAG.getMachineFunction().getFunction().getParent());
1116 CanDescribeGlobalAddressInLocationList =
1117 canDescribeGlobalAddressInLocationList(MF: DAG.getMachineFunction());
1118}
1119
1120void SelectionDAGBuilder::clear() {
1121 NodeMap.clear();
1122 UnusedArgNodeMap.clear();
1123 PendingLoads.clear();
1124 PendingExports.clear();
1125 PendingConstrainedFP.clear();
1126 PendingConstrainedFPStrict.clear();
1127 CurInst = nullptr;
1128 HasTailCall = false;
1129 SDNodeOrder = LowestSDNodeOrder;
1130 StatepointLowering.clear();
1131}
1132
1133void SelectionDAGBuilder::clearDanglingDebugInfo() {
1134 DanglingDebugInfoMap.clear();
1135}
1136
1137// Update DAG root to include dependencies on Pending chains.
1138SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1139 SDValue Root = DAG.getRoot();
1140
1141 if (Pending.empty())
1142 return Root;
1143
1144 // Add current root to PendingChains, unless we already indirectly
1145 // depend on it.
1146 if (Root.getOpcode() != ISD::EntryToken) {
1147 unsigned i = 0, e = Pending.size();
1148 for (; i != e; ++i) {
1149 assert(Pending[i].getNode()->getNumOperands() > 1);
1150 if (Pending[i].getNode()->getOperand(Num: 0) == Root)
1151 break; // Don't add the root if we already indirectly depend on it.
1152 }
1153
1154 if (i == e)
1155 Pending.push_back(Elt: Root);
1156 }
1157
1158 if (Pending.size() == 1)
1159 Root = Pending[0];
1160 else
1161 Root = DAG.getTokenFactor(DL: getCurSDLoc(), Vals&: Pending);
1162
1163 DAG.setRoot(Root);
1164 Pending.clear();
1165 return Root;
1166}
1167
1168SDValue SelectionDAGBuilder::getMemoryRoot() {
1169 return updateRoot(Pending&: PendingLoads);
1170}
1171
1172SDValue SelectionDAGBuilder::getFPOperationRoot(fp::ExceptionBehavior EB) {
1173 // If the new exception behavior differs from that of the pending
1174 // ones, chain up them and update the root.
1175 switch (EB) {
1176 case fp::ExceptionBehavior::ebMayTrap:
1177 case fp::ExceptionBehavior::ebIgnore:
1178 // Floating-point exceptions produced by such operations are not intended
1179 // to be observed, so the sequence of these operations does not need to be
1180 // preserved.
1181 //
1182 // They however must not be mixed with the instructions that have strict
1183 // exception behavior. Placing an operation with 'ebIgnore' behavior between
1184 // 'ebStrict' operations could distort the observed exception behavior.
1185 if (!PendingConstrainedFPStrict.empty()) {
1186 assert(PendingConstrainedFP.empty());
1187 updateRoot(Pending&: PendingConstrainedFPStrict);
1188 }
1189 break;
1190 case fp::ExceptionBehavior::ebStrict:
1191 // Floating-point exception produced by these operations may be observed, so
1192 // they must be correctly chained. If trapping on FP exceptions is
1193 // disabled, the exceptions can be observed only by functions that read
1194 // exception flags, like 'llvm.get_fpenv' or 'fetestexcept'. It means that
1195 // the order of operations is not significant between barriers.
1196 //
1197 // If trapping is enabled, each operation becomes an implicit observation
1198 // point, so the operations must be sequenced according their original
1199 // source order.
1200 if (!PendingConstrainedFP.empty()) {
1201 assert(PendingConstrainedFPStrict.empty());
1202 updateRoot(Pending&: PendingConstrainedFP);
1203 }
1204 // TODO: Add support for trapping-enabled scenarios.
1205 }
1206 return DAG.getRoot();
1207}
1208
1209SDValue SelectionDAGBuilder::getRoot() {
1210 // Chain up all pending constrained intrinsics together with all
1211 // pending loads, by simply appending them to PendingLoads and
1212 // then calling getMemoryRoot().
1213 PendingLoads.reserve(N: PendingLoads.size() +
1214 PendingConstrainedFP.size() +
1215 PendingConstrainedFPStrict.size());
1216 PendingLoads.append(in_start: PendingConstrainedFP.begin(),
1217 in_end: PendingConstrainedFP.end());
1218 PendingLoads.append(in_start: PendingConstrainedFPStrict.begin(),
1219 in_end: PendingConstrainedFPStrict.end());
1220 PendingConstrainedFP.clear();
1221 PendingConstrainedFPStrict.clear();
1222 return getMemoryRoot();
1223}
1224
1225SDValue SelectionDAGBuilder::getControlRoot() {
1226 // We need to emit pending fpexcept.strict constrained intrinsics,
1227 // so append them to the PendingExports list.
1228 PendingExports.append(in_start: PendingConstrainedFPStrict.begin(),
1229 in_end: PendingConstrainedFPStrict.end());
1230 PendingConstrainedFPStrict.clear();
1231 return updateRoot(Pending&: PendingExports);
1232}
1233
1234void SelectionDAGBuilder::handleDebugDeclare(Value *Address,
1235 DILocalVariable *Variable,
1236 DIExpression *Expression,
1237 DebugLoc DL) {
1238 assert(Variable && "Missing variable");
1239
1240 // Check if address has undef value.
1241 if (!Address || isa<UndefValue>(Val: Address) ||
1242 (Address->use_empty() && !isa<Argument>(Val: Address))) {
1243 LLVM_DEBUG(
1244 dbgs()
1245 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1246 return;
1247 }
1248
1249 bool IsParameter = Variable->isParameter() || isa<Argument>(Val: Address);
1250
1251 SDValue &N = NodeMap[Address];
1252 if (!N.getNode() && isa<Argument>(Val: Address))
1253 // Check unused arguments map.
1254 N = UnusedArgNodeMap[Address];
1255 SDDbgValue *SDV;
1256 if (N.getNode()) {
1257 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Val: Address))
1258 Address = BCI->getOperand(i_nocapture: 0);
1259 // Parameters are handled specially.
1260 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: N.getNode());
1261 if (IsParameter && FINode) {
1262 // Byval parameter. We have a frame index at this point.
1263 SDV = DAG.getFrameIndexDbgValue(Var: Variable, Expr: Expression, FI: FINode->getIndex(),
1264 /*IsIndirect*/ true, DL, O: SDNodeOrder);
1265 } else if (isa<Argument>(Val: Address)) {
1266 // Address is an argument, so try to emit its dbg value using
1267 // virtual register info from the FuncInfo.ValueMap.
1268 EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1269 Kind: FuncArgumentDbgValueKind::Declare, N);
1270 return;
1271 } else {
1272 SDV = DAG.getDbgValue(Var: Variable, Expr: Expression, N: N.getNode(), R: N.getResNo(),
1273 IsIndirect: true, DL, O: SDNodeOrder);
1274 }
1275 DAG.AddDbgValue(DB: SDV, isParameter: IsParameter);
1276 } else {
1277 // If Address is an argument then try to emit its dbg value using
1278 // virtual register info from the FuncInfo.ValueMap.
1279 if (!EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1280 Kind: FuncArgumentDbgValueKind::Declare, N)) {
1281 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1282 << " (could not emit func-arg dbg_value)\n");
1283 }
1284 }
1285}
1286
1287void SelectionDAGBuilder::visitDbgInfo(const Instruction &I) {
1288 // Add SDDbgValue nodes for any var locs here. Do so before updating
1289 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1290 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1291 // Add SDDbgValue nodes for any var locs here. Do so before updating
1292 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1293 for (auto It = FnVarLocs->locs_begin(Before: &I), End = FnVarLocs->locs_end(Before: &I);
1294 It != End; ++It) {
1295 auto *Var = FnVarLocs->getDILocalVariable(ID: It->VariableID);
1296 dropDanglingDebugInfo(Variable: Var, Expr: It->Expr);
1297 if (It->Values.isKillLocation(Expression: It->Expr)) {
1298 handleKillDebugValue(Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder);
1299 continue;
1300 }
1301 SmallVector<Value *> Values(It->Values.location_ops());
1302 if (!handleDebugValue(Values, Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder,
1303 IsVariadic: It->Values.hasArgList())) {
1304 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1305 addDanglingDebugInfo(Values&: Vals,
1306 Var: FnVarLocs->getDILocalVariable(ID: It->VariableID),
1307 Expr: It->Expr, IsVariadic: Vals.size() > 1, DL: It->DL, Order: SDNodeOrder);
1308 }
1309 }
1310 }
1311
1312 // We must skip DbgVariableRecords if they've already been processed above as
1313 // we have just emitted the debug values resulting from assignment tracking
1314 // analysis, making any existing DbgVariableRecords redundant (and probably
1315 // less correct). We still need to process DbgLabelRecords. This does sink
1316 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1317 // be important as it does so deterministcally and ordering between
1318 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1319 // printing).
1320 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1321 // Is there is any debug-info attached to this instruction, in the form of
1322 // DbgRecord non-instruction debug-info records.
1323 for (DbgRecord &DR : I.getDbgRecordRange()) {
1324 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1325 assert(DLR->getLabel() && "Missing label");
1326 SDDbgLabel *SDV =
1327 DAG.getDbgLabel(Label: DLR->getLabel(), DL: DLR->getDebugLoc(), O: SDNodeOrder);
1328 DAG.AddDbgLabel(DB: SDV);
1329 continue;
1330 }
1331
1332 if (SkipDbgVariableRecords)
1333 continue;
1334 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1335 DILocalVariable *Variable = DVR.getVariable();
1336 DIExpression *Expression = DVR.getExpression();
1337 dropDanglingDebugInfo(Variable, Expr: Expression);
1338
1339 if (DVR.getType() == DbgVariableRecord::LocationType::Declare) {
1340 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1341 continue;
1342 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1343 << "\n");
1344 handleDebugDeclare(Address: DVR.getVariableLocationOp(OpIdx: 0), Variable, Expression,
1345 DL: DVR.getDebugLoc());
1346 continue;
1347 }
1348
1349 // A DbgVariableRecord with no locations is a kill location.
1350 SmallVector<Value *, 4> Values(DVR.location_ops());
1351 if (Values.empty()) {
1352 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1353 Order: SDNodeOrder);
1354 continue;
1355 }
1356
1357 // A DbgVariableRecord with an undef or absent location is also a kill
1358 // location.
1359 if (llvm::any_of(Range&: Values,
1360 P: [](Value *V) { return !V || isa<UndefValue>(Val: V); })) {
1361 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1362 Order: SDNodeOrder);
1363 continue;
1364 }
1365
1366 bool IsVariadic = DVR.hasArgList();
1367 if (!handleDebugValue(Values, Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1368 Order: SDNodeOrder, IsVariadic)) {
1369 addDanglingDebugInfo(Values, Var: Variable, Expr: Expression, IsVariadic,
1370 DL: DVR.getDebugLoc(), Order: SDNodeOrder);
1371 }
1372 }
1373}
1374
1375void SelectionDAGBuilder::visit(const Instruction &I) {
1376 visitDbgInfo(I);
1377
1378 // Set up outgoing PHI node register values before emitting the terminator.
1379 if (I.isTerminator()) {
1380 HandlePHINodesInSuccessorBlocks(LLVMBB: I.getParent());
1381 }
1382
1383 ++SDNodeOrder;
1384 CurInst = &I;
1385
1386 // Set inserted listener only if required.
1387 bool NodeInserted = false;
1388 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1389 MDNode *PCSectionsMD = I.getMetadata(KindID: LLVMContext::MD_pcsections);
1390 MDNode *MMRA = I.getMetadata(KindID: LLVMContext::MD_mmra);
1391 if (PCSectionsMD || MMRA) {
1392 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1393 args&: DAG, args: [&](SDNode *) { NodeInserted = true; });
1394 }
1395
1396 visit(Opcode: I.getOpcode(), I);
1397
1398 if (!I.isTerminator() && !HasTailCall &&
1399 !isa<GCStatepointInst>(Val: I)) // statepoints handle their exports internally
1400 CopyToExportRegsIfNeeded(V: &I);
1401
1402 // Handle metadata.
1403 if (PCSectionsMD || MMRA) {
1404 auto It = NodeMap.find(Val: &I);
1405 if (It != NodeMap.end()) {
1406 if (PCSectionsMD)
1407 DAG.addPCSections(Node: It->second.getNode(), MD: PCSectionsMD);
1408 if (MMRA)
1409 DAG.addMMRAMetadata(Node: It->second.getNode(), MMRA);
1410 } else if (NodeInserted) {
1411 // This should not happen; if it does, don't let it go unnoticed so we can
1412 // fix it. Relevant visit*() function is probably missing a setValue().
1413 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1414 << I.getModule()->getName() << "]\n";
1415 LLVM_DEBUG(I.dump());
1416 assert(false);
1417 }
1418 }
1419
1420 CurInst = nullptr;
1421}
1422
1423void SelectionDAGBuilder::visitPHI(const PHINode &) {
1424 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1425}
1426
1427void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1428 // Note: this doesn't use InstVisitor, because it has to work with
1429 // ConstantExpr's in addition to instructions.
1430 switch (Opcode) {
1431 default: llvm_unreachable("Unknown instruction type encountered!");
1432 // Build the switch statement using the Instruction.def file.
1433#define HANDLE_INST(NUM, OPCODE, CLASS) \
1434 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1435#include "llvm/IR/Instruction.def"
1436 }
1437}
1438
1439static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,
1440 DILocalVariable *Variable,
1441 DebugLoc DL, unsigned Order,
1442 SmallVectorImpl<Value *> &Values,
1443 DIExpression *Expression) {
1444 // For variadic dbg_values we will now insert poison.
1445 // FIXME: We can potentially recover these!
1446 SmallVector<SDDbgOperand, 2> Locs;
1447 for (const Value *V : Values) {
1448 auto *Poison = PoisonValue::get(T: V->getType());
1449 Locs.push_back(Elt: SDDbgOperand::fromConst(Const: Poison));
1450 }
1451 SDDbgValue *SDV = DAG.getDbgValueList(Var: Variable, Expr: Expression, Locs, Dependencies: {},
1452 /*IsIndirect=*/false, DL, O: Order,
1453 /*IsVariadic=*/true);
1454 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1455 return true;
1456}
1457
1458void SelectionDAGBuilder::addDanglingDebugInfo(SmallVectorImpl<Value *> &Values,
1459 DILocalVariable *Var,
1460 DIExpression *Expr,
1461 bool IsVariadic, DebugLoc DL,
1462 unsigned Order) {
1463 if (IsVariadic) {
1464 handleDanglingVariadicDebugInfo(DAG, Variable: Var, DL, Order, Values, Expression: Expr);
1465 return;
1466 }
1467 // TODO: Dangling debug info will eventually either be resolved or produce
1468 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1469 // between the original dbg.value location and its resolved DBG_VALUE,
1470 // which we should ideally fill with an extra poison DBG_VALUE.
1471 assert(Values.size() == 1);
1472 DanglingDebugInfoMap[Values[0]].emplace_back(args&: Var, args&: Expr, args&: DL, args&: Order);
1473}
1474
1475void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1476 const DIExpression *Expr) {
1477 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1478 DIVariable *DanglingVariable = DDI.getVariable();
1479 DIExpression *DanglingExpr = DDI.getExpression();
1480 if (DanglingVariable == Variable && Expr->fragmentsOverlap(Other: DanglingExpr)) {
1481 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1482 << printDDI(nullptr, DDI) << "\n");
1483 return true;
1484 }
1485 return false;
1486 };
1487
1488 for (auto &DDIMI : DanglingDebugInfoMap) {
1489 DanglingDebugInfoVector &DDIV = DDIMI.second;
1490
1491 // If debug info is to be dropped, run it through final checks to see
1492 // whether it can be salvaged.
1493 for (auto &DDI : DDIV)
1494 if (isMatchingDbgValue(DDI))
1495 salvageUnresolvedDbgValue(V: DDIMI.first, DDI);
1496
1497 erase_if(C&: DDIV, P: isMatchingDbgValue);
1498 }
1499}
1500
1501// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1502// generate the debug data structures now that we've seen its definition.
1503void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1504 SDValue Val) {
1505 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(Key: V);
1506 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1507 return;
1508
1509 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1510 for (auto &DDI : DDIV) {
1511 DebugLoc DL = DDI.getDebugLoc();
1512 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1513 DILocalVariable *Variable = DDI.getVariable();
1514 DIExpression *Expr = DDI.getExpression();
1515 assert(Variable->isValidLocationForIntrinsic(DL) &&
1516 "Expected inlined-at fields to agree");
1517 SDDbgValue *SDV;
1518 if (Val.getNode()) {
1519 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1520 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1521 // we couldn't resolve it directly when examining the DbgValue intrinsic
1522 // in the first place we should not be more successful here). Unless we
1523 // have some test case that prove this to be correct we should avoid
1524 // calling EmitFuncArgumentDbgValue here.
1525 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1526 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1527 Kind: FuncArgumentDbgValueKind::Value, N: Val)) {
1528 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1529 << printDDI(V, DDI) << "\n");
1530 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1531 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1532 // inserted after the definition of Val when emitting the instructions
1533 // after ISel. An alternative could be to teach
1534 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1535 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1536 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1537 << ValSDNodeOrder << "\n");
1538 SDV = getDbgValue(N: Val, Variable, Expr, dl: DL,
1539 DbgSDNodeOrder: std::max(a: DbgSDNodeOrder, b: ValSDNodeOrder));
1540 DAG.AddDbgValue(DB: SDV, isParameter: false);
1541 } else
1542 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1543 << printDDI(V, DDI)
1544 << " in EmitFuncArgumentDbgValue\n");
1545 } else {
1546 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1547 << "\n");
1548 auto Poison = PoisonValue::get(T: V->getType());
1549 auto SDV =
1550 DAG.getConstantDbgValue(Var: Variable, Expr, C: Poison, DL, O: DbgSDNodeOrder);
1551 DAG.AddDbgValue(DB: SDV, isParameter: false);
1552 }
1553 }
1554 DDIV.clear();
1555}
1556
1557/// If \p V is the address of a describable global, possibly displaced by a
1558/// constant, return the global and fold the displacement into location operand
1559/// \p OpIdx of \p Expr. The displacement rides along in the expression rather
1560/// than in the operand, so that it survives into a DBG_INSTR_REF.
1561static const GlobalValue *
1562getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx,
1563 const MachineFunction &MF) {
1564 const auto *C = dyn_cast<Constant>(Val: V);
1565 if (!C)
1566 return nullptr;
1567 int64_t Offset;
1568 const GlobalValue *GV = getDescribableGlobalAddress(C, Offset, MF);
1569 if (GV && Offset) {
1570 SmallVector<uint64_t, 3> Ops;
1571 DIExpression::appendOffset(Ops, Offset);
1572 Expr = DIExpression::appendOpsToArg(Expr, Ops, ArgNo: OpIdx, /*StackValue=*/false);
1573 }
1574 return GV;
1575}
1576
1577void SelectionDAGBuilder::salvageUnresolvedDbgValue(const Value *V,
1578 DanglingDebugInfo &DDI) {
1579 // TODO: For the variadic implementation, instead of only checking the fail
1580 // state of `handleDebugValue`, we need know specifically which values were
1581 // invalid, so that we attempt to salvage only those values when processing
1582 // a DIArgList.
1583 const Value *OrigV = V;
1584 DILocalVariable *Var = DDI.getVariable();
1585 DIExpression *Expr = DDI.getExpression();
1586 DebugLoc DL = DDI.getDebugLoc();
1587 unsigned SDOrder = DDI.getSDNodeOrder();
1588
1589 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1590 // that DW_OP_stack_value is desired.
1591 bool StackValue = true;
1592
1593 // handleDebugValue holds out for a register with the address of a global
1594 // that a location list could not name. With no such register forthcoming,
1595 // naming the global still beats dropping the location.
1596 auto HandleGlobalAddress = [&] {
1597 DIExpression *GVExpr = Expr;
1598 const GlobalValue *GV =
1599 getGlobalAddressDbgOperand(V, Expr&: GVExpr, OpIdx: 0, MF: DAG.getMachineFunction());
1600 if (!GV)
1601 return false;
1602 SDDbgValue *SDV = DAG.getDbgValueList(
1603 Var, Expr: GVExpr, Locs: SDDbgOperand::fromGlobalAddr(GV), /*Dependencies=*/{},
1604 /*IsIndirect=*/false, DL, O: SDOrder, /*IsVariadic=*/false);
1605 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1606 return true;
1607 };
1608
1609 // Can this Value can be encoded without any further work?
1610 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false) ||
1611 HandleGlobalAddress())
1612 return;
1613
1614 // Attempt to salvage back through as many instructions as possible. Bail if
1615 // a non-instruction is seen, such as a constant expression or global
1616 // variable. FIXME: Further work could recover those too.
1617 while (isa<Instruction>(Val: V)) {
1618 const Instruction &VAsInst = *cast<const Instruction>(Val: V);
1619 // Temporary "0", awaiting real implementation.
1620 SmallVector<uint64_t, 16> Ops;
1621 SmallVector<Value *, 4> AdditionalValues;
1622 V = salvageDebugInfoImpl(I&: const_cast<Instruction &>(VAsInst),
1623 CurrentLocOps: Expr->getNumLocationOperands(), Ops,
1624 AdditionalValues);
1625 // If we cannot salvage any further, and haven't yet found a suitable debug
1626 // expression, bail out.
1627 if (!V)
1628 break;
1629
1630 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1631 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1632 // here for variadic dbg_values, remove that condition.
1633 if (!AdditionalValues.empty())
1634 break;
1635
1636 // New value and expr now represent this debuginfo.
1637 Expr = DIExpression::appendOpsToArg(Expr, Ops, ArgNo: 0, StackValue);
1638
1639 // Some kind of simplification occurred: check whether the operand of the
1640 // salvaged debug expression can be encoded in this DAG.
1641 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false) ||
1642 HandleGlobalAddress()) {
1643 LLVM_DEBUG(
1644 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1645 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1646 return;
1647 }
1648 }
1649
1650 // This was the final opportunity to salvage this debug information, and it
1651 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1652 // any earlier variable location.
1653 assert(OrigV && "V shouldn't be null");
1654 auto *Poison = PoisonValue::get(T: OrigV->getType());
1655 auto *SDV = DAG.getConstantDbgValue(Var, Expr, C: Poison, DL, O: SDNodeOrder);
1656 DAG.AddDbgValue(DB: SDV, isParameter: false);
1657 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1658 << printDDI(OrigV, DDI) << "\n");
1659}
1660
1661void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var,
1662 DIExpression *Expr,
1663 DebugLoc DbgLoc,
1664 unsigned Order) {
1665 Value *Poison = PoisonValue::get(T: Type::getInt1Ty(C&: *Context));
1666 DIExpression *NewExpr =
1667 const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr));
1668 handleDebugValue(Values: Poison, Var, Expr: NewExpr, DbgLoc, Order,
1669 /*IsVariadic*/ false);
1670}
1671
1672bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1673 DILocalVariable *Var,
1674 DIExpression *Expr, DebugLoc DbgLoc,
1675 unsigned Order, bool IsVariadic) {
1676 if (Values.empty())
1677 return true;
1678
1679 // Filter EntryValue locations out early.
1680 if (visitEntryValueDbgValue(Values, Variable: Var, Expr, DbgLoc))
1681 return true;
1682
1683 SmallVector<SDDbgOperand> LocationOps;
1684 SmallVector<SDNode *> Dependencies;
1685 for (const auto &[OpIdx, V] : enumerate(First&: Values)) {
1686 // Constant value.
1687 if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V) || isa<UndefValue>(Val: V) ||
1688 isa<ConstantPointerNull>(Val: V)) {
1689 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: V));
1690 continue;
1691 }
1692
1693 // Look through IntToPtr constants.
1694 if (auto *CE = dyn_cast<ConstantExpr>(Val: V))
1695 if (CE->getOpcode() == Instruction::IntToPtr) {
1696 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: CE->getOperand(i_nocapture: 0)));
1697 continue;
1698 }
1699
1700 // The address of a global is a link-time constant, and so is a constant
1701 // displacement from one. A global whose address cannot be described this
1702 // way falls through to be described by whatever materializes it instead.
1703 // So does one that a location list could not name, should the variable
1704 // need one; salvageUnresolvedDbgValue names it if nothing materializes it.
1705 if (CanDescribeGlobalAddressInLocationList)
1706 if (const GlobalValue *GV = getGlobalAddressDbgOperand(
1707 V, Expr, OpIdx, MF: DAG.getMachineFunction())) {
1708 LocationOps.emplace_back(Args: SDDbgOperand::fromGlobalAddr(GV));
1709 continue;
1710 }
1711
1712 // If the Value is a frame index, we can create a FrameIndex debug value
1713 // without relying on the DAG at all.
1714 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
1715 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
1716 if (SI != FuncInfo.StaticAllocaMap.end()) {
1717 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: SI->second));
1718 continue;
1719 }
1720 }
1721
1722 // Do not use getValue() in here; we don't want to generate code at
1723 // this point if it hasn't been done yet.
1724 SDValue N = NodeMap[V];
1725 if (!N.getNode() && isa<Argument>(Val: V)) // Check unused arguments map.
1726 N = UnusedArgNodeMap[V];
1727
1728 if (N.getNode()) {
1729 // Only emit func arg dbg value for non-variadic dbg.values for now.
1730 if (!IsVariadic &&
1731 EmitFuncArgumentDbgValue(V, Variable: Var, Expr, DL: DbgLoc,
1732 Kind: FuncArgumentDbgValueKind::Value, N))
1733 return true;
1734 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
1735 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1736 // describe stack slot locations.
1737 //
1738 // Consider "int x = 0; int *px = &x;". There are two kinds of
1739 // interesting debug values here after optimization:
1740 //
1741 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1742 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1743 //
1744 // Both describe the direct values of their associated variables.
1745 Dependencies.push_back(Elt: N.getNode());
1746 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex()));
1747 continue;
1748 }
1749 LocationOps.emplace_back(
1750 Args: SDDbgOperand::fromNode(Node: N.getNode(), ResNo: N.getResNo()));
1751 continue;
1752 }
1753
1754 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1755 // Special rules apply for the first dbg.values of parameter variables in a
1756 // function. Identify them by the fact they reference Argument Values, that
1757 // they're parameters, and they are parameters of the current function. We
1758 // need to let them dangle until they get an SDNode.
1759 bool IsParamOfFunc =
1760 isa<Argument>(Val: V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1761 if (IsParamOfFunc)
1762 return false;
1763
1764 // The value is not used in this block yet (or it would have an SDNode).
1765 // We still want the value to appear for the user if possible -- if it has
1766 // an associated VReg, we can refer to that instead.
1767 auto VMI = FuncInfo.ValueMap.find(Val: V);
1768 if (VMI != FuncInfo.ValueMap.end()) {
1769 Register Reg = VMI->second;
1770 // If this is a PHI node, it may be split up into several MI PHI nodes
1771 // (in FunctionLoweringInfo::set).
1772 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1773 V->getType(), std::nullopt);
1774 if (RFV.occupiesMultipleRegs()) {
1775 // FIXME: We could potentially support variadic dbg_values here.
1776 if (IsVariadic)
1777 return false;
1778 unsigned Offset = 0;
1779 unsigned BitsToDescribe = 0;
1780 if (auto VarSize = Var->getSizeInBits())
1781 BitsToDescribe = *VarSize;
1782 if (auto Fragment = Expr->getFragmentInfo())
1783 BitsToDescribe = Fragment->SizeInBits;
1784 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1785 // Bail out if all bits are described already.
1786 if (Offset >= BitsToDescribe)
1787 break;
1788 // TODO: handle scalable vectors.
1789 unsigned RegisterSize = RegAndSize.second;
1790 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1791 ? BitsToDescribe - Offset
1792 : RegisterSize;
1793 auto FragmentExpr = DIExpression::createFragmentExpression(
1794 Expr, OffsetInBits: Offset, SizeInBits: FragmentSize);
1795 if (!FragmentExpr)
1796 continue;
1797 SDDbgValue *SDV = DAG.getVRegDbgValue(
1798 Var, Expr: *FragmentExpr, VReg: RegAndSize.first, IsIndirect: false, DL: DbgLoc, O: Order);
1799 DAG.AddDbgValue(DB: SDV, isParameter: false);
1800 Offset += RegisterSize;
1801 }
1802 return true;
1803 }
1804 // We can use simple vreg locations for variadic dbg_values as well.
1805 LocationOps.emplace_back(Args: SDDbgOperand::fromVReg(VReg: Reg));
1806 continue;
1807 }
1808 // We failed to create a SDDbgOperand for V.
1809 return false;
1810 }
1811
1812 // We have created a SDDbgOperand for each Value in Values.
1813 assert(!LocationOps.empty());
1814 SDDbgValue *SDV =
1815 DAG.getDbgValueList(Var, Expr, Locs: LocationOps, Dependencies,
1816 /*IsIndirect=*/false, DL: DbgLoc, O: Order, IsVariadic);
1817 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1818 return true;
1819}
1820
1821void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1822 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1823 for (auto &Pair : DanglingDebugInfoMap)
1824 for (auto &DDI : Pair.second)
1825 salvageUnresolvedDbgValue(V: const_cast<Value *>(Pair.first), DDI);
1826 clearDanglingDebugInfo();
1827}
1828
1829/// getCopyFromRegs - If there was virtual register allocated for the value V
1830/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1831SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1832 auto It = FuncInfo.ValueMap.find(Val: V);
1833 SDValue Result;
1834
1835 if (It != FuncInfo.ValueMap.end()) {
1836 Register InReg = It->second;
1837
1838 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1839 DAG.getDataLayout(), InReg, Ty,
1840 std::nullopt); // This is not an ABI copy.
1841 SDValue Chain = DAG.getEntryNode();
1842 Result = RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr,
1843 V);
1844 resolveDanglingDebugInfo(V, Val: Result);
1845 }
1846
1847 return Result;
1848}
1849
1850/// getValue - Return an SDValue for the given Value.
1851SDValue SelectionDAGBuilder::getValue(const Value *V) {
1852 // If we already have an SDValue for this value, use it. It's important
1853 // to do this first, so that we don't create a CopyFromReg if we already
1854 // have a regular SDValue.
1855 SDValue &N = NodeMap[V];
1856 if (N.getNode()) return N;
1857
1858 // If there's a virtual register allocated and initialized for this
1859 // value, use it.
1860 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
1861 return copyFromReg;
1862
1863 // Otherwise create a new SDValue and remember it.
1864 SDValue Val = getValueImpl(V);
1865 NodeMap[V] = Val;
1866 resolveDanglingDebugInfo(V, Val);
1867 return Val;
1868}
1869
1870void SelectionDAGBuilder::setValueToPoison(const Value *V, const SDLoc &dl) {
1871 if (V->getType()->isVoidTy())
1872 return;
1873
1874 SmallVector<EVT, 4> ValueVTs;
1875 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
1876 Ty: V->getType(), ValueVTs);
1877 if (ValueVTs.empty())
1878 return;
1879 setValue(V, NewN: DAG.getErrorMergeValues(ResultTypes: ValueVTs, Chain: SDValue(), dl));
1880}
1881
1882/// getNonRegisterValue - Return an SDValue for the given Value, but
1883/// don't look in FuncInfo.ValueMap for a virtual register.
1884SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1885 // If we already have an SDValue for this value, use it.
1886 SDValue &N = NodeMap[V];
1887 if (N.getNode()) {
1888 if (isIntOrFPConstant(V: N)) {
1889 // Remove the debug location from the node as the node is about to be used
1890 // in a location which may differ from the original debug location. This
1891 // is relevant to Constant and ConstantFP nodes because they can appear
1892 // as constant expressions inside PHI nodes.
1893 N->setDebugLoc(DebugLoc());
1894 }
1895 return N;
1896 }
1897
1898 // Otherwise create a new SDValue and remember it.
1899 SDValue Val = getValueImpl(V);
1900 NodeMap[V] = Val;
1901 resolveDanglingDebugInfo(V, Val);
1902 return Val;
1903}
1904
1905/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1906/// Create an SDValue for the given value.
1907SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1908 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1909
1910 if (const Constant *C = dyn_cast<Constant>(Val: V)) {
1911 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: V->getType(), AllowUnknown: true);
1912
1913 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
1914 SDLoc DL = getCurSDLoc();
1915
1916 // DAG.getConstant() may attempt to legalise the vector constant which can
1917 // significantly change the combines applied to the DAG. To reduce the
1918 // divergence when enabling ConstantInt based vectors we try to construct
1919 // the DAG in the same way as shufflevector based splats. TODO: The
1920 // divergence sometimes leads to better optimisations. Ideally we should
1921 // prevent DAG.getConstant() from legalising too early but there are some
1922 // degradations preventing this.
1923 if (VT.isScalableVector())
1924 return DAG.getNode(
1925 Opcode: ISD::SPLAT_VECTOR, DL, VT,
1926 Operand: DAG.getConstant(Val: CI->getValue(), DL, VT: VT.getVectorElementType()));
1927 if (VT.isFixedLengthVector())
1928 return DAG.getSplatBuildVector(
1929 VT, DL,
1930 Op: DAG.getConstant(Val: CI->getValue(), DL, VT: VT.getVectorElementType()));
1931 return DAG.getConstant(Val: *CI, DL, VT);
1932 }
1933
1934 if (const ConstantByte *CB = dyn_cast<ConstantByte>(Val: C))
1935 return DAG.getConstant(Val: CB->getValue(), DL: getCurSDLoc(), VT);
1936
1937 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: C))
1938 return DAG.getGlobalAddress(GV, DL: getCurSDLoc(), VT);
1939
1940 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
1941 return DAG.getNode(Opcode: ISD::PtrAuthGlobalAddress, DL: getCurSDLoc(), VT,
1942 N1: getValue(V: CPA->getPointer()), N2: getValue(V: CPA->getKey()),
1943 N3: getValue(V: CPA->getAddrDiscriminator()),
1944 N4: getValue(V: CPA->getDiscriminator()));
1945 }
1946
1947 if (isa<ConstantPointerNull>(Val: C))
1948 return DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT);
1949
1950 if (match(V: C, P: m_VScale()))
1951 return DAG.getVScale(DL: getCurSDLoc(), VT, MulImm: APInt(VT.getSizeInBits(), 1));
1952
1953 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C))
1954 return DAG.getConstantFP(V: *CFP, DL: getCurSDLoc(), VT);
1955
1956 if (isa<UndefValue>(Val: C) && !V->getType()->isAggregateType())
1957 return isa<PoisonValue>(Val: C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1958
1959 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
1960 visit(Opcode: CE->getOpcode(), I: *CE);
1961 SDValue N1 = NodeMap[V];
1962 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1963 return N1;
1964 }
1965
1966 if (isa<ConstantStruct>(Val: C) || isa<ConstantArray>(Val: C)) {
1967 SmallVector<SDValue, 4> Constants;
1968 for (const Use &U : C->operands()) {
1969 SDNode *Val = getValue(V: U).getNode();
1970 // If the operand is an empty aggregate, there are no values.
1971 if (!Val) continue;
1972 // Add each leaf value from the operand to the Constants list
1973 // to form a flattened list of all the values.
1974 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1975 Constants.push_back(Elt: SDValue(Val, i));
1976 }
1977
1978 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
1979 }
1980
1981 if (const ConstantDataSequential *CDS =
1982 dyn_cast<ConstantDataSequential>(Val: C)) {
1983 SmallVector<SDValue, 4> Ops;
1984 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1985 SDNode *Val = getValue(V: CDS->getElementAsConstant(i)).getNode();
1986 // Add each leaf value from the operand to the Constants list
1987 // to form a flattened list of all the values.
1988 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1989 Ops.push_back(Elt: SDValue(Val, i));
1990 }
1991
1992 if (isa<ArrayType>(Val: CDS->getType()))
1993 return DAG.getMergeValues(Ops, dl: getCurSDLoc());
1994 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
1995 }
1996
1997 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1998 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1999 "Unknown struct or array constant!");
2000
2001 SmallVector<EVT, 4> ValueVTs;
2002 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: C->getType(), ValueVTs);
2003 unsigned NumElts = ValueVTs.size();
2004 if (NumElts == 0)
2005 return SDValue(); // empty struct
2006 SmallVector<SDValue, 4> Constants(NumElts);
2007 for (unsigned i = 0; i != NumElts; ++i) {
2008 EVT EltVT = ValueVTs[i];
2009 if (isa<UndefValue>(Val: C))
2010 Constants[i] = DAG.getUNDEF(VT: EltVT);
2011 else if (EltVT.isFloatingPoint())
2012 Constants[i] = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2013 else
2014 Constants[i] = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2015 }
2016
2017 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
2018 }
2019
2020 if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C))
2021 return DAG.getBlockAddress(BA, VT);
2022
2023 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C))
2024 return getValue(V: Equiv->getGlobalValue());
2025
2026 if (const auto *NC = dyn_cast<NoCFIValue>(Val: C))
2027 return getValue(V: NC->getGlobalValue());
2028
2029 if (VT == MVT::aarch64svcount) {
2030 assert(C->isNullValue() && "Can only zero this target type!");
2031 return DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT,
2032 Operand: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::nxv16i1));
2033 }
2034
2035 if (VT.isRISCVVectorTuple()) {
2036 assert(C->isNullValue() && "Can only zero this target type!");
2037 return DAG.getNode(
2038 Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT,
2039 Operand: DAG.getNode(
2040 Opcode: ISD::SPLAT_VECTOR, DL: getCurSDLoc(),
2041 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8,
2042 NumElements: VT.getSizeInBits().getKnownMinValue() / 8, IsScalable: true),
2043 Operand: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::getIntegerVT(BitWidth: 8))));
2044 }
2045
2046 if (VT == MVT::externref || VT == MVT::funcref) {
2047 assert(C->isNullValue() && "Can only zero this target type!");
2048 // The zero value of a WebAssembly reference type is the null reference,
2049 // materialized with ref.null.
2050 Intrinsic::ID IID = VT == MVT::externref ? Intrinsic::wasm_ref_null_extern
2051 : Intrinsic::wasm_ref_null_func;
2052 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: getCurSDLoc(), VT,
2053 Operand: DAG.getTargetConstant(Val: IID, DL: getCurSDLoc(), VT: MVT::i32));
2054 }
2055
2056 VectorType *VecTy = cast<VectorType>(Val: V->getType());
2057
2058 // Now that we know the number and type of the elements, get that number of
2059 // elements into the Ops array based on what kind of constant it is.
2060 if (const ConstantVector *CV = dyn_cast<ConstantVector>(Val: C)) {
2061 SmallVector<SDValue, 16> Ops;
2062 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2063 for (unsigned i = 0; i != NumElements; ++i)
2064 Ops.push_back(Elt: getValue(V: CV->getOperand(i_nocapture: i)));
2065
2066 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
2067 }
2068
2069 if (isa<ConstantAggregateZero>(Val: C)) {
2070 EVT EltVT =
2071 TLI.getValueType(DL: DAG.getDataLayout(), Ty: VecTy->getElementType());
2072
2073 SDValue Op;
2074 if (EltVT.isFloatingPoint())
2075 Op = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2076 else
2077 Op = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2078
2079 return DAG.getSplat(VT, DL: getCurSDLoc(), Op);
2080 }
2081
2082 llvm_unreachable("Unknown vector constant");
2083 }
2084
2085 // If this is a static alloca, generate it as the frameindex instead of
2086 // computation.
2087 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
2088 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
2089 if (SI != FuncInfo.StaticAllocaMap.end())
2090 return DAG.getFrameIndex(
2091 FI: SI->second, VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: AI->getType()));
2092 }
2093
2094 // If this is an instruction which fast-isel has deferred, select it now.
2095 if (const Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
2096 Register InReg = FuncInfo.InitializeRegForValue(V: Inst);
2097 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
2098 Inst->getType(), std::nullopt);
2099 SDValue Chain = DAG.getEntryNode();
2100 return RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V);
2101 }
2102
2103 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(Val: V))
2104 return DAG.getMDNode(MD: cast<MDNode>(Val: MD->getMetadata()));
2105
2106 if (const auto *BB = dyn_cast<BasicBlock>(Val: V))
2107 return DAG.getBasicBlock(MBB: FuncInfo.getMBB(BB));
2108
2109 llvm_unreachable("Can't get register for value!");
2110}
2111
2112void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
2113 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2114 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
2115 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
2116 bool IsSEH = isAsynchronousEHPersonality(Pers);
2117 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
2118 if (IsSEH) {
2119 // For SEH, EHCont Guard needs to know that this catchpad is a target.
2120 CatchPadMBB->setIsEHContTarget(true);
2121 DAG.getMachineFunction().setHasEHContTarget(true);
2122 } else
2123 CatchPadMBB->setIsEHScopeEntry();
2124 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
2125 if (IsMSVCCXX || IsCoreCLR)
2126 CatchPadMBB->setIsEHFuncletEntry();
2127}
2128
2129void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
2130 // Update machine-CFG edge.
2131 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(BB: I.getSuccessor());
2132 FuncInfo.MBB->addSuccessor(Succ: TargetMBB);
2133
2134 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2135 bool IsSEH = isAsynchronousEHPersonality(Pers);
2136 if (IsSEH) {
2137 // If this is not a fall-through branch or optimizations are switched off,
2138 // emit the branch.
2139 if (TargetMBB != NextBlock(MBB: FuncInfo.MBB) ||
2140 TM.getOptLevel() == CodeGenOptLevel::None)
2141 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other,
2142 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: TargetMBB)));
2143 return;
2144 }
2145
2146 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2147 TargetMBB->setIsEHContTarget(true);
2148 DAG.getMachineFunction().setHasEHContTarget(true);
2149
2150 // Figure out the funclet membership for the catchret's successor.
2151 // This will be used by the FuncletLayout pass to determine how to order the
2152 // BB's.
2153 // A 'catchret' returns to the outer scope's color.
2154 Value *ParentPad = I.getCatchSwitchParentPad();
2155 const BasicBlock *SuccessorColor;
2156 if (isa<ConstantTokenNone>(Val: ParentPad))
2157 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2158 else
2159 SuccessorColor = cast<Instruction>(Val: ParentPad)->getParent();
2160 assert(SuccessorColor && "No parent funclet for catchret!");
2161 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(BB: SuccessorColor);
2162 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2163
2164 // Create the terminator node.
2165 SDValue Ret = DAG.getNode(Opcode: ISD::CATCHRET, DL: getCurSDLoc(), VT: MVT::Other,
2166 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: TargetMBB),
2167 N3: DAG.getBasicBlock(MBB: SuccessorColorMBB));
2168 DAG.setRoot(Ret);
2169}
2170
2171void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2172 // Don't emit any special code for the cleanuppad instruction. It just marks
2173 // the start of an EH scope/funclet.
2174 FuncInfo.MBB->setIsEHScopeEntry();
2175 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2176 if (Pers != EHPersonality::Wasm_CXX && Pers != EHPersonality::Wasm_D) {
2177 FuncInfo.MBB->setIsEHFuncletEntry();
2178 FuncInfo.MBB->setIsCleanupFuncletEntry();
2179 }
2180}
2181
2182/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2183/// many places it could ultimately go. In the IR, we have a single unwind
2184/// destination, but in the machine CFG, we enumerate all the possible blocks.
2185/// This function skips over imaginary basic blocks that hold catchswitch
2186/// instructions, and finds all the "real" machine
2187/// basic block destinations. As those destinations may not be successors of
2188/// EHPadBB, here we also calculate the edge probability to those destinations.
2189/// The passed-in Prob is the edge probability to EHPadBB.
2190static void findUnwindDestinations(
2191 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2192 BranchProbability Prob,
2193 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2194 &UnwindDests) {
2195 EHPersonality Personality =
2196 classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2197 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2198 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2199 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2200 bool IsWasmD = Personality == EHPersonality::Wasm_D;
2201 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2202
2203 while (EHPadBB) {
2204 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
2205 BasicBlock *NewEHPadBB = nullptr;
2206 if (isa<LandingPadInst>(Val: Pad)) {
2207 // Stop on landingpads. They are not funclets.
2208 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2209 break;
2210 } else if (isa<CleanupPadInst>(Val: Pad)) {
2211 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2212 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2213 // which always catches an exception.
2214 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2215 UnwindDests.back().first->setIsEHScopeEntry();
2216 // In Wasm, EH scopes are not funclets
2217 if (!IsWasmCXX && !IsWasmD)
2218 UnwindDests.back().first->setIsEHFuncletEntry();
2219 break;
2220 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
2221 // Add the catchpad handlers to the possible destinations.
2222 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2223 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: CatchPadBB), Args&: Prob);
2224 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2225 if (IsMSVCCXX || IsCoreCLR)
2226 UnwindDests.back().first->setIsEHFuncletEntry();
2227 if (!IsSEH)
2228 UnwindDests.back().first->setIsEHScopeEntry();
2229 }
2230 NewEHPadBB = CatchSwitch->getUnwindDest();
2231 } else {
2232 continue;
2233 }
2234
2235 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2236 if (BPI && NewEHPadBB)
2237 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
2238 EHPadBB = NewEHPadBB;
2239 }
2240}
2241
2242void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2243 // Update successor info.
2244 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2245 auto UnwindDest = I.getUnwindDest();
2246 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2247 BranchProbability UnwindDestProb =
2248 (BPI && UnwindDest)
2249 ? BPI->getEdgeProbability(Src: FuncInfo.MBB->getBasicBlock(), Dst: UnwindDest)
2250 : BranchProbability::getZero();
2251 findUnwindDestinations(FuncInfo, EHPadBB: UnwindDest, Prob: UnwindDestProb, UnwindDests);
2252 for (auto &UnwindDest : UnwindDests) {
2253 UnwindDest.first->setIsEHPad();
2254 addSuccessorWithProb(Src: FuncInfo.MBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
2255 }
2256 FuncInfo.MBB->normalizeSuccProbs();
2257
2258 // Create the terminator node.
2259 MachineBasicBlock *CleanupPadMBB =
2260 FuncInfo.getMBB(BB: I.getCleanupPad()->getParent());
2261 SDValue Ret = DAG.getNode(Opcode: ISD::CLEANUPRET, DL: getCurSDLoc(), VT: MVT::Other,
2262 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: CleanupPadMBB));
2263 DAG.setRoot(Ret);
2264}
2265
2266void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2267 report_fatal_error(reason: "visitCatchSwitch not yet implemented!");
2268}
2269
2270void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2271 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2272 auto &DL = DAG.getDataLayout();
2273 SDValue Chain = getControlRoot();
2274 SmallVector<ISD::OutputArg, 8> Outs;
2275 SmallVector<SDValue, 8> OutVals;
2276
2277 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2278 // lower
2279 //
2280 // %val = call <ty> @llvm.experimental.deoptimize()
2281 // ret <ty> %val
2282 //
2283 // differently.
2284 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2285 LowerDeoptimizingReturn();
2286 return;
2287 }
2288
2289 if (!FuncInfo.CanLowerReturn) {
2290 Register DemoteReg = FuncInfo.DemoteRegister;
2291
2292 // Emit a store of the return value through the virtual register.
2293 // Leave Outs empty so that LowerReturn won't try to load return
2294 // registers the usual way.
2295 MVT PtrValueVT = TLI.getPointerTy(DL, AS: DL.getAllocaAddrSpace());
2296 SDValue RetPtr =
2297 DAG.getCopyFromReg(Chain, dl: getCurSDLoc(), Reg: DemoteReg, VT: PtrValueVT);
2298 Type *RetTy = I.getOperand(i_nocapture: 0)->getType();
2299 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
2300 RetPtr =
2301 TLI.annotateStackObjectPointer(Ptr: RetPtr, DAG, DL: getCurSDLoc(), Alignment: BaseAlign);
2302 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2303
2304 SmallVector<EVT, 4> ValueVTs, MemVTs;
2305 SmallVector<uint64_t, 4> Offsets;
2306 ComputeValueVTs(TLI, DL, Ty: RetTy, ValueVTs, MemVTs: &MemVTs, FixedOffsets: &Offsets, StartingOffset: 0);
2307 unsigned NumValues = ValueVTs.size();
2308
2309 SmallVector<SDValue, 4> Chains(NumValues);
2310 for (unsigned i = 0; i != NumValues; ++i) {
2311 // An aggregate return value cannot wrap around the address space, so
2312 // offsets to its parts don't wrap either.
2313 SDValue Ptr = DAG.getObjectPtrOffset(SL: getCurSDLoc(), Ptr: RetPtr,
2314 Offset: TypeSize::getFixed(ExactSize: Offsets[i]));
2315
2316 SDValue Val = RetOp.getValue(R: RetOp.getResNo() + i);
2317 if (MemVTs[i] != ValueVTs[i])
2318 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: getCurSDLoc(), VT: MemVTs[i]);
2319 Chains[i] = DAG.getStore(
2320 Chain, dl: getCurSDLoc(), Val,
2321 // FIXME: better loc info would be nice.
2322 Ptr, PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
2323 Alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]));
2324 }
2325
2326 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: getCurSDLoc(),
2327 VT: MVT::Other, Ops: Chains);
2328 } else if (I.getNumOperands() != 0) {
2329 SmallVector<Type *, 4> Types;
2330 ComputeValueTypes(DL, Ty: I.getOperand(i_nocapture: 0)->getType(), Types);
2331 unsigned NumValues = Types.size();
2332 if (NumValues) {
2333 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2334
2335 const Function *F = I.getParent()->getParent();
2336
2337 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2338 Ty: I.getOperand(i_nocapture: 0)->getType(), CallConv: F->getCallingConv(),
2339 /*IsVarArg*/ isVarArg: false, DL);
2340
2341 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2342 if (F->getAttributes().hasRetAttr(Kind: Attribute::SExt))
2343 ExtendKind = ISD::SIGN_EXTEND;
2344 else if (F->getAttributes().hasRetAttr(Kind: Attribute::ZExt))
2345 ExtendKind = ISD::ZERO_EXTEND;
2346
2347 LLVMContext &Context = F->getContext();
2348 bool RetInReg = F->getAttributes().hasRetAttr(Kind: Attribute::InReg);
2349
2350 for (unsigned j = 0; j != NumValues; ++j) {
2351 EVT VT = TLI.getValueType(DL, Ty: Types[j]);
2352
2353 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2354 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2355
2356 CallingConv::ID CC = F->getCallingConv();
2357
2358 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2359 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2360 SmallVector<SDValue, 4> Parts(NumParts);
2361 getCopyToParts(DAG, DL: getCurSDLoc(),
2362 Val: SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2363 Parts: &Parts[0], NumParts, PartVT, V: &I, CallConv: CC, ExtendKind);
2364
2365 // 'inreg' on function refers to return value
2366 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2367 if (RetInReg)
2368 Flags.setInReg();
2369
2370 if (I.getOperand(i_nocapture: 0)->getType()->isPointerTy()) {
2371 Flags.setPointer();
2372 Flags.setPointerAddrSpace(
2373 cast<PointerType>(Val: I.getOperand(i_nocapture: 0)->getType())->getAddressSpace());
2374 }
2375
2376 if (NeedsRegBlock) {
2377 Flags.setInConsecutiveRegs();
2378 if (j == NumValues - 1)
2379 Flags.setInConsecutiveRegsLast();
2380 }
2381
2382 // Propagate extension type if any
2383 if (ExtendKind == ISD::SIGN_EXTEND)
2384 Flags.setSExt();
2385 else if (ExtendKind == ISD::ZERO_EXTEND)
2386 Flags.setZExt();
2387 else if (F->getAttributes().hasRetAttr(Kind: Attribute::NoExt))
2388 Flags.setNoExt();
2389
2390 for (unsigned i = 0; i < NumParts; ++i) {
2391 Outs.push_back(Elt: ISD::OutputArg(Flags,
2392 Parts[i].getValueType().getSimpleVT(),
2393 VT, Types[j], 0, 0));
2394 OutVals.push_back(Elt: Parts[i]);
2395 }
2396 }
2397 }
2398 }
2399
2400 // Push in swifterror virtual register as the last element of Outs. This makes
2401 // sure swifterror virtual register will be returned in the swifterror
2402 // physical register.
2403 const Function *F = I.getParent()->getParent();
2404 if (TLI.supportSwiftError() &&
2405 F->getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError)) {
2406 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2407 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2408 Flags.setSwiftError();
2409 Outs.push_back(Elt: ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
2410 /*argvt=*/EVT(TLI.getPointerTy(DL)),
2411 PointerType::getUnqual(C&: *DAG.getContext()),
2412 /*origidx=*/1, /*partOffs=*/0));
2413 // Create SDNode for the swifterror virtual register.
2414 OutVals.push_back(
2415 Elt: DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(
2416 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2417 VT: EVT(TLI.getPointerTy(DL))));
2418 }
2419
2420 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2421 CallingConv::ID CallConv =
2422 DAG.getMachineFunction().getFunction().getCallingConv();
2423 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2424 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2425
2426 // Verify that the target's LowerReturn behaved as expected.
2427 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2428 "LowerReturn didn't return a valid chain!");
2429
2430 // Update the DAG with the new chain value resulting from return lowering.
2431 DAG.setRoot(Chain);
2432}
2433
2434/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2435/// created for it, emit nodes to copy the value into the virtual
2436/// registers.
2437void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2438 // Skip empty types
2439 if (V->getType()->isEmptyTy())
2440 return;
2441
2442 auto VMI = FuncInfo.ValueMap.find(Val: V);
2443 if (VMI != FuncInfo.ValueMap.end()) {
2444 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2445 "Unused value assigned virtual registers!");
2446 CopyValueToVirtualRegister(V, Reg: VMI->second);
2447 }
2448}
2449
2450/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2451/// the current basic block, add it to ValueMap now so that we'll get a
2452/// CopyTo/FromReg.
2453void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2454 // No need to export constants.
2455 if (!isa<Instruction>(Val: V) && !isa<Argument>(Val: V)) return;
2456
2457 // Already exported?
2458 if (FuncInfo.isExportedInst(V)) return;
2459
2460 Register Reg = FuncInfo.InitializeRegForValue(V);
2461 CopyValueToVirtualRegister(V, Reg);
2462}
2463
2464bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2465 const BasicBlock *FromBB) {
2466 // The operands of the setcc have to be in this block. We don't know
2467 // how to export them from some other block.
2468 if (const Instruction *VI = dyn_cast<Instruction>(Val: V)) {
2469 // Can export from current BB.
2470 if (VI->getParent() == FromBB)
2471 return true;
2472
2473 // Is already exported, noop.
2474 return FuncInfo.isExportedInst(V);
2475 }
2476
2477 // If this is an argument, we can export it if the BB is the entry block or
2478 // if it is already exported.
2479 if (isa<Argument>(Val: V)) {
2480 if (FromBB->isEntryBlock())
2481 return true;
2482
2483 // Otherwise, can only export this if it is already exported.
2484 return FuncInfo.isExportedInst(V);
2485 }
2486
2487 // Otherwise, constants can always be exported.
2488 return true;
2489}
2490
2491/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2492BranchProbability
2493SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2494 const MachineBasicBlock *Dst) const {
2495 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2496 const BasicBlock *SrcBB = Src->getBasicBlock();
2497 const BasicBlock *DstBB = Dst->getBasicBlock();
2498 if (!BPI) {
2499 // If BPI is not available, set the default probability as 1 / N, where N is
2500 // the number of successors.
2501 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
2502 return BranchProbability(1, SuccSize);
2503 }
2504 return BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
2505}
2506
2507void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2508 MachineBasicBlock *Dst,
2509 BranchProbability Prob) {
2510 if (!FuncInfo.BPI)
2511 Src->addSuccessorWithoutProb(Succ: Dst);
2512 else {
2513 if (Prob.isUnknown())
2514 Prob = getEdgeProbability(Src, Dst);
2515 Src->addSuccessor(Succ: Dst, Prob);
2516 }
2517}
2518
2519static bool InBlock(const Value *V, const BasicBlock *BB) {
2520 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
2521 return I->getParent() == BB;
2522 return true;
2523}
2524
2525/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2526/// This function emits a branch and is used at the leaves of an OR or an
2527/// AND operator tree.
2528void
2529SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2530 MachineBasicBlock *TBB,
2531 MachineBasicBlock *FBB,
2532 MachineBasicBlock *CurBB,
2533 MachineBasicBlock *SwitchBB,
2534 BranchProbability TProb,
2535 BranchProbability FProb,
2536 bool InvertCond) {
2537 const BasicBlock *BB = CurBB->getBasicBlock();
2538
2539 // If the leaf of the tree is a comparison, merge the condition into
2540 // the caseblock.
2541 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
2542 // The operands of the cmp have to be in this block. We don't know
2543 // how to export them from some other block. If this is the first block
2544 // of the sequence, no exporting is needed.
2545 if (CurBB == SwitchBB ||
2546 (isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 0), FromBB: BB) &&
2547 isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 1), FromBB: BB))) {
2548 ISD::CondCode Condition;
2549 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
2550 ICmpInst::Predicate Pred =
2551 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2552 Condition = getICmpCondCode(Pred);
2553 } else {
2554 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
2555 FCmpInst::Predicate Pred =
2556 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2557 Condition = getFCmpCondCode(Pred);
2558 if (FC->hasNoNaNs() ||
2559 (isKnownNeverNaN(V: FC->getOperand(i_nocapture: 0),
2560 SQ: SimplifyQuery(DAG.getDataLayout(), FC)) &&
2561 isKnownNeverNaN(V: FC->getOperand(i_nocapture: 1),
2562 SQ: SimplifyQuery(DAG.getDataLayout(), FC))))
2563 Condition = getFCmpCodeWithoutNaN(CC: Condition);
2564 }
2565
2566 CaseBlock CB(Condition, BOp->getOperand(i_nocapture: 0), BOp->getOperand(i_nocapture: 1), nullptr,
2567 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2568 SL->SwitchCases.push_back(x: CB);
2569 return;
2570 }
2571 }
2572
2573 // Create a CaseBlock record representing this branch.
2574 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2575 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(Context&: *DAG.getContext()),
2576 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2577 SL->SwitchCases.push_back(x: CB);
2578}
2579
2580// Collect dependencies on V recursively. This is used for the cost analysis in
2581// `shouldKeepJumpConditionsTogether`.
2582static bool collectInstructionDeps(
2583 SmallMapVector<const Instruction *, bool, 8> *Deps, const Value *V,
2584 SmallMapVector<const Instruction *, bool, 8> *Necessary = nullptr,
2585 unsigned Depth = 0) {
2586 // Return false if we have an incomplete count.
2587 if (Depth >= SelectionDAG::MaxRecursionDepth)
2588 return false;
2589
2590 auto *I = dyn_cast<Instruction>(Val: V);
2591 if (I == nullptr)
2592 return true;
2593
2594 if (Necessary != nullptr) {
2595 // This instruction is necessary for the other side of the condition so
2596 // don't count it.
2597 if (Necessary->contains(Key: I))
2598 return true;
2599 }
2600
2601 // Already added this dep.
2602 if (!Deps->try_emplace(Key: I, Args: false).second)
2603 return true;
2604
2605 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2606 if (!collectInstructionDeps(Deps, V: I->getOperand(i: OpIdx), Necessary,
2607 Depth: Depth + 1))
2608 return false;
2609 return true;
2610}
2611
2612bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether(
2613 const FunctionLoweringInfo &FuncInfo, const CondBrInst &I,
2614 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2615 TargetLoweringBase::CondMergingParams Params) const {
2616 if (Params.BaseCost < 0)
2617 return false;
2618
2619 // Baseline cost.
2620 InstructionCost CostThresh = Params.BaseCost;
2621
2622 BranchProbabilityInfo *BPI = nullptr;
2623 if (Params.LikelyBias || Params.UnlikelyBias)
2624 BPI = FuncInfo.BPI;
2625 if (BPI != nullptr) {
2626 // See if we are either likely to get an early out or compute both lhs/rhs
2627 // of the condition.
2628 BasicBlock *IfFalse = I.getSuccessor(i: 0);
2629 BasicBlock *IfTrue = I.getSuccessor(i: 1);
2630
2631 std::optional<bool> Likely;
2632 if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfTrue))
2633 Likely = true;
2634 else if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfFalse))
2635 Likely = false;
2636
2637 if (Likely) {
2638 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2639 // Its likely we will have to compute both lhs and rhs of condition
2640 CostThresh += Params.LikelyBias;
2641 else {
2642 if (Params.UnlikelyBias < 0)
2643 return false;
2644 // Its likely we will get an early out.
2645 CostThresh -= Params.UnlikelyBias;
2646 }
2647 }
2648 }
2649
2650 if (CostThresh <= 0)
2651 return false;
2652
2653 // Collect "all" instructions that lhs condition is dependent on.
2654 // Use map for stable iteration (to avoid non-determanism of iteration of
2655 // SmallPtrSet). The `bool` value is just a dummy.
2656 SmallMapVector<const Instruction *, bool, 8> LhsDeps, RhsDeps;
2657 collectInstructionDeps(Deps: &LhsDeps, V: Lhs);
2658 // Collect "all" instructions that rhs condition is dependent on AND are
2659 // dependencies of lhs. This gives us an estimate on which instructions we
2660 // stand to save by splitting the condition.
2661 if (!collectInstructionDeps(Deps: &RhsDeps, V: Rhs, Necessary: &LhsDeps))
2662 return false;
2663 // Add the compare instruction itself unless its a dependency on the LHS.
2664 if (const auto *RhsI = dyn_cast<Instruction>(Val: Rhs))
2665 if (!LhsDeps.contains(Key: RhsI))
2666 RhsDeps.try_emplace(Key: RhsI, Args: false);
2667
2668 InstructionCost CostOfIncluding = 0;
2669 // See if this instruction will need to computed independently of whether RHS
2670 // is.
2671 Value *BrCond = I.getCondition();
2672 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2673 for (const auto *U : Ins->users()) {
2674 // If user is independent of RHS calculation we don't need to count it.
2675 if (auto *UIns = dyn_cast<Instruction>(Val: U))
2676 if (UIns != BrCond && !RhsDeps.contains(Key: UIns))
2677 return false;
2678 }
2679 return true;
2680 };
2681
2682 // Prune instructions from RHS Deps that are dependencies of unrelated
2683 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2684 // arbitrary and just meant to cap the how much time we spend in the pruning
2685 // loop. Its highly unlikely to come into affect.
2686 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2687 // Stop after a certain point. No incorrectness from including too many
2688 // instructions.
2689 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2690 const Instruction *ToDrop = nullptr;
2691 for (const auto &InsPair : RhsDeps) {
2692 if (!ShouldCountInsn(InsPair.first)) {
2693 ToDrop = InsPair.first;
2694 break;
2695 }
2696 }
2697 if (ToDrop == nullptr)
2698 break;
2699 RhsDeps.erase(Key: ToDrop);
2700 }
2701
2702 for (const auto &InsPair : RhsDeps) {
2703 // Finally accumulate latency that we can only attribute to computing the
2704 // RHS condition. Use latency because we are essentially trying to calculate
2705 // the cost of the dependency chain.
2706 // Possible TODO: We could try to estimate ILP and make this more precise.
2707 CostOfIncluding += TTI->getInstructionCost(
2708 U: InsPair.first, CostKind: TargetTransformInfo::TCK_Latency);
2709
2710 if (CostOfIncluding > CostThresh)
2711 return false;
2712 }
2713 return true;
2714}
2715
2716void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2717 MachineBasicBlock *TBB,
2718 MachineBasicBlock *FBB,
2719 MachineBasicBlock *CurBB,
2720 MachineBasicBlock *SwitchBB,
2721 Instruction::BinaryOps Opc,
2722 BranchProbability TProb,
2723 BranchProbability FProb,
2724 bool InvertCond) {
2725 // Skip over not part of the tree and remember to invert op and operands at
2726 // next level.
2727 Value *NotCond;
2728 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
2729 InBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
2730 FindMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2731 InvertCond: !InvertCond);
2732 return;
2733 }
2734
2735 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
2736 const Value *BOpOp0, *BOpOp1;
2737 // Compute the effective opcode for Cond, taking into account whether it needs
2738 // to be inverted, e.g.
2739 // and (not (or A, B)), C
2740 // gets lowered as
2741 // and (and (not A, not B), C)
2742 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2743 if (BOp) {
2744 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2745 ? Instruction::And
2746 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2747 ? Instruction::Or
2748 : (Instruction::BinaryOps)0);
2749 if (InvertCond) {
2750 if (BOpc == Instruction::And)
2751 BOpc = Instruction::Or;
2752 else if (BOpc == Instruction::Or)
2753 BOpc = Instruction::And;
2754 }
2755 }
2756
2757 // If this node is not part of the or/and tree, emit it as a branch.
2758 // Note that all nodes in the tree should have same opcode.
2759 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2760 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2761 !InBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
2762 !InBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
2763 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2764 TProb, FProb, InvertCond);
2765 return;
2766 }
2767
2768 // Create TmpBB after CurBB.
2769 MachineFunction::iterator BBI(CurBB);
2770 MachineFunction &MF = DAG.getMachineFunction();
2771 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
2772 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
2773
2774 if (Opc == Instruction::Or) {
2775 // Codegen X | Y as:
2776 // BB1:
2777 // jmp_if_X TBB
2778 // jmp TmpBB
2779 // TmpBB:
2780 // jmp_if_Y TBB
2781 // jmp FBB
2782 //
2783
2784 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2785 // The requirement is that
2786 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2787 // = TrueProb for original BB.
2788 // Assuming the original probabilities are A and B, one choice is to set
2789 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2790 // A/(1+B) and 2B/(1+B). This choice assumes that
2791 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2792 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2793 // TmpBB, but the math is more complicated.
2794
2795 auto NewTrueProb = TProb / 2;
2796 auto NewFalseProb = TProb / 2 + FProb;
2797 // Emit the LHS condition.
2798 FindMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2799 FProb: NewFalseProb, InvertCond);
2800
2801 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2802 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2803 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2804 // Emit the RHS condition into TmpBB.
2805 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2806 FProb: Probs[1], InvertCond);
2807 } else {
2808 assert(Opc == Instruction::And && "Unknown merge op!");
2809 // Codegen X & Y as:
2810 // BB1:
2811 // jmp_if_X TmpBB
2812 // jmp FBB
2813 // TmpBB:
2814 // jmp_if_Y TBB
2815 // jmp FBB
2816 //
2817 // This requires creation of TmpBB after CurBB.
2818
2819 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2820 // The requirement is that
2821 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2822 // = FalseProb for original BB.
2823 // Assuming the original probabilities are A and B, one choice is to set
2824 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2825 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2826 // TrueProb for BB1 * FalseProb for TmpBB.
2827
2828 auto NewTrueProb = TProb + FProb / 2;
2829 auto NewFalseProb = FProb / 2;
2830 // Emit the LHS condition.
2831 FindMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2832 FProb: NewFalseProb, InvertCond);
2833
2834 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2835 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2836 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2837 // Emit the RHS condition into TmpBB.
2838 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2839 FProb: Probs[1], InvertCond);
2840 }
2841}
2842
2843/// If the set of cases should be emitted as a series of branches, return true.
2844/// If we should emit this as a bunch of and/or'd together conditions, return
2845/// false.
2846bool
2847SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2848 if (Cases.size() != 2) return true;
2849
2850 // If this is two comparisons of the same values or'd or and'd together, they
2851 // will get folded into a single comparison, so don't emit two blocks.
2852 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2853 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2854 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2855 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2856 return false;
2857 }
2858
2859 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2860 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2861 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2862 Cases[0].CC == Cases[1].CC &&
2863 isa<Constant>(Val: Cases[0].CmpRHS) &&
2864 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
2865 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2866 return false;
2867 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2868 return false;
2869 }
2870
2871 return true;
2872}
2873
2874void SelectionDAGBuilder::visitUncondBr(const UncondBrInst &I) {
2875 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2876
2877 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2878
2879 // Update machine-CFG edges.
2880 BrMBB->addSuccessor(Succ: Succ0MBB);
2881
2882 // If this is not a fall-through branch or optimizations are switched off,
2883 // emit the branch.
2884 if (Succ0MBB != NextBlock(MBB: BrMBB) ||
2885 TM.getOptLevel() == CodeGenOptLevel::None) {
2886 auto Br = DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other, N1: getControlRoot(),
2887 N2: DAG.getBasicBlock(MBB: Succ0MBB));
2888 setValue(V: &I, NewN: Br);
2889 DAG.setRoot(Br);
2890 }
2891}
2892
2893void SelectionDAGBuilder::visitCondBr(const CondBrInst &I) {
2894 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2895
2896 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2897
2898 // If this condition is one of the special cases we handle, do special stuff
2899 // now.
2900 const Value *CondVal = I.getCondition();
2901 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 1));
2902
2903 // If this is a series of conditions that are or'd or and'd together, emit
2904 // this as a sequence of branches instead of setcc's with and/or operations.
2905 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2906 // unpredictable branches, and vector extracts because those jumps are likely
2907 // expensive for any target), this should improve performance.
2908 // For example, instead of something like:
2909 // cmp A, B
2910 // C = seteq
2911 // cmp D, E
2912 // F = setle
2913 // or C, F
2914 // jnz foo
2915 // Emit:
2916 // cmp A, B
2917 // je foo
2918 // cmp D, E
2919 // jle foo
2920 bool IsUnpredictable = I.hasMetadata(KindID: LLVMContext::MD_unpredictable);
2921 const Instruction *BOp = dyn_cast<Instruction>(Val: CondVal);
2922 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2923 BOp->hasOneUse() && !IsUnpredictable) {
2924 Value *Vec;
2925 const Value *BOp0, *BOp1;
2926 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2927 if (match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2928 Opcode = Instruction::And;
2929 else if (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2930 Opcode = Instruction::Or;
2931
2932 if (Opcode &&
2933 !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
2934 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value()))) &&
2935 !shouldKeepJumpConditionsTogether(
2936 FuncInfo, I, Opc: Opcode, Lhs: BOp0, Rhs: BOp1,
2937 Params: DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2938 Opcode, BOp0, BOp1, FuncInfo.Fn))) {
2939 FindMergedConditions(Cond: BOp, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: BrMBB, SwitchBB: BrMBB, Opc: Opcode,
2940 TProb: getEdgeProbability(Src: BrMBB, Dst: Succ0MBB),
2941 FProb: getEdgeProbability(Src: BrMBB, Dst: Succ1MBB),
2942 /*InvertCond=*/false);
2943 // If the compares in later blocks need to use values not currently
2944 // exported from this block, export them now. This block should always
2945 // be the first entry.
2946 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2947
2948 // Allow some cases to be rejected.
2949 if (ShouldEmitAsBranches(Cases: SL->SwitchCases)) {
2950 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2951 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpLHS);
2952 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpRHS);
2953 }
2954
2955 // Emit the branch for this block.
2956 visitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: BrMBB);
2957 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
2958 return;
2959 }
2960
2961 // Okay, we decided not to do this, remove any inserted MBB's and clear
2962 // SwitchCases.
2963 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2964 FuncInfo.MF->erase(MBBI: SL->SwitchCases[i].ThisBB);
2965
2966 SL->SwitchCases.clear();
2967 }
2968 }
2969
2970 // Create a CaseBlock record representing this branch.
2971 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(Context&: *DAG.getContext()),
2972 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2973 BranchProbability::getUnknown(), BranchProbability::getUnknown(),
2974 IsUnpredictable);
2975
2976 // Use visitSwitchCase to actually insert the fast branch sequence for this
2977 // cond branch.
2978 visitSwitchCase(CB, SwitchBB: BrMBB);
2979}
2980
2981/// visitSwitchCase - Emits the necessary code to represent a single node in
2982/// the binary search tree resulting from lowering a switch instruction.
2983void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2984 MachineBasicBlock *SwitchBB) {
2985 SDValue Cond;
2986 SDValue CondLHS = getValue(V: CB.CmpLHS);
2987 SDLoc dl = CB.DL;
2988
2989 if (CB.CC == ISD::SETTRUE) {
2990 // Branch or fall through to TrueBB.
2991 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
2992 SwitchBB->normalizeSuccProbs();
2993 if (CB.TrueBB != NextBlock(MBB: SwitchBB)) {
2994 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: getControlRoot(),
2995 N2: DAG.getBasicBlock(MBB: CB.TrueBB)));
2996 }
2997 return;
2998 }
2999
3000 auto &TLI = DAG.getTargetLoweringInfo();
3001 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: CB.CmpLHS->getType());
3002
3003 // Build the setcc now.
3004 if (!CB.CmpMHS) {
3005 // Fold "(X == true)" to X and "(X == false)" to !X to
3006 // handle common cases produced by branch lowering.
3007 if (CB.CmpRHS == ConstantInt::getTrue(Context&: *DAG.getContext()) &&
3008 CB.CC == ISD::SETEQ)
3009 Cond = CondLHS;
3010 else if (CB.CmpRHS == ConstantInt::getFalse(Context&: *DAG.getContext()) &&
3011 CB.CC == ISD::SETEQ) {
3012 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: CondLHS.getValueType());
3013 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CondLHS.getValueType(), N1: CondLHS, N2: True);
3014 } else {
3015 SDValue CondRHS = getValue(V: CB.CmpRHS);
3016
3017 // If a pointer's DAG type is larger than its memory type then the DAG
3018 // values are zero-extended. This breaks signed comparisons so truncate
3019 // back to the underlying type before doing the compare.
3020 if (CondLHS.getValueType() != MemVT) {
3021 CondLHS = DAG.getPtrExtOrTrunc(Op: CondLHS, DL: getCurSDLoc(), VT: MemVT);
3022 CondRHS = DAG.getPtrExtOrTrunc(Op: CondRHS, DL: getCurSDLoc(), VT: MemVT);
3023 }
3024 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: CondLHS, RHS: CondRHS, Cond: CB.CC);
3025 }
3026 } else {
3027 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
3028
3029 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
3030 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
3031
3032 SDValue CmpOp = getValue(V: CB.CmpMHS);
3033 EVT VT = CmpOp.getValueType();
3034
3035 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
3036 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: CmpOp, RHS: DAG.getConstant(Val: High, DL: dl, VT),
3037 Cond: ISD::SETLE);
3038 } else {
3039 SDValue SUB = DAG.getNode(Opcode: ISD::SUB, DL: dl,
3040 VT, N1: CmpOp, N2: DAG.getConstant(Val: Low, DL: dl, VT));
3041 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: SUB,
3042 RHS: DAG.getConstant(Val: High-Low, DL: dl, VT), Cond: ISD::SETULE);
3043 }
3044 }
3045
3046 // Update successor info
3047 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
3048 // TrueBB and FalseBB are always different unless the incoming IR is
3049 // degenerate. This only happens when running llc on weird IR.
3050 if (CB.TrueBB != CB.FalseBB)
3051 addSuccessorWithProb(Src: SwitchBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
3052 SwitchBB->normalizeSuccProbs();
3053
3054 // If the lhs block is the next block, invert the condition so that we can
3055 // fall through to the lhs instead of the rhs block.
3056 if (CB.TrueBB == NextBlock(MBB: SwitchBB)) {
3057 std::swap(a&: CB.TrueBB, b&: CB.FalseBB);
3058 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: Cond.getValueType());
3059 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: Cond.getValueType(), N1: Cond, N2: True);
3060 }
3061
3062 SDNodeFlags Flags;
3063 Flags.setUnpredictable(CB.IsUnpredictable);
3064 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: getControlRoot(),
3065 N2: Cond, N3: DAG.getBasicBlock(MBB: CB.TrueBB), Flags);
3066
3067 setValue(V: CurInst, NewN: BrCond);
3068
3069 // Insert the false branch. Do this even if it's a fall through branch,
3070 // this makes it easier to do DAG optimizations which require inverting
3071 // the branch condition.
3072 BrCond = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrCond,
3073 N2: DAG.getBasicBlock(MBB: CB.FalseBB));
3074
3075 DAG.setRoot(BrCond);
3076}
3077
3078/// visitJumpTable - Emit JumpTable node in the current MBB
3079void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
3080 // Emit the code for the jump table
3081 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3082 assert(JT.Reg && "Should lower JT Header first!");
3083 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DL: DAG.getDataLayout());
3084 SDValue Index = DAG.getCopyFromReg(Chain: getControlRoot(), dl: *JT.SL, Reg: JT.Reg, VT: PTy);
3085 SDValue Table = DAG.getJumpTable(JTI: JT.JTI, VT: PTy);
3086 SDValue BrJumpTable = DAG.getNode(Opcode: ISD::BR_JT, DL: *JT.SL, VT: MVT::Other,
3087 N1: Index.getValue(R: 1), N2: Table, N3: Index);
3088 DAG.setRoot(BrJumpTable);
3089}
3090
3091/// visitJumpTableHeader - This function emits necessary code to produce index
3092/// in the JumpTable from switch case.
3093void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
3094 JumpTableHeader &JTH,
3095 MachineBasicBlock *SwitchBB) {
3096 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3097 const SDLoc &dl = *JT.SL;
3098
3099 // Subtract the lowest switch case value from the value being switched on.
3100 SDValue SwitchOp = getValue(V: JTH.SValue);
3101 EVT VT = SwitchOp.getValueType();
3102 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp,
3103 N2: DAG.getConstant(Val: JTH.First, DL: dl, VT));
3104
3105 // The SDNode we just created, which holds the value being switched on minus
3106 // the smallest case value, needs to be copied to a virtual register so it
3107 // can be used as an index into the jump table in a subsequent basic block.
3108 // This value may be smaller or larger than the target's pointer type, and
3109 // therefore require extension or truncating.
3110 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3111 SwitchOp =
3112 DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
3113
3114 Register JumpTableReg =
3115 FuncInfo.CreateReg(VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
3116 SDValue CopyTo =
3117 DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: JumpTableReg, N: SwitchOp);
3118 JT.Reg = JumpTableReg;
3119
3120 if (!JTH.FallthroughUnreachable) {
3121 // Emit the range check for the jump table, and branch to the default block
3122 // for the switch statement if the value being switched on exceeds the
3123 // largest case in the switch.
3124 SDValue CMP = DAG.getSetCC(
3125 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3126 VT: Sub.getValueType()),
3127 LHS: Sub, RHS: DAG.getConstant(Val: JTH.Last - JTH.First, DL: dl, VT), Cond: ISD::SETUGT);
3128
3129 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl,
3130 VT: MVT::Other, N1: CopyTo, N2: CMP,
3131 N3: DAG.getBasicBlock(MBB: JT.Default));
3132
3133 // Avoid emitting unnecessary branches to the next block.
3134 if (JT.MBB != NextBlock(MBB: SwitchBB))
3135 BrCond = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrCond,
3136 N2: DAG.getBasicBlock(MBB: JT.MBB));
3137
3138 DAG.setRoot(BrCond);
3139 } else {
3140 // Avoid emitting unnecessary branches to the next block.
3141 if (JT.MBB != NextBlock(MBB: SwitchBB))
3142 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: CopyTo,
3143 N2: DAG.getBasicBlock(MBB: JT.MBB)));
3144 else
3145 DAG.setRoot(CopyTo);
3146 }
3147}
3148
3149/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3150/// variable if there exists one.
3151static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
3152 SDValue &Chain) {
3153 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3154 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
3155 EVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout());
3156 MachineFunction &MF = DAG.getMachineFunction();
3157 Value *Global =
3158 TLI.getSDagStackGuard(M: *MF.getFunction().getParent(), Libcalls: DAG.getLibcalls());
3159 MachineSDNode *Node =
3160 DAG.getMachineNode(Opcode: TargetOpcode::LOAD_STACK_GUARD, dl: DL, VT: PtrTy, Op1: Chain);
3161 if (Global) {
3162 MachinePointerInfo MPInfo(Global);
3163 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
3164 MachineMemOperand::MODereferenceable;
3165 MachineMemOperand *MemRef = MF.getMachineMemOperand(
3166 PtrInfo: MPInfo, F: Flags, Size: PtrTy.getSizeInBits() / 8, BaseAlignment: DAG.getEVTAlign(MemoryVT: PtrTy));
3167 DAG.setNodeMemRefs(N: Node, NewMemRefs: {MemRef});
3168 }
3169 if (PtrTy != PtrMemTy)
3170 return DAG.getPtrExtOrTrunc(Op: SDValue(Node, 0), DL, VT: PtrMemTy);
3171 return SDValue(Node, 0);
3172}
3173
3174/// Codegen a new tail for a stack protector check ParentMBB which has had its
3175/// tail spliced into a stack protector check success bb.
3176///
3177/// For a high level explanation of how this fits into the stack protector
3178/// generation see the comment on the declaration of class
3179/// StackProtectorDescriptor.
3180void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
3181 MachineBasicBlock *ParentBB) {
3182
3183 // First create the loads to the guard/stack slot for the comparison.
3184 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3185 auto &DL = DAG.getDataLayout();
3186 EVT PtrTy = TLI.getFrameIndexTy(DL);
3187 EVT PtrMemTy = TLI.getPointerMemTy(DL, AS: DL.getAllocaAddrSpace());
3188
3189 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3190 int FI = MFI.getStackProtectorIndex();
3191
3192 SDValue Guard;
3193 SDLoc dl = getCurSDLoc();
3194 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3195 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3196 Align Align = DL.getPrefTypeAlign(
3197 Ty: PointerType::get(C&: M.getContext(), AddressSpace: DL.getAllocaAddrSpace()));
3198
3199 // Generate code to load the content of the guard slot.
3200 SDValue GuardVal = DAG.getLoad(
3201 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3202 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3203 MMOFlags: MachineMemOperand::MOVolatile);
3204
3205 // If cookie mixing is enabled, unmix the stored GuardVal to get back the
3206 // original cookie for comparison. The prologue stored (FP - Cookie) or
3207 // (FP XOR Cookie), so we apply the same operation again to unmix:
3208 // FP - (FP - Cookie) = Cookie, or (FP XOR Cookie) XOR FP = Cookie.
3209 if (TLI.useStackGuardMixFP())
3210 GuardVal = TLI.emitStackGuardMixFP(DAG, Val: GuardVal, DL: dl);
3211
3212 // If we're using function-based instrumentation, call the guard check
3213 // function
3214 if (SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3215 // Get the guard check function from the target and verify it exists since
3216 // we're using function-based instrumentation
3217 const Function *GuardCheckFn =
3218 TLI.getSSPStackGuardCheck(M, Libcalls: DAG.getLibcalls());
3219 assert(GuardCheckFn && "Guard check function is null");
3220
3221 // The target provides a guard check function to validate the guard value.
3222 // Generate a call to that function with the content of the guard slot as
3223 // argument.
3224 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3225 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3226
3227 TargetLowering::ArgListTy Args;
3228 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(i: 0));
3229 if (GuardCheckFn->hasParamAttribute(ArgNo: 0, Kind: Attribute::AttrKind::InReg))
3230 Entry.IsInReg = true;
3231 Args.push_back(x: Entry);
3232
3233 TargetLowering::CallLoweringInfo CLI(DAG);
3234 CLI.setDebugLoc(getCurSDLoc())
3235 .setChain(DAG.getEntryNode())
3236 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3237 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3238
3239 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3240 DAG.setRoot(Result.second);
3241 return;
3242 }
3243
3244 // Load the fresh guard value for comparison.
3245 // For targets that mix the cookie in LOAD_STACK_GUARD expansion, we need to
3246 // load directly without using LOAD_STACK_GUARD to avoid unwanted mixing.
3247 SDValue Chain = DAG.getEntryNode();
3248 if (TLI.useStackGuardMixFP()) {
3249 // Mixing targets: load cookie directly to avoid mixing in LOAD_STACK_GUARD
3250 if (const Value *IRGuard = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls())) {
3251 SDValue GuardPtr = getValue(V: IRGuard);
3252 Guard = DAG.getLoad(VT: PtrMemTy, dl, Chain, Ptr: GuardPtr,
3253 PtrInfo: MachinePointerInfo(IRGuard, 0), Alignment: Align,
3254 MMOFlags: MachineMemOperand::MOVolatile);
3255 } else {
3256 LLVMContext &Ctx = *DAG.getContext();
3257 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
3258 Guard = DAG.getPOISON(VT: PtrMemTy);
3259 }
3260 } else {
3261 // Non-mixing targets: use LOAD_STACK_GUARD or direct load as usual
3262 if (TLI.useLoadStackGuardNode(M)) {
3263 Guard = getLoadStackGuard(DAG, DL: dl, Chain);
3264 } else {
3265 if (const Value *IRGuard = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls())) {
3266 SDValue GuardPtr = getValue(V: IRGuard);
3267 Guard = DAG.getLoad(VT: PtrMemTy, dl, Chain, Ptr: GuardPtr,
3268 PtrInfo: MachinePointerInfo(IRGuard, 0), Alignment: Align,
3269 MMOFlags: MachineMemOperand::MOVolatile);
3270 } else {
3271 LLVMContext &Ctx = *DAG.getContext();
3272 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
3273 Guard = DAG.getPOISON(VT: PtrMemTy);
3274 }
3275 }
3276 }
3277
3278 // Now both Guard (fresh cookie) and GuardVal (unmixed from stored value)
3279 // contain unmixed cookie values that can be compared directly.
3280
3281 // Perform the comparison via a getsetcc.
3282 SDValue Cmp = DAG.getSetCC(
3283 DL: dl, VT: TLI.getSetCCResultType(DL, Context&: *DAG.getContext(), VT: Guard.getValueType()),
3284 LHS: Guard, RHS: GuardVal, Cond: ISD::SETNE);
3285
3286 // If the guard/stackslot do not equal, branch to failure MBB.
3287 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: getControlRoot(),
3288 N2: Cmp, N3: DAG.getBasicBlock(MBB: SPD.getFailureMBB()));
3289 // Otherwise branch to success MBB.
3290 SDValue Br = DAG.getNode(Opcode: ISD::BR, DL: dl,
3291 VT: MVT::Other, N1: BrCond,
3292 N2: DAG.getBasicBlock(MBB: SPD.getSuccessMBB()));
3293
3294 DAG.setRoot(Br);
3295}
3296
3297/// Codegen the failure basic block for a stack protector check.
3298///
3299/// A failure stack protector machine basic block consists simply of a call to
3300/// __stack_chk_fail().
3301///
3302/// For a high level explanation of how this fits into the stack protector
3303/// generation see the comment on the declaration of class
3304/// StackProtectorDescriptor.
3305void SelectionDAGBuilder::visitSPDescriptorFailure(
3306 StackProtectorDescriptor &SPD) {
3307
3308 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3309 MachineBasicBlock *ParentBB = SPD.getParentMBB();
3310 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3311 SDValue Chain;
3312
3313 // For -Oz builds with a guard check function, we use function-based
3314 // instrumentation. Otherwise, if we have a guard check function, we call it
3315 // in the failure block.
3316 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M, Libcalls: DAG.getLibcalls());
3317 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3318 // First create the loads to the guard/stack slot for the comparison.
3319 auto &DL = DAG.getDataLayout();
3320 EVT PtrTy = TLI.getFrameIndexTy(DL);
3321 EVT PtrMemTy = TLI.getPointerMemTy(DL, AS: DL.getAllocaAddrSpace());
3322
3323 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3324 int FI = MFI.getStackProtectorIndex();
3325
3326 SDLoc dl = getCurSDLoc();
3327 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3328 Align Align = DL.getPrefTypeAlign(
3329 Ty: PointerType::get(C&: M.getContext(), AddressSpace: DL.getAllocaAddrSpace()));
3330
3331 // Generate code to load the content of the guard slot.
3332 SDValue GuardVal = DAG.getLoad(
3333 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3334 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3335 MMOFlags: MachineMemOperand::MOVolatile);
3336
3337 if (TLI.useStackGuardMixFP())
3338 GuardVal = TLI.emitStackGuardMixFP(DAG, Val: GuardVal, DL: dl);
3339
3340 // The target provides a guard check function to validate the guard value.
3341 // Generate a call to that function with the content of the guard slot as
3342 // argument.
3343 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3344 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3345
3346 TargetLowering::ArgListTy Args;
3347 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(i: 0));
3348 if (GuardCheckFn->hasParamAttribute(ArgNo: 0, Kind: Attribute::AttrKind::InReg))
3349 Entry.IsInReg = true;
3350 Args.push_back(x: Entry);
3351
3352 TargetLowering::CallLoweringInfo CLI(DAG);
3353 CLI.setDebugLoc(getCurSDLoc())
3354 .setChain(DAG.getEntryNode())
3355 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3356 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3357
3358 Chain = TLI.LowerCallTo(CLI).second;
3359 } else {
3360 TargetLowering::MakeLibCallOptions CallOptions;
3361 CallOptions.setDiscardResult(true);
3362 Chain = TLI.makeLibCall(DAG, LC: RTLIB::STACKPROTECTOR_CHECK_FAIL, RetVT: MVT::isVoid,
3363 Ops: {}, CallOptions, dl: getCurSDLoc())
3364 .second;
3365 }
3366
3367 // Emit a trap instruction if we are required to do so.
3368 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3369 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3370 Chain = DAG.getNode(Opcode: ISD::TRAP, DL: getCurSDLoc(), VT: MVT::Other, Operand: Chain);
3371
3372 DAG.setRoot(Chain);
3373}
3374
3375/// visitBitTestHeader - This function emits necessary code to produce value
3376/// suitable for "bit tests"
3377void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
3378 MachineBasicBlock *SwitchBB) {
3379 SDLoc dl = getCurSDLoc();
3380
3381 // Subtract the minimum value.
3382 SDValue SwitchOp = getValue(V: B.SValue);
3383 EVT VT = SwitchOp.getValueType();
3384 SDValue RangeSub =
3385 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp, N2: DAG.getConstant(Val: B.First, DL: dl, VT));
3386
3387 // Determine the type of the test operands.
3388 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3389 bool UsePtrType = false;
3390 if (!TLI.isTypeLegal(VT)) {
3391 UsePtrType = true;
3392 } else {
3393 for (const BitTestCase &Case : B.Cases)
3394 if (!isUIntN(N: VT.getSizeInBits(), x: Case.Mask)) {
3395 // Switch table case range are encoded into series of masks.
3396 // Just use pointer type, it's guaranteed to fit.
3397 UsePtrType = true;
3398 break;
3399 }
3400 }
3401 SDValue Sub = RangeSub;
3402 if (UsePtrType) {
3403 VT = TLI.getPointerTy(DL: DAG.getDataLayout());
3404 Sub = DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT);
3405 }
3406
3407 B.RegVT = VT.getSimpleVT();
3408 B.Reg = FuncInfo.CreateReg(VT: B.RegVT);
3409 SDValue CopyTo = DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: B.Reg, N: Sub);
3410
3411 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3412
3413 if (!B.FallthroughUnreachable)
3414 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
3415 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
3416 SwitchBB->normalizeSuccProbs();
3417
3418 SDValue Root = CopyTo;
3419 if (!B.FallthroughUnreachable) {
3420 // Conditional branch to the default block.
3421 SDValue RangeCmp = DAG.getSetCC(DL: dl,
3422 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3423 VT: RangeSub.getValueType()),
3424 LHS: RangeSub, RHS: DAG.getConstant(Val: B.Range, DL: dl, VT: RangeSub.getValueType()),
3425 Cond: ISD::SETUGT);
3426
3427 Root = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: Root, N2: RangeCmp,
3428 N3: DAG.getBasicBlock(MBB: B.Default));
3429 }
3430
3431 // Avoid emitting unnecessary branches to the next block.
3432 if (MBB != NextBlock(MBB: SwitchBB))
3433 Root = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: Root, N2: DAG.getBasicBlock(MBB));
3434
3435 DAG.setRoot(Root);
3436}
3437
3438/// visitBitTestCase - this function produces one "bit test"
3439void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
3440 MachineBasicBlock *NextMBB,
3441 BranchProbability BranchProbToNext,
3442 Register Reg, BitTestCase &B,
3443 MachineBasicBlock *SwitchBB) {
3444 SDLoc dl = getCurSDLoc();
3445 MVT VT = BB.RegVT;
3446 SDValue ShiftOp = DAG.getCopyFromReg(Chain: getControlRoot(), dl, Reg, VT);
3447 SDValue Cmp;
3448 unsigned PopCount = llvm::popcount(Value: B.Mask);
3449 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3450 if (PopCount == 1) {
3451 // Testing for a single bit; just compare the shift count with what it
3452 // would need to be to shift a 1 bit in that position.
3453 Cmp = DAG.getSetCC(
3454 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3455 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_zero(Val: B.Mask), DL: dl, VT),
3456 Cond: ISD::SETEQ);
3457 } else if (PopCount == BB.Range) {
3458 // There is only one zero bit in the range, test for it directly.
3459 Cmp = DAG.getSetCC(
3460 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3461 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_one(Value: B.Mask), DL: dl, VT), Cond: ISD::SETNE);
3462 } else {
3463 // Make desired shift
3464 SDValue SwitchVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT,
3465 N1: DAG.getConstant(Val: 1, DL: dl, VT), N2: ShiftOp);
3466
3467 // Emit bit tests and jumps
3468 SDValue AndOp = DAG.getNode(Opcode: ISD::AND, DL: dl,
3469 VT, N1: SwitchVal, N2: DAG.getConstant(Val: B.Mask, DL: dl, VT));
3470 Cmp = DAG.getSetCC(
3471 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3472 LHS: AndOp, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
3473 }
3474
3475 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3476 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
3477 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3478 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
3479 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3480 // one as they are relative probabilities (and thus work more like weights),
3481 // and hence we need to normalize them to let the sum of them become one.
3482 SwitchBB->normalizeSuccProbs();
3483
3484 SDValue BrAnd = DAG.getNode(Opcode: ISD::BRCOND, DL: dl,
3485 VT: MVT::Other, N1: getControlRoot(),
3486 N2: Cmp, N3: DAG.getBasicBlock(MBB: B.TargetBB));
3487
3488 // Avoid emitting unnecessary branches to the next block.
3489 if (NextMBB != NextBlock(MBB: SwitchBB))
3490 BrAnd = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrAnd,
3491 N2: DAG.getBasicBlock(MBB: NextMBB));
3492
3493 DAG.setRoot(BrAnd);
3494}
3495
3496void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3497 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3498
3499 // Retrieve successors. Look through artificial IR level blocks like
3500 // catchswitch for successors.
3501 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
3502 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3503 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(BB: EHPadBB);
3504
3505 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3506 // have to do anything here to lower funclet bundles.
3507 failForInvalidBundles(I, Name: "invokes",
3508 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3509 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3510 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3511 LLVMContext::OB_clang_arc_attachedcall,
3512 LLVMContext::OB_kcfi});
3513
3514 const Value *Callee(I.getCalledOperand());
3515 const Function *Fn = dyn_cast<Function>(Val: Callee);
3516 if (isa<InlineAsm>(Val: Callee))
3517 visitInlineAsm(Call: I, EHPadBB);
3518 else if (Fn && Fn->isIntrinsic()) {
3519 switch (Fn->getIntrinsicID()) {
3520 default:
3521 llvm_unreachable("Cannot invoke this intrinsic");
3522 case Intrinsic::donothing:
3523 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3524 case Intrinsic::seh_try_begin:
3525 case Intrinsic::seh_scope_begin:
3526 case Intrinsic::seh_try_end:
3527 case Intrinsic::seh_scope_end:
3528 if (EHPadMBB)
3529 // a block referenced by EH table
3530 // so dtor-funclet not removed by opts
3531 EHPadMBB->setMachineBlockAddressTaken();
3532 break;
3533 case Intrinsic::experimental_patchpoint_void:
3534 case Intrinsic::experimental_patchpoint:
3535 visitPatchpoint(CB: I, EHPadBB);
3536 break;
3537 case Intrinsic::experimental_gc_statepoint:
3538 LowerStatepoint(I: cast<GCStatepointInst>(Val: I), EHPadBB);
3539 break;
3540 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3541 // but these intrinsics are special because they can be invoked, so we
3542 // manually lower it to a DAG node here.
3543 case Intrinsic::wasm_throw: {
3544 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3545 std::array<SDValue, 4> Ops = {
3546 getControlRoot(), // inchain for the terminator node
3547 DAG.getTargetConstant(Val: Intrinsic::wasm_throw, DL: getCurSDLoc(),
3548 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3549 getValue(V: I.getArgOperand(i: 0)), // tag
3550 getValue(V: I.getArgOperand(i: 1)) // thrown value
3551 };
3552 SDVTList VTs = DAG.getVTList(VTs: ArrayRef<EVT>({MVT::Other})); // outchain
3553 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3554 break;
3555 }
3556 case Intrinsic::wasm_rethrow: {
3557 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3558 std::array<SDValue, 2> Ops = {
3559 getControlRoot(), // inchain for the terminator node
3560 DAG.getTargetConstant(Val: Intrinsic::wasm_rethrow, DL: getCurSDLoc(),
3561 VT: TLI.getPointerTy(DL: DAG.getDataLayout()))};
3562 SDVTList VTs = DAG.getVTList(VTs: ArrayRef<EVT>({MVT::Other})); // outchain
3563 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3564 break;
3565 }
3566 }
3567 } else if (I.hasDeoptState()) {
3568 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3569 // Eventually we will support lowering the @llvm.experimental.deoptimize
3570 // intrinsic, and right now there are no plans to support other intrinsics
3571 // with deopt state.
3572 LowerCallSiteWithDeoptBundle(Call: &I, Callee: getValue(V: Callee), EHPadBB);
3573 } else if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
3574 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), EHPadBB);
3575 } else {
3576 LowerCallTo(CB: I, Callee: getValue(V: Callee), IsTailCall: false, IsMustTailCall: false, EHPadBB);
3577 }
3578
3579 // If the value of the invoke is used outside of its defining block, make it
3580 // available as a virtual register.
3581 // We already took care of the exported value for the statepoint instruction
3582 // during call to the LowerStatepoint.
3583 if (!isa<GCStatepointInst>(Val: I)) {
3584 CopyToExportRegsIfNeeded(V: &I);
3585 }
3586
3587 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3588 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3589 BranchProbability EHPadBBProb =
3590 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3591 : BranchProbability::getZero();
3592 findUnwindDestinations(FuncInfo, EHPadBB, Prob: EHPadBBProb, UnwindDests);
3593
3594 // Update successor info.
3595 addSuccessorWithProb(Src: InvokeMBB, Dst: Return);
3596 for (auto &UnwindDest : UnwindDests) {
3597 UnwindDest.first->setIsEHPad();
3598 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3599 }
3600 InvokeMBB->normalizeSuccProbs();
3601
3602 // Drop into normal successor.
3603 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other, N1: getControlRoot(),
3604 N2: DAG.getBasicBlock(MBB: Return)));
3605}
3606
3607/// The intrinsics currently supported by callbr are implicit control flow
3608/// intrinsics such as amdgcn.kill.
3609/// - they should be called (no "dontcall-" attributes)
3610/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
3611/// - they do not need custom argument handling (no
3612/// TLI.CollectTargetIntrinsicOperands())
3613void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
3614#ifndef NDEBUG
3615 SmallVector<TargetLowering::IntrinsicInfo, 2> Infos;
3616 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3617 Infos, I, DAG.getMachineFunction(), I.getIntrinsicID());
3618 assert(Infos.empty() && "Intrinsic touches memory");
3619#endif
3620
3621 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
3622
3623 SmallVector<SDValue, 8> Ops =
3624 getTargetIntrinsicOperands(I, HasChain, OnlyLoad);
3625 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
3626
3627 // Create the node.
3628 SDValue Result =
3629 getTargetNonMemIntrinsicNode(IntrinsicVT: *I.getType(), HasChain, Ops, VTs);
3630 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
3631
3632 setValue(V: &I, NewN: Result);
3633}
3634
3635void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3636 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3637
3638 if (I.isInlineAsm()) {
3639 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3640 // have to do anything here to lower funclet bundles.
3641 failForInvalidBundles(I, Name: "callbrs",
3642 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_funclet});
3643 visitInlineAsm(Call: I);
3644 } else {
3645 assert(!I.hasOperandBundles() &&
3646 "Can't have operand bundles for intrinsics");
3647 visitCallBrIntrinsic(I);
3648 }
3649 CopyToExportRegsIfNeeded(V: &I);
3650
3651 // Retrieve successors.
3652 SmallPtrSet<BasicBlock *, 8> Dests;
3653 Dests.insert(Ptr: I.getDefaultDest());
3654 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getDefaultDest());
3655
3656 // Update successor info.
3657 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3658 // TODO: For most of the cases where there is an intrinsic callbr, we're
3659 // having exactly one indirect target, which will be unreachable. As soon as
3660 // this changes, we might need to enhance
3661 // Target->setIsInlineAsmBrIndirectTarget or add something similar for
3662 // intrinsic indirect branches.
3663 if (I.isInlineAsm()) {
3664 for (BasicBlock *Dest : I.getIndirectDests()) {
3665 MachineBasicBlock *Target = FuncInfo.getMBB(BB: Dest);
3666 Target->setIsInlineAsmBrIndirectTarget();
3667 // If we introduce a type of asm goto statement that is permitted to use
3668 // an indirect call instruction to jump to its labels, then we should add
3669 // a call to Target->setMachineBlockAddressTaken() here, to mark the
3670 // target block as requiring a BTI.
3671
3672 Target->setLabelMustBeEmitted();
3673 // Don't add duplicate machine successors.
3674 if (Dests.insert(Ptr: Dest).second)
3675 addSuccessorWithProb(Src: CallBrMBB, Dst: Target, Prob: BranchProbability::getZero());
3676 }
3677 }
3678 CallBrMBB->normalizeSuccProbs();
3679
3680 // Drop into default successor.
3681 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(),
3682 VT: MVT::Other, N1: getControlRoot(),
3683 N2: DAG.getBasicBlock(MBB: Return)));
3684}
3685
3686void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3687 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3688}
3689
3690void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3691 assert(FuncInfo.MBB->isEHPad() &&
3692 "Call to landingpad not in landing pad!");
3693
3694 // If there aren't registers to copy the values into (e.g., during SjLj
3695 // exceptions), then don't bother to create these DAG nodes.
3696 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3697 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3698 if (TLI.getExceptionPointerRegister(EH: FuncInfo.ExceptionModel, PersonalityFn) ==
3699 0 &&
3700 TLI.getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel,
3701 PersonalityFn) == 0)
3702 return;
3703
3704 // If landingpad's return type is token type, we don't create DAG nodes
3705 // for its exception pointer and selector value. The extraction of exception
3706 // pointer or selector value from token type landingpads is not currently
3707 // supported.
3708 if (LP.getType()->isTokenTy())
3709 return;
3710
3711 // LangRef leaves the result type target-specific, so diagnose types this
3712 // lowering cannot represent instead of asserting.
3713 SDLoc dl = getCurSDLoc();
3714 if (!isExceptionPointerAndSelectorType(Ty: LP.getType())) {
3715 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3716 *LP.getFunction(),
3717 "landingpad result type must be a struct of an exception pointer and "
3718 "an integer selector",
3719 dl.getDebugLoc()));
3720 setValueToPoison(V: &LP, dl);
3721 return;
3722 }
3723
3724 SmallVector<EVT, 2> ValueVTs;
3725 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: LP.getType(), ValueVTs);
3726 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3727
3728 // Get the two live-in registers as SDValues. The physregs have already been
3729 // copied into virtual registers.
3730 SDValue Ops[2];
3731 if (FuncInfo.ExceptionPointerVirtReg) {
3732 Ops[0] = DAG.getZExtOrTrunc(
3733 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3734 Reg: FuncInfo.ExceptionPointerVirtReg,
3735 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3736 DL: dl, VT: ValueVTs[0]);
3737 } else {
3738 Ops[0] = DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3739 }
3740 Ops[1] = DAG.getZExtOrTrunc(
3741 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3742 Reg: FuncInfo.ExceptionSelectorVirtReg,
3743 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3744 DL: dl, VT: ValueVTs[1]);
3745
3746 // Merge into one.
3747 SDValue Res = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
3748 VTList: DAG.getVTList(VTs: ValueVTs), Ops);
3749 setValue(V: &LP, NewN: Res);
3750}
3751
3752void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3753 MachineBasicBlock *Last) {
3754 // Update JTCases.
3755 for (JumpTableBlock &JTB : SL->JTCases)
3756 if (JTB.first.HeaderBB == First)
3757 JTB.first.HeaderBB = Last;
3758
3759 // Update BitTestCases.
3760 for (BitTestBlock &BTB : SL->BitTestCases)
3761 if (BTB.Parent == First)
3762 BTB.Parent = Last;
3763}
3764
3765void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3766 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3767
3768 // Update machine-CFG edges with unique successors.
3769 SmallPtrSet<BasicBlock *, 32> Done;
3770 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3771 BasicBlock *BB = I.getSuccessor(i);
3772 bool Inserted = Done.insert(Ptr: BB).second;
3773 if (!Inserted)
3774 continue;
3775
3776 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3777 addSuccessorWithProb(Src: IndirectBrMBB, Dst: Succ);
3778 }
3779 IndirectBrMBB->normalizeSuccProbs();
3780
3781 DAG.setRoot(DAG.getNode(Opcode: ISD::BRIND, DL: getCurSDLoc(),
3782 VT: MVT::Other, N1: getControlRoot(),
3783 N2: getValue(V: I.getAddress())));
3784}
3785
3786void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3787 if (!I.shouldLowerToTrap(TrapUnreachable: DAG.getTarget().Options.TrapUnreachable,
3788 NoTrapAfterNoreturn: DAG.getTarget().Options.NoTrapAfterNoreturn))
3789 return;
3790
3791 DAG.setRoot(DAG.getNode(Opcode: ISD::TRAP, DL: getCurSDLoc(), VT: MVT::Other, Operand: DAG.getRoot()));
3792}
3793
3794void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3795 SDNodeFlags Flags;
3796 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3797 Flags.copyFMF(FPMO: *FPOp);
3798
3799 SDValue Op = getValue(V: I.getOperand(i: 0));
3800 SDValue UnNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op.getValueType(),
3801 Operand: Op, Flags);
3802 setValue(V: &I, NewN: UnNodeValue);
3803}
3804
3805void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3806 SDNodeFlags Flags;
3807 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(Val: &I)) {
3808 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3809 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3810 }
3811 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(Val: &I))
3812 Flags.setExact(ExactOp->isExact());
3813 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(Val: &I))
3814 Flags.setDisjoint(DisjointOp->isDisjoint());
3815 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3816 Flags.copyFMF(FPMO: *FPOp);
3817
3818 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3819 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3820 SDValue BinNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(),
3821 N1: Op1, N2: Op2, Flags);
3822 setValue(V: &I, NewN: BinNodeValue);
3823}
3824
3825void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3826 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3827 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3828
3829 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3830 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
3831
3832 // Coerce the shift amount to the right type if we can. This exposes the
3833 // truncate or zext to optimization early.
3834 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3835 assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3836 "Unexpected shift type");
3837 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
3838 }
3839
3840 bool nuw = false;
3841 bool nsw = false;
3842 bool exact = false;
3843
3844 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3845
3846 if (const OverflowingBinaryOperator *OFBinOp =
3847 dyn_cast<const OverflowingBinaryOperator>(Val: &I)) {
3848 nuw = OFBinOp->hasNoUnsignedWrap();
3849 nsw = OFBinOp->hasNoSignedWrap();
3850 }
3851 if (const PossiblyExactOperator *ExactOp =
3852 dyn_cast<const PossiblyExactOperator>(Val: &I))
3853 exact = ExactOp->isExact();
3854 }
3855 SDNodeFlags Flags;
3856 Flags.setExact(exact);
3857 Flags.setNoSignedWrap(nsw);
3858 Flags.setNoUnsignedWrap(nuw);
3859 SDValue Res = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1, N2: Op2,
3860 Flags);
3861 setValue(V: &I, NewN: Res);
3862}
3863
3864void SelectionDAGBuilder::visitSDiv(const User &I) {
3865 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3866 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3867
3868 SDNodeFlags Flags;
3869 Flags.setExact(isa<PossiblyExactOperator>(Val: &I) &&
3870 cast<PossiblyExactOperator>(Val: &I)->isExact());
3871 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SDIV, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1,
3872 N2: Op2, Flags));
3873}
3874
3875void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3876 ICmpInst::Predicate predicate = I.getPredicate();
3877 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3878 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3879 ISD::CondCode Opcode = getICmpCondCode(Pred: predicate);
3880
3881 auto &TLI = DAG.getTargetLoweringInfo();
3882 EVT MemVT =
3883 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
3884
3885 // If a pointer's DAG type is larger than its memory type then the DAG values
3886 // are zero-extended. This breaks signed comparisons so truncate back to the
3887 // underlying type before doing the compare.
3888 if (Op1.getValueType() != MemVT) {
3889 Op1 = DAG.getPtrExtOrTrunc(Op: Op1, DL: getCurSDLoc(), VT: MemVT);
3890 Op2 = DAG.getPtrExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: MemVT);
3891 }
3892
3893 SDNodeFlags Flags;
3894 Flags.setSameSign(I.hasSameSign());
3895
3896 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3897 Ty: I.getType());
3898 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Opcode,
3899 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3900}
3901
3902void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3903 FCmpInst::Predicate predicate = I.getPredicate();
3904 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3905 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3906
3907 ISD::CondCode Condition = getFCmpCondCode(Pred: predicate);
3908 auto *FPMO = cast<FPMathOperator>(Val: &I);
3909 if (FPMO->hasNoNaNs() ||
3910 (DAG.isKnownNeverNaN(Op: Op1) && DAG.isKnownNeverNaN(Op: Op2)))
3911 Condition = getFCmpCodeWithoutNaN(CC: Condition);
3912
3913 SDNodeFlags Flags;
3914 Flags.copyFMF(FPMO: *FPMO);
3915
3916 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3917 Ty: I.getType());
3918 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Condition,
3919 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3920}
3921
3922// Check if the condition of the select has one use or two users that are both
3923// selects with the same condition.
3924static bool hasOnlySelectUsers(const Value *Cond) {
3925 return llvm::all_of(Range: Cond->users(), P: [](const Value *V) {
3926 return isa<SelectInst>(Val: V);
3927 });
3928}
3929
3930void SelectionDAGBuilder::visitSelect(const User &I) {
3931 SmallVector<EVT, 4> ValueVTs;
3932 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
3933 ValueVTs);
3934 unsigned NumValues = ValueVTs.size();
3935 if (NumValues == 0) return;
3936
3937 SmallVector<SDValue, 4> Values(NumValues);
3938 SDValue Cond = getValue(V: I.getOperand(i: 0));
3939 SDValue LHSVal = getValue(V: I.getOperand(i: 1));
3940 SDValue RHSVal = getValue(V: I.getOperand(i: 2));
3941 SmallVector<SDValue, 1> BaseOps(1, Cond);
3942 ISD::NodeType OpCode =
3943 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3944
3945 bool IsUnaryAbs = false;
3946 bool Negate = false;
3947
3948 SDNodeFlags Flags;
3949 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3950 Flags.copyFMF(FPMO: *FPOp);
3951
3952 Flags.setUnpredictable(
3953 cast<SelectInst>(Val: I).getMetadata(KindID: LLVMContext::MD_unpredictable));
3954
3955 // Min/max matching is only viable if all output VTs are the same.
3956 if (all_equal(Range&: ValueVTs)) {
3957 EVT VT = ValueVTs[0];
3958 LLVMContext &Ctx = *DAG.getContext();
3959 auto &TLI = DAG.getTargetLoweringInfo();
3960
3961 // We care about the legality of the operation after it has been type
3962 // legalized.
3963 while (TLI.getTypeAction(Context&: Ctx, VT) != TargetLoweringBase::TypeLegal)
3964 VT = TLI.getTypeToTransformTo(Context&: Ctx, VT);
3965
3966 // If the vselect is legal, assume we want to leave this as a vector setcc +
3967 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3968 // min/max is legal on the scalar type.
3969 bool UseScalarMinMax = VT.isVector() &&
3970 !TLI.isOperationLegalOrCustom(Op: ISD::VSELECT, VT);
3971
3972 // ValueTracking's select pattern matching does not account for -0.0,
3973 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3974 // -0.0 is less than +0.0.
3975 const Value *LHS, *RHS;
3976 auto SPR = matchSelectPattern(V: &I, LHS, RHS);
3977 ISD::NodeType Opc = ISD::DELETED_NODE;
3978 switch (SPR.Flavor) {
3979 case SPF_UMAX: Opc = ISD::UMAX; break;
3980 case SPF_UMIN: Opc = ISD::UMIN; break;
3981 case SPF_SMAX: Opc = ISD::SMAX; break;
3982 case SPF_SMIN: Opc = ISD::SMIN; break;
3983 case SPF_FMINNUM:
3984 if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
3985 break;
3986
3987 switch (SPR.NaNBehavior) {
3988 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3989 case SPNB_RETURNS_ANY:
3990 case SPNB_RETURNS_NAN:
3991 break;
3992 case SPNB_RETURNS_OTHER:
3993 Opc = ISD::FMINIMUMNUM;
3994 Flags.setNoSignedZeros(true);
3995 break;
3996 }
3997 break;
3998 case SPF_FMAXNUM:
3999 if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
4000 break;
4001
4002 switch (SPR.NaNBehavior) {
4003 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4004 case SPNB_RETURNS_NAN:
4005 case SPNB_RETURNS_ANY:
4006 break;
4007 case SPNB_RETURNS_OTHER:
4008 Opc = ISD::FMAXIMUMNUM;
4009 Flags.setNoSignedZeros(true);
4010 break;
4011 }
4012 break;
4013 case SPF_NABS:
4014 Negate = true;
4015 [[fallthrough]];
4016 case SPF_ABS:
4017 IsUnaryAbs = true;
4018 Opc = ISD::ABS;
4019 break;
4020 default: break;
4021 }
4022
4023 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
4024 (TLI.isOperationLegalOrCustom(Op: Opc, VT) ||
4025 (UseScalarMinMax &&
4026 TLI.isOperationLegalOrCustom(Op: Opc, VT: VT.getScalarType()))) &&
4027 // If the underlying comparison instruction is used by any other
4028 // instruction, the consumed instructions won't be destroyed, so it is
4029 // not profitable to convert to a min/max.
4030 hasOnlySelectUsers(Cond: cast<SelectInst>(Val: I).getCondition())) {
4031 OpCode = Opc;
4032 LHSVal = getValue(V: LHS);
4033 RHSVal = getValue(V: RHS);
4034 BaseOps.clear();
4035 }
4036
4037 if (IsUnaryAbs) {
4038 OpCode = Opc;
4039 LHSVal = getValue(V: LHS);
4040 BaseOps.clear();
4041 }
4042 }
4043
4044 if (IsUnaryAbs) {
4045 for (unsigned i = 0; i != NumValues; ++i) {
4046 SDLoc dl = getCurSDLoc();
4047 EVT VT = LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i);
4048 Values[i] =
4049 DAG.getNode(Opcode: OpCode, DL: dl, VT, Operand: LHSVal.getValue(R: LHSVal.getResNo() + i));
4050 if (Negate)
4051 Values[i] = DAG.getNegative(Val: Values[i], DL: dl, VT);
4052 }
4053 } else {
4054 for (unsigned i = 0; i != NumValues; ++i) {
4055 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
4056 Ops.push_back(Elt: SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
4057 Ops.push_back(Elt: SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
4058 Values[i] = DAG.getNode(
4059 Opcode: OpCode, DL: getCurSDLoc(),
4060 VT: LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i), Ops, Flags);
4061 }
4062 }
4063
4064 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4065 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
4066}
4067
4068void SelectionDAGBuilder::visitTrunc(const User &I) {
4069 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
4070 SDValue N = getValue(V: I.getOperand(i: 0));
4071 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4072 Ty: I.getType());
4073 SDNodeFlags Flags;
4074 if (auto *Trunc = dyn_cast<TruncInst>(Val: &I)) {
4075 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
4076 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
4077 }
4078
4079 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4080}
4081
4082void SelectionDAGBuilder::visitZExt(const User &I) {
4083 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4084 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
4085 SDValue N = getValue(V: I.getOperand(i: 0));
4086 auto &TLI = DAG.getTargetLoweringInfo();
4087 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4088
4089 SDNodeFlags Flags;
4090 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(Val: &I))
4091 Flags.setNonNeg(PNI->hasNonNeg());
4092
4093 // Eagerly use nonneg information to canonicalize towards sign_extend if
4094 // that is the target's preference.
4095 // TODO: Let the target do this later.
4096 if (Flags.hasNonNeg() &&
4097 TLI.isSExtCheaperThanZExt(FromTy: N.getValueType(), ToTy: DestVT)) {
4098 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4099 return;
4100 }
4101
4102 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4103}
4104
4105void SelectionDAGBuilder::visitSExt(const User &I) {
4106 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4107 // SExt also can't be a cast to bool for same reason. So, nothing much to do
4108 SDValue N = getValue(V: I.getOperand(i: 0));
4109 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4110 Ty: I.getType());
4111 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4112}
4113
4114void SelectionDAGBuilder::visitFPTrunc(const User &I) {
4115 // FPTrunc is never a no-op cast, no need to check
4116 SDValue N = getValue(V: I.getOperand(i: 0));
4117 SDLoc dl = getCurSDLoc();
4118 SDNodeFlags Flags;
4119 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
4120 Flags.copyFMF(FPMO: *FPOp);
4121 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4122 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4123 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: DestVT, N1: N,
4124 N2: DAG.getTargetConstant(
4125 Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4126 Flags));
4127}
4128
4129void SelectionDAGBuilder::visitFPExt(const User &I) {
4130 // FPExt is never a no-op cast, no need to check
4131 SDValue N = getValue(V: I.getOperand(i: 0));
4132 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4133 Ty: I.getType());
4134 SDNodeFlags Flags;
4135 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
4136 Flags.copyFMF(FPMO: *FPOp);
4137 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4138}
4139
4140void SelectionDAGBuilder::visitFPToUI(const User &I) {
4141 // FPToUI is never a no-op cast, no need to check
4142 SDValue N = getValue(V: I.getOperand(i: 0));
4143 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4144 Ty: I.getType());
4145 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4146}
4147
4148void SelectionDAGBuilder::visitFPToSI(const User &I) {
4149 // FPToSI is never a no-op cast, no need to check
4150 SDValue N = getValue(V: I.getOperand(i: 0));
4151 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4152 Ty: I.getType());
4153 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4154}
4155
4156void SelectionDAGBuilder::visitUIToFP(const User &I) {
4157 // UIToFP is never a no-op cast, no need to check
4158 SDValue N = getValue(V: I.getOperand(i: 0));
4159 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4160 Ty: I.getType());
4161 SDNodeFlags Flags;
4162 Flags.setNonNeg(cast<PossiblyNonNegInst>(Val: &I)->hasNonNeg());
4163 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
4164
4165 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4166}
4167
4168void SelectionDAGBuilder::visitSIToFP(const User &I) {
4169 // SIToFP is never a no-op cast, no need to check
4170 SDValue N = getValue(V: I.getOperand(i: 0));
4171 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4172 Ty: I.getType());
4173 SDNodeFlags Flags;
4174 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
4175
4176 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4177}
4178
4179void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
4180 SDValue N = getValue(V: I.getOperand(i: 0));
4181 // By definition the type of the ptrtoaddr must be equal to the address type.
4182 const auto &TLI = DAG.getTargetLoweringInfo();
4183 EVT AddrVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4184 // The address width must be smaller or equal to the pointer representation
4185 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).
4186 N = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: AddrVT, Operand: N);
4187 setValue(V: &I, NewN: N);
4188}
4189
4190void SelectionDAGBuilder::visitPtrToInt(const User &I) {
4191 // What to do depends on the size of the integer and the size of the pointer.
4192 // We can either truncate, zero extend, or no-op, accordingly.
4193 SDValue N = getValue(V: I.getOperand(i: 0));
4194 auto &TLI = DAG.getTargetLoweringInfo();
4195 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4196 Ty: I.getType());
4197 EVT PtrMemVT =
4198 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i: 0)->getType());
4199 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
4200 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
4201 setValue(V: &I, NewN: N);
4202}
4203
4204void SelectionDAGBuilder::visitIntToPtr(const User &I) {
4205 // What to do depends on the size of the integer and the size of the pointer.
4206 // We can either truncate, zero extend, or no-op, accordingly.
4207 SDValue N = getValue(V: I.getOperand(i: 0));
4208 auto &TLI = DAG.getTargetLoweringInfo();
4209 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4210 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4211 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
4212 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
4213 setValue(V: &I, NewN: N);
4214}
4215
4216void SelectionDAGBuilder::visitBitCast(const User &I) {
4217 SDValue N = getValue(V: I.getOperand(i: 0));
4218 SDLoc dl = getCurSDLoc();
4219 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4220 Ty: I.getType());
4221
4222 // BitCast assures us that source and destination are the same size so this is
4223 // either a BITCAST or a no-op.
4224 if (DestVT != N.getValueType())
4225 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITCAST, DL: dl,
4226 VT: DestVT, Operand: N)); // convert types.
4227 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
4228 // might fold any kind of constant expression to an integer constant and that
4229 // is not what we are looking for. Only recognize a bitcast of a genuine
4230 // constant integer as an opaque constant.
4231 else if(ConstantInt *C = dyn_cast<ConstantInt>(Val: I.getOperand(i: 0)))
4232 setValue(V: &I, NewN: DAG.getConstant(Val: C->getValue(), DL: dl, VT: DestVT, /*isTarget=*/false,
4233 /*isOpaque*/true));
4234 else
4235 setValue(V: &I, NewN: N); // noop cast.
4236}
4237
4238void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
4239 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4240 const Value *SV = I.getOperand(i: 0);
4241 SDValue N = getValue(V: SV);
4242 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4243
4244 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
4245 unsigned DestAS = I.getType()->getPointerAddressSpace();
4246
4247 if (!TM.isNoopAddrSpaceCast(DL: DAG.getDataLayout(), SrcAS, DestAS)) {
4248 SDNodeFlags Flags;
4249 if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &I))
4250 Flags.setNonNull(ASC->hasNonNull());
4251 N = DAG.getAddrSpaceCast(dl: getCurSDLoc(), VT: DestVT, Ptr: N, SrcAS, DestAS, Flags);
4252 }
4253
4254 setValue(V: &I, NewN: N);
4255}
4256
4257void SelectionDAGBuilder::visitInsertElement(const User &I) {
4258 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4259 SDValue InVec = getValue(V: I.getOperand(i: 0));
4260 SDValue InVal = getValue(V: I.getOperand(i: 1));
4261 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 2)), DL: getCurSDLoc(),
4262 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
4263 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: getCurSDLoc(),
4264 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
4265 N1: InVec, N2: InVal, N3: InIdx));
4266}
4267
4268void SelectionDAGBuilder::visitBitInsert(const User &I) {
4269 SDValue Base = getValue(V: I.getOperand(i: 0));
4270 SDValue Val = getValue(V: I.getOperand(i: 1));
4271 SDValue Offset = getValue(V: I.getOperand(i: 2));
4272 EVT BaseVT = Base.getValueType();
4273 EVT ValVT = Val.getValueType();
4274 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4275 SDLoc dl = getCurSDLoc();
4276
4277 assert(BaseVT.getSizeInBits() >= ValVT.getSizeInBits() &&
4278 "bitinsert val wider than base should be rejected by verifier");
4279
4280 // If Val is a float, cast it to an integer of the same bitwidth
4281 // so DAG.getZExtOrTrunc can process it safely.
4282 if (!ValVT.isInteger()) {
4283 ValVT = ValVT.changeTypeToInteger();
4284 Val = DAG.getBitcast(VT: ValVT, V: Val);
4285 }
4286
4287 // Legalize shift amount to the target's shift amount type.
4288 EVT ShiftAmtTy = TLI.getShiftAmountTy(LHSTy: BaseVT, DL: DAG.getDataLayout());
4289 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Op: Offset, DL: dl, VT: ShiftAmtTy);
4290
4291 unsigned BaseBitWidth = BaseVT.getScalarSizeInBits();
4292 unsigned ValBitWidth = ValVT.getScalarSizeInBits();
4293 APInt InsertMask = APInt::getLowBitsSet(numBits: BaseBitWidth, loBitsSet: ValBitWidth);
4294 SDValue ShiftedMask =
4295 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: BaseVT, N1: DAG.getConstant(Val: InsertMask, DL: dl, VT: BaseVT),
4296 N2: LegalShiftAmount);
4297 SDValue ClearMask = DAG.getNOT(DL: dl, Val: ShiftedMask, VT: BaseVT);
4298 SDValue ClearedBase = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BaseVT, N1: Base, N2: ClearMask);
4299
4300 SDValue ExtVal = DAG.getZExtOrTrunc(Op: Val, DL: dl, VT: BaseVT);
4301 SDValue ShiftedVal =
4302 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: BaseVT, N1: ExtVal, N2: LegalShiftAmount);
4303 SDValue Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BaseVT, N1: ClearedBase, N2: ShiftedVal);
4304 setValue(V: &I, NewN: Result);
4305}
4306
4307void SelectionDAGBuilder::visitBitExtract(const User &I) {
4308 SDValue Src = getValue(V: I.getOperand(i: 0));
4309 SDValue Offset = getValue(V: I.getOperand(i: 1));
4310 EVT SrcVT = Src.getValueType();
4311 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4312 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4313 SDLoc dl = getCurSDLoc();
4314
4315 assert(ResultVT.getSizeInBits() <= SrcVT.getSizeInBits() &&
4316 "bitextract result wider than source should be rejected by verifier");
4317
4318 // Legalize shift amount to the target's shift amount type.
4319 EVT ShiftAmtTy = TLI.getShiftAmountTy(LHSTy: SrcVT, DL: DAG.getDataLayout());
4320 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Op: Offset, DL: dl, VT: ShiftAmtTy);
4321
4322 // Shift right by Offset - brings target field to bit 0
4323 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Src, N2: LegalShiftAmount);
4324
4325 SDValue Result;
4326 if (!ResultVT.isInteger()) {
4327 // Drop into the integer domain to safely truncate the shifted bits
4328 EVT IntResultVT = ResultVT.changeTypeToInteger();
4329 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: IntResultVT, Operand: Shifted);
4330 Result = DAG.getBitcast(VT: ResultVT, V: Result);
4331 } else {
4332 // Normal integer path
4333 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResultVT, Operand: Shifted);
4334 }
4335
4336 setValue(V: &I, NewN: Result);
4337}
4338
4339void SelectionDAGBuilder::visitExtractElement(const User &I) {
4340 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4341 SDValue InVec = getValue(V: I.getOperand(i: 0));
4342 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 1)), DL: getCurSDLoc(),
4343 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
4344 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: getCurSDLoc(),
4345 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
4346 N1: InVec, N2: InIdx));
4347}
4348
4349void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4350 SDValue Src1 = getValue(V: I.getOperand(i: 0));
4351 SDValue Src2 = getValue(V: I.getOperand(i: 1));
4352 ArrayRef<int> Mask;
4353 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &I))
4354 Mask = SVI->getShuffleMask();
4355 else
4356 Mask = cast<ConstantExpr>(Val: I).getShuffleMask();
4357 SDLoc DL = getCurSDLoc();
4358 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4359 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4360 EVT SrcVT = Src1.getValueType();
4361
4362 if (all_of(Range&: Mask, P: equal_to(Arg: 0)) && VT.isScalableVector()) {
4363 // Canonical splat form of first element of first input vector.
4364 SDValue FirstElt =
4365 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SrcVT.getScalarType(), N1: Src1,
4366 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4367 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: FirstElt));
4368 return;
4369 }
4370
4371 // For now, we only handle splats for scalable vectors.
4372 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4373 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4374 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4375
4376 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4377 unsigned MaskNumElts = Mask.size();
4378
4379 if (SrcNumElts == MaskNumElts) {
4380 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask));
4381 return;
4382 }
4383
4384 // Normalize the shuffle vector since mask and vector length don't match.
4385 if (SrcNumElts < MaskNumElts) {
4386 // Mask is longer than the source vectors. We can use concatenate vector to
4387 // make the mask and vectors lengths match.
4388
4389 if (MaskNumElts % SrcNumElts == 0) {
4390 // Mask length is a multiple of the source vector length.
4391 // Check if the shuffle is some kind of concatenation of the input
4392 // vectors.
4393 unsigned NumConcat = MaskNumElts / SrcNumElts;
4394 bool IsConcat = true;
4395 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4396 for (unsigned i = 0; i != MaskNumElts; ++i) {
4397 int Idx = Mask[i];
4398 if (Idx < 0)
4399 continue;
4400 // Ensure the indices in each SrcVT sized piece are sequential and that
4401 // the same source is used for the whole piece.
4402 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4403 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4404 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4405 IsConcat = false;
4406 break;
4407 }
4408 // Remember which source this index came from.
4409 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4410 }
4411
4412 // The shuffle is concatenating multiple vectors together. Just emit
4413 // a CONCAT_VECTORS operation.
4414 if (IsConcat) {
4415 SmallVector<SDValue, 8> ConcatOps;
4416 for (auto Src : ConcatSrcs) {
4417 if (Src < 0)
4418 ConcatOps.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
4419 else if (Src == 0)
4420 ConcatOps.push_back(Elt: Src1);
4421 else
4422 ConcatOps.push_back(Elt: Src2);
4423 }
4424 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps));
4425 return;
4426 }
4427 }
4428
4429 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
4430 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4431 EVT PaddedVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(),
4432 NumElements: PaddedMaskNumElts);
4433
4434 // Pad both vectors with undefs to make them the same length as the mask.
4435 SDValue UndefVal = DAG.getUNDEF(VT: SrcVT);
4436
4437 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4438 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4439 MOps1[0] = Src1;
4440 MOps2[0] = Src2;
4441
4442 Src1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps1);
4443 Src2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps2);
4444
4445 // Readjust mask for new input vector length.
4446 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4447 for (unsigned i = 0; i != MaskNumElts; ++i) {
4448 int Idx = Mask[i];
4449 if (Idx >= (int)SrcNumElts)
4450 Idx -= SrcNumElts - PaddedMaskNumElts;
4451 MappedOps[i] = Idx;
4452 }
4453
4454 SDValue Result = DAG.getVectorShuffle(VT: PaddedVT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps);
4455
4456 // If the concatenated vector was padded, extract a subvector with the
4457 // correct number of elements.
4458 if (MaskNumElts != PaddedMaskNumElts)
4459 Result = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Result,
4460 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4461
4462 setValue(V: &I, NewN: Result);
4463 return;
4464 }
4465
4466 assert(SrcNumElts > MaskNumElts);
4467
4468 // Analyze the access pattern of the vector to see if we can extract
4469 // two subvectors and do the shuffle.
4470 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4471 bool CanExtract = true;
4472 for (int Idx : Mask) {
4473 unsigned Input = 0;
4474 if (Idx < 0)
4475 continue;
4476
4477 if (Idx >= (int)SrcNumElts) {
4478 Input = 1;
4479 Idx -= SrcNumElts;
4480 }
4481
4482 // If all the indices come from the same MaskNumElts sized portion of
4483 // the sources we can use extract. Also make sure the extract wouldn't
4484 // extract past the end of the source.
4485 int NewStartIdx = alignDown(Value: Idx, Align: MaskNumElts);
4486 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4487 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4488 CanExtract = false;
4489 // Make sure we always update StartIdx as we use it to track if all
4490 // elements are undef.
4491 StartIdx[Input] = NewStartIdx;
4492 }
4493
4494 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4495 setValue(V: &I, NewN: DAG.getUNDEF(VT)); // Vectors are not used.
4496 return;
4497 }
4498 if (CanExtract) {
4499 // Extract appropriate subvector and generate a vector shuffle
4500 for (unsigned Input = 0; Input < 2; ++Input) {
4501 SDValue &Src = Input == 0 ? Src1 : Src2;
4502 if (StartIdx[Input] < 0)
4503 Src = DAG.getUNDEF(VT);
4504 else {
4505 Src = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Src,
4506 N2: DAG.getVectorIdxConstant(Val: StartIdx[Input], DL));
4507 }
4508 }
4509
4510 // Calculate new mask.
4511 SmallVector<int, 8> MappedOps(Mask);
4512 for (int &Idx : MappedOps) {
4513 if (Idx >= (int)SrcNumElts)
4514 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4515 else if (Idx >= 0)
4516 Idx -= StartIdx[0];
4517 }
4518
4519 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps));
4520 return;
4521 }
4522
4523 // We can't use either concat vectors or extract subvectors so fall back to
4524 // replacing the shuffle with extract and build vector.
4525 // to insert and build vector.
4526 EVT EltVT = VT.getVectorElementType();
4527 SmallVector<SDValue,8> Ops;
4528 for (int Idx : Mask) {
4529 SDValue Res;
4530
4531 if (Idx < 0) {
4532 Res = DAG.getUNDEF(VT: EltVT);
4533 } else {
4534 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4535 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4536
4537 Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Src,
4538 N2: DAG.getVectorIdxConstant(Val: Idx, DL));
4539 }
4540
4541 Ops.push_back(Elt: Res);
4542 }
4543
4544 setValue(V: &I, NewN: DAG.getBuildVector(VT, DL, Ops));
4545}
4546
4547void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4548 ArrayRef<unsigned> Indices = I.getIndices();
4549 const Value *Op0 = I.getOperand(i_nocapture: 0);
4550 const Value *Op1 = I.getOperand(i_nocapture: 1);
4551 Type *AggTy = I.getType();
4552 Type *ValTy = Op1->getType();
4553 bool IntoUndef = isa<UndefValue>(Val: Op0);
4554 bool FromUndef = isa<UndefValue>(Val: Op1);
4555
4556 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4557
4558 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4559 SmallVector<EVT, 4> AggValueVTs;
4560 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: AggTy, ValueVTs&: AggValueVTs);
4561 SmallVector<EVT, 4> ValValueVTs;
4562 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4563
4564 unsigned NumAggValues = AggValueVTs.size();
4565 unsigned NumValValues = ValValueVTs.size();
4566 SmallVector<SDValue, 4> Values(NumAggValues);
4567
4568 // Ignore an insertvalue that produces an empty object
4569 if (!NumAggValues) {
4570 setValue(V: &I, NewN: DAG.getUNDEF(VT: MVT(MVT::Other)));
4571 return;
4572 }
4573
4574 SDValue Agg = getValue(V: Op0);
4575 unsigned i = 0;
4576 // Copy the beginning value(s) from the original aggregate.
4577 for (; i != LinearIndex; ++i)
4578 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4579 SDValue(Agg.getNode(), Agg.getResNo() + i);
4580 // Copy values from the inserted value(s).
4581 if (NumValValues) {
4582 SDValue Val = getValue(V: Op1);
4583 for (; i != LinearIndex + NumValValues; ++i)
4584 Values[i] = FromUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4585 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4586 }
4587 // Copy remaining value(s) from the original aggregate.
4588 for (; i != NumAggValues; ++i)
4589 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4590 SDValue(Agg.getNode(), Agg.getResNo() + i);
4591
4592 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4593 VTList: DAG.getVTList(VTs: AggValueVTs), Ops: Values));
4594}
4595
4596void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4597 ArrayRef<unsigned> Indices = I.getIndices();
4598 const Value *Op0 = I.getOperand(i_nocapture: 0);
4599 Type *AggTy = Op0->getType();
4600 Type *ValTy = I.getType();
4601 bool OutOfUndef = isa<UndefValue>(Val: Op0);
4602
4603 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4604
4605 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4606 SmallVector<EVT, 4> ValValueVTs;
4607 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4608
4609 unsigned NumValValues = ValValueVTs.size();
4610
4611 // Ignore a extractvalue that produces an empty object
4612 if (!NumValValues) {
4613 setValue(V: &I, NewN: DAG.getUNDEF(VT: MVT(MVT::Other)));
4614 return;
4615 }
4616
4617 SmallVector<SDValue, 4> Values(NumValValues);
4618
4619 SDValue Agg = getValue(V: Op0);
4620 // Copy out the selected value(s).
4621 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4622 Values[i - LinearIndex] =
4623 OutOfUndef ?
4624 DAG.getUNDEF(VT: Agg.getNode()->getValueType(ResNo: Agg.getResNo() + i)) :
4625 SDValue(Agg.getNode(), Agg.getResNo() + i);
4626
4627 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4628 VTList: DAG.getVTList(VTs: ValValueVTs), Ops: Values));
4629}
4630
4631void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4632 Value *Op0 = I.getOperand(i: 0);
4633 // Note that the pointer operand may be a vector of pointers. Take the scalar
4634 // element which holds a pointer.
4635 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4636 SDValue N = getValue(V: Op0);
4637 SDLoc dl = getCurSDLoc();
4638 auto &TLI = DAG.getTargetLoweringInfo();
4639 GEPNoWrapFlags NW = cast<GEPOperator>(Val: I).getNoWrapFlags();
4640
4641 // For a vector GEP, keep the prefix scalar as long as possible, then
4642 // convert any scalars encountered after the first vector operand to vectors.
4643 bool IsVectorGEP = I.getType()->isVectorTy();
4644 ElementCount VectorElementCount =
4645 IsVectorGEP ? cast<VectorType>(Val: I.getType())->getElementCount()
4646 : ElementCount::getFixed(MinVal: 0);
4647
4648 for (gep_type_iterator GTI = gep_type_begin(GEP: &I), E = gep_type_end(GEP: &I);
4649 GTI != E; ++GTI) {
4650 const Value *Idx = GTI.getOperand();
4651 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4652 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
4653 if (Field) {
4654 // N = N + Offset
4655 uint64_t Offset =
4656 DAG.getDataLayout().getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
4657
4658 // In an inbounds GEP with an offset that is nonnegative even when
4659 // interpreted as signed, assume there is no unsigned overflow.
4660 SDNodeFlags Flags;
4661 if (NW.hasNoUnsignedWrap() ||
4662 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4663 Flags |= SDNodeFlags::NoUnsignedWrap;
4664 Flags.setInBounds(NW.isInBounds());
4665
4666 N = DAG.getMemBasePlusOffset(
4667 Base: N, Offset: DAG.getConstant(Val: Offset, DL: dl, VT: N.getValueType()), DL: dl, Flags);
4668 }
4669 } else {
4670 // IdxSize is the width of the arithmetic according to IR semantics.
4671 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4672 // (and fix up the result later).
4673 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4674 MVT IdxTy = MVT::getIntegerVT(BitWidth: IdxSize);
4675 TypeSize ElementSize =
4676 GTI.getSequentialElementStride(DL: DAG.getDataLayout());
4677 // We intentionally mask away the high bits here; ElementSize may not
4678 // fit in IdxTy.
4679 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4680 /*isSigned=*/false, /*implicitTrunc=*/true);
4681 bool ElementScalable = ElementSize.isScalable();
4682
4683 // If this is a scalar constant or a splat vector of constants,
4684 // handle it quickly.
4685 const auto *C = dyn_cast<Constant>(Val: Idx);
4686 if (C && isa<VectorType>(Val: C->getType()))
4687 C = C->getSplatValue();
4688
4689 const auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
4690 if (CI && CI->isZero())
4691 continue;
4692 if (CI && !ElementScalable) {
4693 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(width: IdxSize);
4694 LLVMContext &Context = *DAG.getContext();
4695 SDValue OffsVal;
4696 if (N.getValueType().isVector())
4697 OffsVal = DAG.getConstant(
4698 Val: Offs, DL: dl, VT: EVT::getVectorVT(Context, VT: IdxTy, EC: VectorElementCount));
4699 else
4700 OffsVal = DAG.getConstant(Val: Offs, DL: dl, VT: IdxTy);
4701
4702 // In an inbounds GEP with an offset that is nonnegative even when
4703 // interpreted as signed, assume there is no unsigned overflow.
4704 SDNodeFlags Flags;
4705 if (NW.hasNoUnsignedWrap() ||
4706 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4707 Flags.setNoUnsignedWrap(true);
4708 Flags.setInBounds(NW.isInBounds());
4709
4710 OffsVal = DAG.getSExtOrTrunc(Op: OffsVal, DL: dl, VT: N.getValueType());
4711
4712 N = DAG.getMemBasePlusOffset(Base: N, Offset: OffsVal, DL: dl, Flags);
4713 continue;
4714 }
4715
4716 // N = N + Idx * ElementMul;
4717 SDValue IdxN = getValue(V: Idx);
4718
4719 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
4720 if (N.getValueType().isVector()) {
4721 EVT VT = EVT::getVectorVT(Context&: *Context, VT: IdxN.getValueType(),
4722 EC: VectorElementCount);
4723 IdxN = DAG.getSplat(VT, DL: dl, Op: IdxN);
4724 } else {
4725 EVT VT =
4726 EVT::getVectorVT(Context&: *Context, VT: N.getValueType(), EC: VectorElementCount);
4727 N = DAG.getSplat(VT, DL: dl, Op: N);
4728 }
4729 }
4730
4731 // If the index is smaller or larger than intptr_t, truncate or extend
4732 // it.
4733 IdxN = DAG.getSExtOrTrunc(Op: IdxN, DL: dl, VT: N.getValueType());
4734
4735 SDNodeFlags ScaleFlags;
4736 // The multiplication of an index by the type size does not wrap the
4737 // pointer index type in a signed sense (mul nsw).
4738 ScaleFlags.setNoSignedWrap(NW.hasNoUnsignedSignedWrap());
4739
4740 // The multiplication of an index by the type size does not wrap the
4741 // pointer index type in an unsigned sense (mul nuw).
4742 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4743
4744 if (ElementScalable) {
4745 EVT VScaleTy = N.getValueType().getScalarType();
4746 SDValue VScale = DAG.getNode(
4747 Opcode: ISD::VSCALE, DL: dl, VT: VScaleTy,
4748 Operand: DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl, VT: VScaleTy));
4749 if (N.getValueType().isVector())
4750 VScale = DAG.getSplatVector(VT: N.getValueType(), DL: dl, Op: VScale);
4751 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: VScale,
4752 Flags: ScaleFlags);
4753 } else {
4754 // If this is a multiply by a power of two, turn it into a shl
4755 // immediately. This is a very common case.
4756 if (ElementMul != 1) {
4757 if (ElementMul.isPowerOf2()) {
4758 unsigned Amt = ElementMul.logBase2();
4759 IdxN = DAG.getNode(
4760 Opcode: ISD::SHL, DL: dl, VT: N.getValueType(), N1: IdxN,
4761 N2: DAG.getShiftAmountConstant(Val: Amt, VT: N.getValueType(), DL: dl),
4762 Flags: ScaleFlags);
4763 } else {
4764 SDValue Scale = DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl,
4765 VT: IdxN.getValueType());
4766 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: Scale,
4767 Flags: ScaleFlags);
4768 }
4769 }
4770 }
4771
4772 // The successive addition of the current address, truncated to the
4773 // pointer index type and interpreted as an unsigned number, and each
4774 // offset, also interpreted as an unsigned number, does not wrap the
4775 // pointer index type (add nuw).
4776 SDNodeFlags AddFlags;
4777 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4778 AddFlags.setInBounds(NW.isInBounds());
4779
4780 N = DAG.getMemBasePlusOffset(Base: N, Offset: IdxN, DL: dl, Flags: AddFlags);
4781 }
4782 }
4783
4784 if (IsVectorGEP && !N.getValueType().isVector()) {
4785 EVT VT = EVT::getVectorVT(Context&: *Context, VT: N.getValueType(), EC: VectorElementCount);
4786 N = DAG.getSplat(VT, DL: dl, Op: N);
4787 }
4788
4789 MVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout(), AS);
4790 MVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout(), AS);
4791 if (IsVectorGEP) {
4792 PtrTy = MVT::getVectorVT(VT: PtrTy, EC: VectorElementCount);
4793 PtrMemTy = MVT::getVectorVT(VT: PtrMemTy, EC: VectorElementCount);
4794 }
4795
4796 if (PtrMemTy != PtrTy && !cast<GEPOperator>(Val: I).isInBounds())
4797 N = DAG.getPtrExtendInReg(Op: N, DL: dl, VT: PtrMemTy);
4798
4799 setValue(V: &I, NewN: N);
4800}
4801
4802void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4803 // If this is a fixed sized alloca in the entry block of the function,
4804 // allocate it statically on the stack.
4805 if (FuncInfo.StaticAllocaMap.count(Val: &I))
4806 return; // getValue will auto-populate this.
4807
4808 SDLoc dl = getCurSDLoc();
4809 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4810 auto &DL = DAG.getDataLayout();
4811 TypeSize TySize = I.getAllocationBaseSize(DL);
4812 MaybeAlign Alignment = I.getAlign();
4813
4814 SDValue AllocSize = getValue(V: I.getArraySize());
4815
4816 EVT IntPtr = TLI.getPointerTy(DL, AS: I.getAddressSpace());
4817 if (AllocSize.getValueType() != IntPtr)
4818 AllocSize = DAG.getZExtOrTrunc(Op: AllocSize, DL: dl, VT: IntPtr);
4819
4820 AllocSize = DAG.getNode(
4821 Opcode: ISD::MUL, DL: dl, VT: IntPtr, N1: AllocSize,
4822 N2: DAG.getZExtOrTrunc(Op: DAG.getTypeSize(DL: dl, VT: MVT::i64, TS: TySize), DL: dl, VT: IntPtr));
4823
4824 // Handle alignment. If the requested alignment is less than or equal to
4825 // the stack alignment, ignore it. If the size is greater than or equal to
4826 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4827 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4828 if (*Alignment <= StackAlign)
4829 Alignment = std::nullopt;
4830
4831 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4832 // Round the size of the allocation up to the stack alignment size
4833 // by add SA-1 to the size. This doesn't overflow because we're computing
4834 // an address inside an alloca.
4835 AllocSize = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4836 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT: IntPtr),
4837 Flags: SDNodeFlags::NoUnsignedWrap);
4838
4839 // Mask out the low bits for alignment purposes.
4840 AllocSize = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4841 N2: DAG.getSignedConstant(Val: ~StackAlignMask, DL: dl, VT: IntPtr));
4842
4843 SDValue Ops[] = {
4844 getRoot(), AllocSize,
4845 DAG.getConstant(Val: Alignment ? Alignment->value() : 0, DL: dl, VT: IntPtr)};
4846 SDVTList VTs = DAG.getVTList(VT1: AllocSize.getValueType(), VT2: MVT::Other);
4847 SDValue DSA = DAG.getNode(Opcode: ISD::DYNAMIC_STACKALLOC, DL: dl, VTList: VTs, Ops);
4848 setValue(V: &I, NewN: DSA);
4849 DAG.setRoot(DSA.getValue(R: 1));
4850
4851 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4852}
4853
4854static const MDNode *getRangeMetadata(const Instruction &I) {
4855 return I.getMetadata(KindID: LLVMContext::MD_range);
4856}
4857
4858static std::optional<ConstantRange> getRange(const Instruction &I) {
4859 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4860 if (std::optional<ConstantRange> CR = CB->getRange())
4861 return CR;
4862 if (const MDNode *Range = getRangeMetadata(I))
4863 return getConstantRangeFromMetadata(RangeMD: *Range);
4864 return std::nullopt;
4865}
4866
4867static FPClassTest getNoFPClass(const Instruction &I) {
4868 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4869 return CB->getRetNoFPClass();
4870 return fcNone;
4871}
4872
4873void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4874 if (I.isAtomic())
4875 return visitAtomicLoad(I);
4876
4877 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4878 const Value *SV = I.getOperand(i_nocapture: 0);
4879 if (TLI.supportSwiftError()) {
4880 // Swifterror values can come from either a function parameter with
4881 // swifterror attribute or an alloca with swifterror attribute.
4882 if (const Argument *Arg = dyn_cast<Argument>(Val: SV)) {
4883 if (Arg->hasSwiftErrorAttr())
4884 return visitLoadFromSwiftError(I);
4885 }
4886
4887 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: SV)) {
4888 if (Alloca->isSwiftError())
4889 return visitLoadFromSwiftError(I);
4890 }
4891 }
4892
4893 SDValue Ptr = getValue(V: SV);
4894
4895 Type *Ty = I.getType();
4896 SmallVector<EVT, 4> ValueVTs, MemVTs;
4897 SmallVector<TypeSize, 4> Offsets;
4898 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty, ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
4899 unsigned NumValues = ValueVTs.size();
4900 if (NumValues == 0)
4901 return;
4902
4903 Align Alignment = I.getAlign();
4904 AAMDNodes AAInfo = I.getAAMetadata();
4905 const MDNode *Ranges = getRangeMetadata(I);
4906 const MDNode *MemCacheHint = getMemCacheHintMetadata(I);
4907 bool isVolatile = I.isVolatile();
4908 MachineMemOperand::Flags MMOFlags =
4909 TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
4910
4911 SDValue Root;
4912 bool ConstantMemory = false;
4913 if (isVolatile)
4914 // Serialize volatile loads with other side effects.
4915 Root = getRoot();
4916 else if (NumValues > MaxParallelChains)
4917 Root = getMemoryRoot();
4918 else if (BatchAA &&
4919 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(
4920 SV,
4921 LocationSize::precise(Value: DAG.getDataLayout().getTypeStoreSize(Ty)),
4922 AAInfo))) {
4923 // Do not serialize (non-volatile) loads of constant memory with anything.
4924 Root = DAG.getEntryNode();
4925 ConstantMemory = true;
4926 MMOFlags |= MachineMemOperand::MOInvariant;
4927 } else {
4928 // Do not serialize non-volatile loads against each other.
4929 Root = DAG.getRoot();
4930 }
4931
4932 SDLoc dl = getCurSDLoc();
4933
4934 if (isVolatile)
4935 Root = TLI.prepareVolatileOrAtomicLoad(Chain: Root, DL: dl, DAG);
4936
4937 SmallVector<SDValue, 4> Values(NumValues);
4938 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
4939
4940 unsigned ChainI = 0;
4941 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4942 // Serializing loads here may result in excessive register pressure, and
4943 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4944 // could recover a bit by hoisting nodes upward in the chain by recognizing
4945 // they are side-effect free or do not alias. The optimizer should really
4946 // avoid this case by converting large object/array copies to llvm.memcpy
4947 // (MaxParallelChains should always remain as failsafe).
4948 if (ChainI == MaxParallelChains) {
4949 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4950 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4951 Ops: ArrayRef(Chains.data(), ChainI));
4952 Root = Chain;
4953 ChainI = 0;
4954 }
4955
4956 // TODO: MachinePointerInfo only supports a fixed length offset.
4957 MachinePointerInfo PtrInfo =
4958 !Offsets[i].isScalable() || Offsets[i].isZero()
4959 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4960 : MachinePointerInfo();
4961
4962 SDValue A = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
4963 SDValue L =
4964 DAG.getLoad(VT: MemVTs[i], dl, Chain: Root, Ptr: A, PtrInfo, Alignment, MMOFlags,
4965 Metadata: MMOMetadata(AAInfo, Ranges, MemCacheHint));
4966 Chains[ChainI] = L.getValue(R: 1);
4967
4968 if (MemVTs[i] != ValueVTs[i])
4969 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT: ValueVTs[i]);
4970
4971 if (MDNode *NoFPClassMD = I.getMetadata(KindID: LLVMContext::MD_nofpclass)) {
4972 uint64_t FPTestInt =
4973 cast<ConstantInt>(
4974 Val: cast<ConstantAsMetadata>(Val: NoFPClassMD->getOperand(I: 0))->getValue())
4975 ->getZExtValue();
4976 if (FPTestInt != fcNone) {
4977 SDValue FPTestConst =
4978 DAG.getTargetConstant(Val: FPTestInt, DL: SDLoc(), VT: MVT::i32);
4979 L = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: L.getValueType(), N1: L,
4980 N2: FPTestConst);
4981 }
4982 }
4983 Values[i] = L;
4984 }
4985
4986 if (!ConstantMemory) {
4987 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4988 Ops: ArrayRef(Chains.data(), ChainI));
4989 if (isVolatile)
4990 DAG.setRoot(Chain);
4991 else
4992 PendingLoads.push_back(Elt: Chain);
4993 }
4994
4995 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
4996 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
4997}
4998
4999void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
5000 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5001 "call visitStoreToSwiftError when backend supports swifterror");
5002
5003 SmallVector<EVT, 4> ValueVTs;
5004 SmallVector<uint64_t, 4> Offsets;
5005 const Value *SrcV = I.getOperand(i_nocapture: 0);
5006 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
5007 Ty: SrcV->getType(), ValueVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
5008 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5009 "expect a single EVT for swifterror");
5010
5011 SDValue Src = getValue(V: SrcV);
5012 // Create a virtual register, then update the virtual register.
5013 Register VReg =
5014 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
5015 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
5016 // Chain can be getRoot or getControlRoot.
5017 SDValue CopyNode = DAG.getCopyToReg(Chain: getRoot(), dl: getCurSDLoc(), Reg: VReg,
5018 N: SDValue(Src.getNode(), Src.getResNo()));
5019 DAG.setRoot(CopyNode);
5020}
5021
5022void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
5023 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5024 "call visitLoadFromSwiftError when backend supports swifterror");
5025
5026 assert(!I.isVolatile() &&
5027 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
5028 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
5029 "Support volatile, non temporal, invariant for load_from_swift_error");
5030
5031 const Value *SV = I.getOperand(i_nocapture: 0);
5032 Type *Ty = I.getType();
5033 assert(
5034 (!BatchAA ||
5035 !BatchAA->pointsToConstantMemory(MemoryLocation(
5036 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
5037 I.getAAMetadata()))) &&
5038 "load_from_swift_error should not be constant memory");
5039
5040 SmallVector<EVT, 4> ValueVTs;
5041 SmallVector<uint64_t, 4> Offsets;
5042 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty,
5043 ValueVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
5044 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5045 "expect a single EVT for swifterror");
5046
5047 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
5048 SDValue L = DAG.getCopyFromReg(
5049 Chain: getRoot(), dl: getCurSDLoc(),
5050 Reg: SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), VT: ValueVTs[0]);
5051
5052 setValue(V: &I, NewN: L);
5053}
5054
5055void SelectionDAGBuilder::visitStore(const StoreInst &I) {
5056 if (I.isAtomic())
5057 return visitAtomicStore(I);
5058
5059 const Value *SrcV = I.getOperand(i_nocapture: 0);
5060 const Value *PtrV = I.getOperand(i_nocapture: 1);
5061
5062 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5063 if (TLI.supportSwiftError()) {
5064 // Swifterror values can come from either a function parameter with
5065 // swifterror attribute or an alloca with swifterror attribute.
5066 if (const Argument *Arg = dyn_cast<Argument>(Val: PtrV)) {
5067 if (Arg->hasSwiftErrorAttr())
5068 return visitStoreToSwiftError(I);
5069 }
5070
5071 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: PtrV)) {
5072 if (Alloca->isSwiftError())
5073 return visitStoreToSwiftError(I);
5074 }
5075 }
5076
5077 SmallVector<EVT, 4> ValueVTs, MemVTs;
5078 SmallVector<TypeSize, 4> Offsets;
5079 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
5080 Ty: SrcV->getType(), ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
5081 unsigned NumValues = ValueVTs.size();
5082 if (NumValues == 0)
5083 return;
5084
5085 // Get the lowered operands. Note that we do this after
5086 // checking if NumResults is zero, because with zero results
5087 // the operands won't have values in the map.
5088 SDValue Src = getValue(V: SrcV);
5089 SDValue Ptr = getValue(V: PtrV);
5090
5091 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
5092 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
5093 SDLoc dl = getCurSDLoc();
5094 Align Alignment = I.getAlign();
5095 AAMDNodes AAInfo = I.getAAMetadata();
5096 const MDNode *MemCacheHint =
5097 getMemCacheHintMetadata(I, OperandNo: I.getPointerOperandIndex());
5098
5099 auto MMOFlags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
5100
5101 unsigned ChainI = 0;
5102 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
5103 // See visitLoad comments.
5104 if (ChainI == MaxParallelChains) {
5105 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
5106 Ops: ArrayRef(Chains.data(), ChainI));
5107 Root = Chain;
5108 ChainI = 0;
5109 }
5110
5111 // TODO: MachinePointerInfo only supports a fixed length offset.
5112 MachinePointerInfo PtrInfo =
5113 !Offsets[i].isScalable() || Offsets[i].isZero()
5114 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
5115 : MachinePointerInfo();
5116
5117 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
5118 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
5119 if (MemVTs[i] != ValueVTs[i])
5120 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVTs[i]);
5121 SDValue St =
5122 DAG.getStore(Chain: Root, dl, Val, Ptr: Add, PtrInfo, Alignment, MMOFlags,
5123 Metadata: MMOMetadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
5124 Chains[ChainI] = St;
5125 }
5126
5127 SDValue StoreNode = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
5128 Ops: ArrayRef(Chains.data(), ChainI));
5129 setValue(V: &I, NewN: StoreNode);
5130 DAG.setRoot(StoreNode);
5131}
5132
5133void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
5134 bool IsCompressing) {
5135 SDLoc sdl = getCurSDLoc();
5136
5137 Value *Src0Operand = I.getArgOperand(i: 0);
5138 Value *PtrOperand = I.getArgOperand(i: 1);
5139 Value *MaskOperand = I.getArgOperand(i: 2);
5140 Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
5141
5142 SDValue Ptr = getValue(V: PtrOperand);
5143 SDValue Src0 = getValue(V: Src0Operand);
5144 SDValue Mask = getValue(V: MaskOperand);
5145 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
5146
5147 EVT VT = Src0.getValueType();
5148
5149 const auto &TLI = DAG.getTargetLoweringInfo();
5150
5151 auto MMOFlags = MachineMemOperand::MOStore;
5152 MMOFlags |= TLI.getTargetMMOFlags(I);
5153 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5154 MMOFlags |= MachineMemOperand::MONonTemporal;
5155
5156 const MDNode *MemCacheHint = getMemCacheHintMetadata(I, /*OperandNo=*/1);
5157
5158 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5159 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5160 Size: LocationSize::upperBound(Value: VT.getStoreSize()), BaseAlignment: Alignment,
5161 Metadata: MMOMetadata(I.getAAMetadata(), /*Ranges=*/nullptr, MemCacheHint));
5162
5163 SDValue StoreNode =
5164 !IsCompressing && TTI->hasConditionalLoadStoreForType(
5165 Ty: I.getArgOperand(i: 0)->getType(), /*IsStore=*/true)
5166 ? TLI.visitMaskedStore(DAG, DL: sdl, Chain: getMemoryRoot(), MMO, Ptr, Val: Src0,
5167 Mask)
5168 : DAG.getMaskedStore(Chain: getMemoryRoot(), dl: sdl, Val: Src0, Base: Ptr, Offset, Mask,
5169 MemVT: VT, MMO, AM: ISD::UNINDEXED, /*Truncating=*/IsTruncating: false,
5170 IsCompressing);
5171 DAG.setRoot(StoreNode);
5172 setValue(V: &I, NewN: StoreNode);
5173}
5174
5175// Get a uniform base for the Gather/Scatter intrinsic.
5176// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
5177// We try to represent it as a base pointer + vector of indices.
5178// Usually, the vector of pointers comes from a 'getelementptr' instruction.
5179// The first operand of the GEP may be a single pointer or a vector of pointers
5180// Example:
5181// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
5182// or
5183// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
5184// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
5185//
5186// When the first GEP operand is a single pointer - it is the uniform base we
5187// are looking for. If first operand of the GEP is a splat vector - we
5188// extract the splat value and use it as a uniform base.
5189// In all other cases the function returns 'false'.
5190static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
5191 SDValue &Scale, SelectionDAGBuilder *SDB,
5192 const BasicBlock *CurBB, uint64_t ElemSize) {
5193 SelectionDAG& DAG = SDB->DAG;
5194 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5195 const DataLayout &DL = DAG.getDataLayout();
5196
5197 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
5198
5199 // Handle splat constant pointer.
5200 if (auto *C = dyn_cast<Constant>(Val: Ptr)) {
5201 C = C->getSplatValue();
5202 if (!C)
5203 return false;
5204
5205 Base = SDB->getValue(V: C);
5206
5207 ElementCount NumElts = cast<VectorType>(Val: Ptr->getType())->getElementCount();
5208 EVT VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TLI.getPointerTy(DL), EC: NumElts);
5209 Index = DAG.getConstant(Val: 0, DL: SDB->getCurSDLoc(), VT);
5210 Scale = DAG.getTargetConstant(Val: 1, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
5211 return true;
5212 }
5213
5214 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
5215 if (!GEP || GEP->getParent() != CurBB)
5216 return false;
5217
5218 if (GEP->getNumOperands() != 2)
5219 return false;
5220
5221 const Value *BasePtr = GEP->getPointerOperand();
5222 const Value *IndexVal = GEP->getOperand(i_nocapture: GEP->getNumOperands() - 1);
5223
5224 // Make sure the base is scalar and the index is a vector.
5225 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
5226 return false;
5227
5228 TypeSize ScaleVal = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
5229 if (ScaleVal.isScalable())
5230 return false;
5231
5232 // Target may not support the required addressing mode.
5233 if (ScaleVal != 1 &&
5234 !TLI.isLegalScaleForGatherScatter(Scale: ScaleVal.getFixedValue(), ElemSize))
5235 return false;
5236
5237 Base = SDB->getValue(V: BasePtr);
5238 Index = SDB->getValue(V: IndexVal);
5239
5240 Scale =
5241 DAG.getTargetConstant(Val: ScaleVal, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
5242 return true;
5243}
5244
5245void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
5246 SDLoc sdl = getCurSDLoc();
5247
5248 // llvm.masked.scatter.*(Src0, Ptrs, Mask)
5249 const Value *Ptr = I.getArgOperand(i: 1);
5250 SDValue Src0 = getValue(V: I.getArgOperand(i: 0));
5251 SDValue Mask = getValue(V: I.getArgOperand(i: 2));
5252 EVT VT = Src0.getValueType();
5253 Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
5254 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5255
5256 SDValue Base;
5257 SDValue Index;
5258 SDValue Scale;
5259 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
5260 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5261
5262 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5263 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5264 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOStore,
5265 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, Metadata: I.getAAMetadata());
5266 if (!UniformBase) {
5267 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5268 Index = getValue(V: Ptr);
5269 Scale =
5270 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5271 }
5272
5273 EVT IdxVT = Index.getValueType();
5274 EVT EltTy = IdxVT.getVectorElementType();
5275 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5276 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
5277 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5278 }
5279
5280 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
5281 SDValue Scatter = DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: VT, dl: sdl,
5282 Ops, MMO, IndexType: ISD::SIGNED_SCALED, IsTruncating: false);
5283 DAG.setRoot(Scatter);
5284 setValue(V: &I, NewN: Scatter);
5285}
5286
5287void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
5288 SDLoc sdl = getCurSDLoc();
5289
5290 Value *PtrOperand = I.getArgOperand(i: 0);
5291 Value *MaskOperand = I.getArgOperand(i: 1);
5292 Value *Src0Operand = I.getArgOperand(i: 2);
5293 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5294
5295 SDValue Ptr = getValue(V: PtrOperand);
5296 SDValue Src0 = getValue(V: Src0Operand);
5297 SDValue Mask = getValue(V: MaskOperand);
5298 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
5299
5300 EVT VT = Src0.getValueType();
5301 AAMDNodes AAInfo = I.getAAMetadata();
5302 const MDNode *Ranges = getRangeMetadata(I);
5303 const MDNode *MemCacheHint = getMemCacheHintMetadata(I, /*OperandNo=*/0);
5304
5305 // Do not serialize masked loads of constant memory with anything.
5306 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
5307 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
5308
5309 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5310
5311 const auto &TLI = DAG.getTargetLoweringInfo();
5312
5313 auto MMOFlags = MachineMemOperand::MOLoad;
5314 MMOFlags |= TLI.getTargetMMOFlags(I);
5315 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5316 MMOFlags |= MachineMemOperand::MONonTemporal;
5317 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
5318 MMOFlags |= MachineMemOperand::MOInvariant;
5319
5320 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5321 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5322 Size: LocationSize::upperBound(Value: VT.getStoreSize()), BaseAlignment: Alignment,
5323 Metadata: MMOMetadata(AAInfo, Ranges, MemCacheHint));
5324
5325 // The Load/Res may point to different values and both of them are output
5326 // variables.
5327 SDValue Load;
5328 SDValue Res;
5329 if (!IsExpanding &&
5330 TTI->hasConditionalLoadStoreForType(Ty: Src0Operand->getType(),
5331 /*IsStore=*/false))
5332 Res = TLI.visitMaskedLoad(DAG, DL: sdl, Chain: InChain, MMO, NewLoad&: Load, Ptr, PassThru: Src0, Mask);
5333 else
5334 Res = Load =
5335 DAG.getMaskedLoad(VT, dl: sdl, Chain: InChain, Base: Ptr, Offset, Mask, Src0, MemVT: VT, MMO,
5336 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5337 if (AddToChain)
5338 PendingLoads.push_back(Elt: Load.getValue(R: 1));
5339 setValue(V: &I, NewN: Res);
5340}
5341
5342void SelectionDAGBuilder::visitSpeculativeLoad(const CallInst &I) {
5343 SDLoc sdl = getCurSDLoc();
5344 Value *PtrOperand = I.getArgOperand(i: 0);
5345 // The remaining arguments (num_accessible_bytes or oracle function + args)
5346 // are IR-level semantics only; they are not needed at codegen.
5347 SDValue Ptr = getValue(V: PtrOperand);
5348
5349 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5350 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5351 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5352 AAMDNodes AAInfo = I.getAAMetadata();
5353
5354 SDValue InChain = DAG.getRoot();
5355
5356 // Use MOLoad but NOT MODereferenceable - the memory may not be
5357 // fully dereferenceable.
5358 auto MMOFlags = MachineMemOperand::MOLoad;
5359 MMOFlags |= TLI.getTargetMMOFlags(I);
5360 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5361 MMOFlags |= MachineMemOperand::MONonTemporal;
5362 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
5363 MMOFlags |= MachineMemOperand::MOInvariant;
5364
5365 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5366 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5367 Size: LocationSize::precise(Value: VT.getStoreSize()), BaseAlignment: Alignment, Metadata: AAInfo);
5368
5369 SDValue Load = DAG.getLoad(VT, dl: sdl, Chain: InChain, Ptr, MMO);
5370 PendingLoads.push_back(Elt: Load.getValue(R: 1));
5371 setValue(V: &I, NewN: Load);
5372}
5373
5374void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5375 SDLoc sdl = getCurSDLoc();
5376
5377 // @llvm.masked.gather.*(Ptrs, Mask, Src0)
5378 const Value *Ptr = I.getArgOperand(i: 0);
5379 SDValue Src0 = getValue(V: I.getArgOperand(i: 2));
5380 SDValue Mask = getValue(V: I.getArgOperand(i: 1));
5381
5382 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5383 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5384 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5385
5386 const MDNode *Ranges = getRangeMetadata(I);
5387
5388 SDValue Root = DAG.getRoot();
5389 SDValue Base;
5390 SDValue Index;
5391 SDValue Scale;
5392 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
5393 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5394 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5395 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5396 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOLoad,
5397 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
5398 Metadata: MMOMetadata(I.getAAMetadata(), Ranges));
5399
5400 if (!UniformBase) {
5401 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5402 Index = getValue(V: Ptr);
5403 Scale =
5404 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5405 }
5406
5407 EVT IdxVT = Index.getValueType();
5408 EVT EltTy = IdxVT.getVectorElementType();
5409 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5410 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
5411 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5412 }
5413
5414 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5415 SDValue Gather =
5416 DAG.getMaskedGather(VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), MemVT: VT, dl: sdl, Ops, MMO,
5417 IndexType: ISD::SIGNED_SCALED, ExtTy: ISD::NON_EXTLOAD);
5418
5419 PendingLoads.push_back(Elt: Gather.getValue(R: 1));
5420 setValue(V: &I, NewN: Gather);
5421}
5422
5423void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5424 SDLoc dl = getCurSDLoc();
5425 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5426 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5427 SyncScope::ID SSID = I.getSyncScopeID();
5428
5429 SDValue InChain = getRoot();
5430
5431 MVT MemVT = getValue(V: I.getCompareOperand()).getSimpleValueType();
5432 SDVTList VTs = DAG.getVTList(VT1: MemVT, VT2: MVT::i1, VT3: MVT::Other);
5433
5434 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5435 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5436
5437 MachineFunction &MF = DAG.getMachineFunction();
5438 const MDNode *MemCacheHint = getMemCacheHintMetadata(I);
5439 MachineMemOperand *MMO = MF.getMachineMemOperand(
5440 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5441 BaseAlignment: I.getAlign(), Metadata: MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5442 SSID, Ordering: SuccessOrdering, FailureOrdering);
5443
5444 SDValue L = DAG.getAtomicCmpSwap(Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
5445 dl, MemVT, VTs, Chain: InChain,
5446 Ptr: getValue(V: I.getPointerOperand()),
5447 Cmp: getValue(V: I.getCompareOperand()),
5448 Swp: getValue(V: I.getNewValOperand()), MMO);
5449
5450 SDValue OutChain = L.getValue(R: 2);
5451
5452 setValue(V: &I, NewN: L);
5453 DAG.setRoot(OutChain);
5454}
5455
5456void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5457 SDLoc dl = getCurSDLoc();
5458 ISD::NodeType NT;
5459 switch (I.getOperation()) {
5460 default: llvm_unreachable("Unknown atomicrmw operation");
5461 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
5462 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
5463 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
5464 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
5465 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
5466 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
5467 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
5468 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
5469 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
5470 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
5471 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
5472 case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
5473 case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
5474 case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
5475 case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
5476 case AtomicRMWInst::FMaximum:
5477 NT = ISD::ATOMIC_LOAD_FMAXIMUM;
5478 break;
5479 case AtomicRMWInst::FMinimum:
5480 NT = ISD::ATOMIC_LOAD_FMINIMUM;
5481 break;
5482 case AtomicRMWInst::FMaximumNum:
5483 NT = ISD::ATOMIC_LOAD_FMAXIMUMNUM;
5484 break;
5485 case AtomicRMWInst::FMinimumNum:
5486 NT = ISD::ATOMIC_LOAD_FMINIMUMNUM;
5487 break;
5488 case AtomicRMWInst::UIncWrap:
5489 NT = ISD::ATOMIC_LOAD_UINC_WRAP;
5490 break;
5491 case AtomicRMWInst::UDecWrap:
5492 NT = ISD::ATOMIC_LOAD_UDEC_WRAP;
5493 break;
5494 case AtomicRMWInst::USubCond:
5495 NT = ISD::ATOMIC_LOAD_USUB_COND;
5496 break;
5497 case AtomicRMWInst::USubSat:
5498 NT = ISD::ATOMIC_LOAD_USUB_SAT;
5499 break;
5500 }
5501 AtomicOrdering Ordering = I.getOrdering();
5502 SyncScope::ID SSID = I.getSyncScopeID();
5503
5504 SDValue InChain = getRoot();
5505
5506 auto MemVT = getValue(V: I.getValOperand()).getSimpleValueType();
5507 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5508 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5509
5510 MachineFunction &MF = DAG.getMachineFunction();
5511 const MDNode *MemCacheHint = getMemCacheHintMetadata(I);
5512 MachineMemOperand *MMO = MF.getMachineMemOperand(
5513 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5514 BaseAlignment: I.getAlign(), Metadata: MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5515 SSID, Ordering);
5516
5517 SDValue L =
5518 DAG.getAtomic(Opcode: NT, dl, MemVT, Chain: InChain,
5519 Ptr: getValue(V: I.getPointerOperand()), Val: getValue(V: I.getValOperand()),
5520 MMO);
5521
5522 SDValue OutChain = L.getValue(R: 1);
5523
5524 setValue(V: &I, NewN: L);
5525 DAG.setRoot(OutChain);
5526}
5527
5528void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5529 SDLoc dl = getCurSDLoc();
5530 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5531 SDValue Ops[3];
5532 Ops[0] = getRoot();
5533 Ops[1] = DAG.getTargetConstant(Val: (unsigned)I.getOrdering(), DL: dl,
5534 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5535 Ops[2] = DAG.getTargetConstant(Val: I.getSyncScopeID(), DL: dl,
5536 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5537 SDValue N = DAG.getNode(Opcode: ISD::ATOMIC_FENCE, DL: dl, VT: MVT::Other, Ops);
5538 setValue(V: &I, NewN: N);
5539 DAG.setRoot(N);
5540}
5541
5542void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5543 SDLoc dl = getCurSDLoc();
5544 AtomicOrdering Order = I.getOrdering();
5545 SyncScope::ID SSID = I.getSyncScopeID();
5546
5547 SDValue InChain = getRoot();
5548
5549 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5550 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5551 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5552
5553 if (!TLI.isAtomicAlignmentSupported(Alignment: I.getAlign(), SizeInBytes: MemVT.getSizeInBits() / 8))
5554 report_fatal_error(reason: "Cannot generate unaligned atomic load");
5555
5556 auto Flags = TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
5557
5558 const MDNode *Ranges = getRangeMetadata(I);
5559 const MDNode *MemCacheHint = getMemCacheHintMetadata(I);
5560 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5561 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5562 BaseAlignment: I.getAlign(), Metadata: MMOMetadata(AAMDNodes(), Ranges, MemCacheHint), SSID,
5563 Ordering: Order);
5564
5565 InChain = TLI.prepareVolatileOrAtomicLoad(Chain: InChain, DL: dl, DAG);
5566
5567 SDValue Ptr = getValue(V: I.getPointerOperand());
5568 SDValue L =
5569 DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT, VT: MemVT, Chain: InChain, Ptr, MMO);
5570
5571 SDValue OutChain = L.getValue(R: 1);
5572 if (MemVT != VT)
5573 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT);
5574
5575 setValue(V: &I, NewN: L);
5576 DAG.setRoot(OutChain);
5577}
5578
5579void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5580 SDLoc dl = getCurSDLoc();
5581
5582 AtomicOrdering Ordering = I.getOrdering();
5583 SyncScope::ID SSID = I.getSyncScopeID();
5584
5585 SDValue InChain = getRoot();
5586
5587 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5588 EVT MemVT =
5589 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getValueOperand()->getType());
5590
5591 if (!TLI.isAtomicAlignmentSupported(Alignment: I.getAlign(), SizeInBytes: MemVT.getSizeInBits() / 8))
5592 report_fatal_error(reason: "Cannot generate unaligned atomic store");
5593
5594 auto Flags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
5595
5596 MachineFunction &MF = DAG.getMachineFunction();
5597 const MDNode *MemCacheHint =
5598 getMemCacheHintMetadata(I, OperandNo: I.getPointerOperandIndex());
5599 MachineMemOperand *MMO = MF.getMachineMemOperand(
5600 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5601 BaseAlignment: I.getAlign(), Metadata: MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5602 SSID, Ordering);
5603
5604 SDValue Val = getValue(V: I.getValueOperand());
5605 if (Val.getValueType() != MemVT)
5606 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVT);
5607 SDValue Ptr = getValue(V: I.getPointerOperand());
5608
5609 SDValue OutChain =
5610 DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT, Chain: InChain, Ptr: Val, Val: Ptr, MMO);
5611
5612 setValue(V: &I, NewN: OutChain);
5613 DAG.setRoot(OutChain);
5614}
5615
5616/// Check if this intrinsic call depends on the chain (1st return value)
5617/// and if it only *loads* memory.
5618/// Ignore the callsite's attributes. A specific call site may be marked with
5619/// readnone, but the lowering code will expect the chain based on the
5620/// definition.
5621std::pair<bool, bool>
5622SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {
5623 const Function *F = I.getCalledFunction();
5624 bool HasChain = !F->doesNotAccessMemory();
5625 bool OnlyLoad =
5626 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5627
5628 return {HasChain, OnlyLoad};
5629}
5630
5631SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
5632 const CallBase &I, bool HasChain, bool OnlyLoad,
5633 TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
5634 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5635
5636 // Build the operand list.
5637 SmallVector<SDValue, 8> Ops;
5638 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5639 if (OnlyLoad) {
5640 // We don't need to serialize loads against other loads.
5641 Ops.push_back(Elt: DAG.getRoot());
5642 } else {
5643 Ops.push_back(Elt: getRoot());
5644 }
5645 }
5646
5647 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5648 if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||
5649 TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)
5650 Ops.push_back(Elt: DAG.getTargetConstant(Val: I.getIntrinsicID(), DL: getCurSDLoc(),
5651 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
5652
5653 // Add all operands of the call to the operand list.
5654 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5655 const Value *Arg = I.getArgOperand(i);
5656 if (!I.paramHasAttr(ArgNo: i, Kind: Attribute::ImmArg)) {
5657 Ops.push_back(Elt: getValue(V: Arg));
5658 continue;
5659 }
5660
5661 // Use TargetConstant instead of a regular constant for immarg.
5662 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: Arg->getType(), AllowUnknown: true);
5663 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg)) {
5664 assert(CI->getBitWidth() <= 64 &&
5665 "large intrinsic immediates not handled");
5666 Ops.push_back(Elt: DAG.getTargetConstant(Val: *CI, DL: SDLoc(), VT));
5667 } else {
5668 Ops.push_back(
5669 Elt: DAG.getTargetConstantFP(Val: *cast<ConstantFP>(Val: Arg), DL: SDLoc(), VT));
5670 }
5671 }
5672
5673 if (std::optional<OperandBundleUse> Bundle =
5674 I.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
5675 auto *Sym = Bundle->Inputs[0].get();
5676 SDValue SDSym = getValue(V: Sym);
5677 SDSym = DAG.getDeactivationSymbol(GV: cast<GlobalValue>(Val: Sym));
5678 Ops.push_back(Elt: SDSym);
5679 }
5680
5681 if (std::optional<OperandBundleUse> Bundle =
5682 I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
5683 Value *Token = Bundle->Inputs[0].get();
5684 SDValue ConvControlToken = getValue(V: Token);
5685 assert(Ops.back().getValueType() != MVT::Glue &&
5686 "Did not expect another glue node here.");
5687 ConvControlToken =
5688 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: {}, VT: MVT::Glue, Operand: ConvControlToken);
5689 Ops.push_back(Elt: ConvControlToken);
5690 }
5691
5692 return Ops;
5693}
5694
5695SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
5696 bool HasChain) {
5697 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5698
5699 SmallVector<EVT, 4> ValueVTs;
5700 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
5701
5702 if (HasChain)
5703 ValueVTs.push_back(Elt: MVT::Other);
5704
5705 return DAG.getVTList(VTs: ValueVTs);
5706}
5707
5708/// Get an INTRINSIC node for a target intrinsic which does not touch memory.
5709SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5710 const Type &IntrinsicVT, bool HasChain, ArrayRef<SDValue> Ops,
5711 const SDVTList &VTs) {
5712 if (!HasChain)
5713 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5714 if (!IntrinsicVT.isVoidTy())
5715 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5716 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops);
5717}
5718
5719/// Set root, convert return type if necessary and check alignment.
5720SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
5721 bool HasChain,
5722 bool OnlyLoad,
5723 SDValue Result) {
5724 if (HasChain) {
5725 SDValue Chain = Result.getValue(R: Result.getNode()->getNumValues() - 1);
5726 if (OnlyLoad)
5727 PendingLoads.push_back(Elt: Chain);
5728 else
5729 DAG.setRoot(Chain);
5730 }
5731
5732 if (I.getType()->isVoidTy())
5733 return Result;
5734
5735 if (MaybeAlign Alignment = I.getRetAlign(); InsertAssertAlign && Alignment) {
5736 // Insert `assertalign` node if there's an alignment.
5737 Result = DAG.getAssertAlign(DL: getCurSDLoc(), V: Result, A: Alignment.valueOrOne());
5738 } else if (!isa<VectorType>(Val: I.getType())) {
5739 Result = lowerRangeToAssertZExt(DAG, I, Op: Result);
5740 }
5741
5742 return Result;
5743}
5744
5745/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5746/// node.
5747void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5748 unsigned Intrinsic) {
5749 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
5750 Intrinsic::ID IntrinsicID = static_cast<Intrinsic::ID>(Intrinsic);
5751
5752 if (!DAG.getMachineFunction().getSubtarget().isIntrinsicSupported(
5753 IntrinsicID: Intrinsic)) {
5754 SDLoc DL = getCurSDLoc();
5755 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupportedTargetIntrinsic(
5756 *I.getFunction(), IntrinsicID, DL.getDebugLoc()));
5757
5758 // The intrinsic is not available on this subtarget. Preserve the chain for
5759 // side-effecting intrinsics and lower any result to poison so that
5760 // compilation can continue and collect further diagnostics.
5761 if (HasChain && !OnlyLoad)
5762 DAG.setRoot(getRoot());
5763
5764 setValueToPoison(V: &I, dl: DL);
5765 return;
5766 }
5767
5768 // Infos is set by getTgtMemIntrinsic.
5769 SmallVector<TargetLowering::IntrinsicInfo> Infos;
5770 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5771 TLI.getTgtMemIntrinsic(Infos, I, MF&: DAG.getMachineFunction(), Intrinsic);
5772 // Use the first (primary) info determines the node opcode.
5773 TargetLowering::IntrinsicInfo *Info = !Infos.empty() ? &Infos[0] : nullptr;
5774
5775 SmallVector<SDValue, 8> Ops =
5776 getTargetIntrinsicOperands(I, HasChain, OnlyLoad, TgtMemIntrinsicInfo: Info);
5777 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
5778
5779 // Propagate fast-math-flags from IR to node(s).
5780 SDNodeFlags Flags;
5781 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
5782 Flags.copyFMF(FPMO: *FPMO);
5783 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5784
5785 // Create the node.
5786 SDValue Result;
5787
5788 // In some cases, custom collection of operands from CallInst I may be needed.
5789 TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);
5790 if (!Infos.empty()) {
5791 // This is target intrinsic that touches memory
5792 // Create MachineMemOperands for each memory access described by the target.
5793 MachineFunction &MF = DAG.getMachineFunction();
5794 SmallVector<MachineMemOperand *> MMOs;
5795 for (const auto &Info : Infos) {
5796 // TODO: We currently just fallback to address space 0 if
5797 // getTgtMemIntrinsic didn't yield anything useful.
5798 MachinePointerInfo MPI;
5799 if (Info.ptrVal)
5800 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5801 else if (Info.fallbackAddressSpace)
5802 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5803 EVT MemVT = Info.memVT;
5804 LocationSize Size = LocationSize::precise(Value: Info.size);
5805 if (Size.hasValue() && !Size.getValue())
5806 Size = LocationSize::precise(Value: MemVT.getStoreSize());
5807 Align Alignment = Info.align.value_or(u: DAG.getEVTAlign(MemoryVT: MemVT));
5808 MachineMemOperand *MMO = MF.getMachineMemOperand(
5809 PtrInfo: MPI, F: Info.flags, Size, BaseAlignment: Alignment, Metadata: I.getAAMetadata(), SSID: Info.ssid,
5810 Ordering: Info.order, FailureOrdering: Info.failureOrder);
5811 MMOs.push_back(Elt: MMO);
5812 }
5813
5814 Result = DAG.getMemIntrinsicNode(Opcode: Info->opc, dl: getCurSDLoc(), VTList: VTs, Ops,
5815 MemVT: Info->memVT, MMOs);
5816 } else {
5817 Result = getTargetNonMemIntrinsicNode(IntrinsicVT: *I.getType(), HasChain, Ops, VTs);
5818 }
5819
5820 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
5821
5822 setValue(V: &I, NewN: Result);
5823}
5824
5825/// GetSignificand - Get the significand and build it into a floating-point
5826/// number with exponent of 1:
5827///
5828/// Op = (Op & 0x007fffff) | 0x3f800000;
5829///
5830/// where Op is the hexadecimal representation of floating point value.
5831static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
5832 SDValue t1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Op,
5833 N2: DAG.getConstant(Val: 0x007fffff, DL: dl, VT: MVT::i32));
5834 SDValue t2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: t1,
5835 N2: DAG.getConstant(Val: 0x3f800000, DL: dl, VT: MVT::i32));
5836 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: t2);
5837}
5838
5839/// GetExponent - Get the exponent:
5840///
5841/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5842///
5843/// where Op is the hexadecimal representation of floating point value.
5844static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
5845 const TargetLowering &TLI, const SDLoc &dl) {
5846 SDValue t0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Op,
5847 N2: DAG.getConstant(Val: 0x7f800000, DL: dl, VT: MVT::i32));
5848 SDValue t1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: t0,
5849 N2: DAG.getShiftAmountConstant(Val: 23, VT: MVT::i32, DL: dl));
5850 SDValue t2 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: t1,
5851 N2: DAG.getConstant(Val: 127, DL: dl, VT: MVT::i32));
5852 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::f32, Operand: t2);
5853}
5854
5855/// getF32Constant - Get 32-bit floating point constant.
5856static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
5857 const SDLoc &dl) {
5858 return DAG.getConstantFP(Val: APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), DL: dl,
5859 VT: MVT::f32);
5860}
5861
5862static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
5863 SelectionDAG &DAG) {
5864 // TODO: What fast-math-flags should be set on the floating-point nodes?
5865
5866 // IntegerPartOfX = ((int32_t)(t0);
5867 SDValue IntegerPartOfX = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: t0);
5868
5869 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5870 SDValue t1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::f32, Operand: IntegerPartOfX);
5871 SDValue X = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0, N2: t1);
5872
5873 // IntegerPartOfX <<= 23;
5874 IntegerPartOfX = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: IntegerPartOfX,
5875 N2: DAG.getShiftAmountConstant(Val: 23, VT: MVT::i32, DL: dl));
5876
5877 SDValue TwoToFractionalPartOfX;
5878 if (LimitFloatPrecision <= 6) {
5879 // For floating-point precision of 6:
5880 //
5881 // TwoToFractionalPartOfX =
5882 // 0.997535578f +
5883 // (0.735607626f + 0.252464424f * x) * x;
5884 //
5885 // error 0.0144103317, which is 6 bits
5886 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5887 N2: getF32Constant(DAG, Flt: 0x3e814304, dl));
5888 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5889 N2: getF32Constant(DAG, Flt: 0x3f3c50c8, dl));
5890 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5891 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5892 N2: getF32Constant(DAG, Flt: 0x3f7f5e7e, dl));
5893 } else if (LimitFloatPrecision <= 12) {
5894 // For floating-point precision of 12:
5895 //
5896 // TwoToFractionalPartOfX =
5897 // 0.999892986f +
5898 // (0.696457318f +
5899 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5900 //
5901 // error 0.000107046256, which is 13 to 14 bits
5902 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5903 N2: getF32Constant(DAG, Flt: 0x3da235e3, dl));
5904 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5905 N2: getF32Constant(DAG, Flt: 0x3e65b8f3, dl));
5906 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5907 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5908 N2: getF32Constant(DAG, Flt: 0x3f324b07, dl));
5909 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5910 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5911 N2: getF32Constant(DAG, Flt: 0x3f7ff8fd, dl));
5912 } else { // LimitFloatPrecision <= 18
5913 // For floating-point precision of 18:
5914 //
5915 // TwoToFractionalPartOfX =
5916 // 0.999999982f +
5917 // (0.693148872f +
5918 // (0.240227044f +
5919 // (0.554906021e-1f +
5920 // (0.961591928e-2f +
5921 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5922 // error 2.47208000*10^(-7), which is better than 18 bits
5923 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5924 N2: getF32Constant(DAG, Flt: 0x3924b03e, dl));
5925 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5926 N2: getF32Constant(DAG, Flt: 0x3ab24b87, dl));
5927 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5928 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5929 N2: getF32Constant(DAG, Flt: 0x3c1d8c17, dl));
5930 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5931 SDValue t7 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5932 N2: getF32Constant(DAG, Flt: 0x3d634a1d, dl));
5933 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5934 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
5935 N2: getF32Constant(DAG, Flt: 0x3e75fe14, dl));
5936 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
5937 SDValue t11 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t10,
5938 N2: getF32Constant(DAG, Flt: 0x3f317234, dl));
5939 SDValue t12 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t11, N2: X);
5940 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t12,
5941 N2: getF32Constant(DAG, Flt: 0x3f800000, dl));
5942 }
5943
5944 // Add the exponent into the result in integer domain.
5945 SDValue t13 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: TwoToFractionalPartOfX);
5946 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32,
5947 Operand: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: t13, N2: IntegerPartOfX));
5948}
5949
5950/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5951/// limited-precision mode.
5952static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5953 const TargetLowering &TLI, SDNodeFlags Flags) {
5954 if (Op.getValueType() == MVT::f32 &&
5955 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5956
5957 // Put the exponent in the right bit position for later addition to the
5958 // final result:
5959 //
5960 // t0 = Op * log2(e)
5961
5962 // TODO: What fast-math-flags should be set here?
5963 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Op,
5964 N2: DAG.getConstantFP(Val: numbers::log2ef, DL: dl, VT: MVT::f32));
5965 return getLimitedPrecisionExp2(t0, dl, DAG);
5966 }
5967
5968 // No special expansion.
5969 return DAG.getNode(Opcode: ISD::FEXP, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5970}
5971
5972/// expandLog - Lower a log intrinsic. Handles the special sequences for
5973/// limited-precision mode.
5974static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5975 const TargetLowering &TLI, SDNodeFlags Flags) {
5976 // TODO: What fast-math-flags should be set on the floating-point nodes?
5977
5978 if (Op.getValueType() == MVT::f32 &&
5979 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5980 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
5981
5982 // Scale the exponent by log(2).
5983 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
5984 SDValue LogOfExponent =
5985 DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Exp,
5986 N2: DAG.getConstantFP(Val: numbers::ln2f, DL: dl, VT: MVT::f32));
5987
5988 // Get the significand and build it into a floating-point number with
5989 // exponent of 1.
5990 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5991
5992 SDValue LogOfMantissa;
5993 if (LimitFloatPrecision <= 6) {
5994 // For floating-point precision of 6:
5995 //
5996 // LogofMantissa =
5997 // -1.1609546f +
5998 // (1.4034025f - 0.23903021f * x) * x;
5999 //
6000 // error 0.0034276066, which is better than 8 bits
6001 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6002 N2: getF32Constant(DAG, Flt: 0xbe74c456, dl));
6003 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6004 N2: getF32Constant(DAG, Flt: 0x3fb3a2b1, dl));
6005 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6006 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6007 N2: getF32Constant(DAG, Flt: 0x3f949a29, dl));
6008 } else if (LimitFloatPrecision <= 12) {
6009 // For floating-point precision of 12:
6010 //
6011 // LogOfMantissa =
6012 // -1.7417939f +
6013 // (2.8212026f +
6014 // (-1.4699568f +
6015 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
6016 //
6017 // error 0.000061011436, which is 14 bits
6018 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6019 N2: getF32Constant(DAG, Flt: 0xbd67b6d6, dl));
6020 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6021 N2: getF32Constant(DAG, Flt: 0x3ee4f4b8, dl));
6022 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6023 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6024 N2: getF32Constant(DAG, Flt: 0x3fbc278b, dl));
6025 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6026 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
6027 N2: getF32Constant(DAG, Flt: 0x40348e95, dl));
6028 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6029 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
6030 N2: getF32Constant(DAG, Flt: 0x3fdef31a, dl));
6031 } else { // LimitFloatPrecision <= 18
6032 // For floating-point precision of 18:
6033 //
6034 // LogOfMantissa =
6035 // -2.1072184f +
6036 // (4.2372794f +
6037 // (-3.7029485f +
6038 // (2.2781945f +
6039 // (-0.87823314f +
6040 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
6041 //
6042 // error 0.0000023660568, which is better than 18 bits
6043 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6044 N2: getF32Constant(DAG, Flt: 0xbc91e5ac, dl));
6045 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6046 N2: getF32Constant(DAG, Flt: 0x3e4350aa, dl));
6047 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6048 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6049 N2: getF32Constant(DAG, Flt: 0x3f60d3e3, dl));
6050 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6051 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
6052 N2: getF32Constant(DAG, Flt: 0x4011cdf0, dl));
6053 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6054 SDValue t7 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
6055 N2: getF32Constant(DAG, Flt: 0x406cfd1c, dl));
6056 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
6057 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
6058 N2: getF32Constant(DAG, Flt: 0x408797cb, dl));
6059 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
6060 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t10,
6061 N2: getF32Constant(DAG, Flt: 0x4006dcab, dl));
6062 }
6063
6064 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: LogOfMantissa);
6065 }
6066
6067 // No special expansion.
6068 return DAG.getNode(Opcode: ISD::FLOG, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6069}
6070
6071/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
6072/// limited-precision mode.
6073static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
6074 const TargetLowering &TLI, SDNodeFlags Flags) {
6075 // TODO: What fast-math-flags should be set on the floating-point nodes?
6076
6077 if (Op.getValueType() == MVT::f32 &&
6078 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
6079 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
6080
6081 // Get the exponent.
6082 SDValue LogOfExponent = GetExponent(DAG, Op: Op1, TLI, dl);
6083
6084 // Get the significand and build it into a floating-point number with
6085 // exponent of 1.
6086 SDValue X = GetSignificand(DAG, Op: Op1, dl);
6087
6088 // Different possible minimax approximations of significand in
6089 // floating-point for various degrees of accuracy over [1,2].
6090 SDValue Log2ofMantissa;
6091 if (LimitFloatPrecision <= 6) {
6092 // For floating-point precision of 6:
6093 //
6094 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
6095 //
6096 // error 0.0049451742, which is more than 7 bits
6097 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6098 N2: getF32Constant(DAG, Flt: 0xbeb08fe0, dl));
6099 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6100 N2: getF32Constant(DAG, Flt: 0x40019463, dl));
6101 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6102 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6103 N2: getF32Constant(DAG, Flt: 0x3fd6633d, dl));
6104 } else if (LimitFloatPrecision <= 12) {
6105 // For floating-point precision of 12:
6106 //
6107 // Log2ofMantissa =
6108 // -2.51285454f +
6109 // (4.07009056f +
6110 // (-2.12067489f +
6111 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
6112 //
6113 // error 0.0000876136000, which is better than 13 bits
6114 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6115 N2: getF32Constant(DAG, Flt: 0xbda7262e, dl));
6116 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6117 N2: getF32Constant(DAG, Flt: 0x3f25280b, dl));
6118 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6119 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6120 N2: getF32Constant(DAG, Flt: 0x4007b923, dl));
6121 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6122 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
6123 N2: getF32Constant(DAG, Flt: 0x40823e2f, dl));
6124 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6125 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
6126 N2: getF32Constant(DAG, Flt: 0x4020d29c, dl));
6127 } else { // LimitFloatPrecision <= 18
6128 // For floating-point precision of 18:
6129 //
6130 // Log2ofMantissa =
6131 // -3.0400495f +
6132 // (6.1129976f +
6133 // (-5.3420409f +
6134 // (3.2865683f +
6135 // (-1.2669343f +
6136 // (0.27515199f -
6137 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
6138 //
6139 // error 0.0000018516, which is better than 18 bits
6140 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6141 N2: getF32Constant(DAG, Flt: 0xbcd2769e, dl));
6142 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6143 N2: getF32Constant(DAG, Flt: 0x3e8ce0b9, dl));
6144 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6145 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6146 N2: getF32Constant(DAG, Flt: 0x3fa22ae7, dl));
6147 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6148 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
6149 N2: getF32Constant(DAG, Flt: 0x40525723, dl));
6150 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6151 SDValue t7 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
6152 N2: getF32Constant(DAG, Flt: 0x40aaf200, dl));
6153 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
6154 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
6155 N2: getF32Constant(DAG, Flt: 0x40c39dad, dl));
6156 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
6157 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t10,
6158 N2: getF32Constant(DAG, Flt: 0x4042902c, dl));
6159 }
6160
6161 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: Log2ofMantissa);
6162 }
6163
6164 // No special expansion.
6165 return DAG.getNode(Opcode: ISD::FLOG2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6166}
6167
6168/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
6169/// limited-precision mode.
6170static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
6171 const TargetLowering &TLI, SDNodeFlags Flags) {
6172 // TODO: What fast-math-flags should be set on the floating-point nodes?
6173
6174 if (Op.getValueType() == MVT::f32 &&
6175 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
6176 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
6177
6178 // Scale the exponent by log10(2) [0.30102999f].
6179 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
6180 SDValue LogOfExponent = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Exp,
6181 N2: getF32Constant(DAG, Flt: 0x3e9a209a, dl));
6182
6183 // Get the significand and build it into a floating-point number with
6184 // exponent of 1.
6185 SDValue X = GetSignificand(DAG, Op: Op1, dl);
6186
6187 SDValue Log10ofMantissa;
6188 if (LimitFloatPrecision <= 6) {
6189 // For floating-point precision of 6:
6190 //
6191 // Log10ofMantissa =
6192 // -0.50419619f +
6193 // (0.60948995f - 0.10380950f * x) * x;
6194 //
6195 // error 0.0014886165, which is 6 bits
6196 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6197 N2: getF32Constant(DAG, Flt: 0xbdd49a13, dl));
6198 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6199 N2: getF32Constant(DAG, Flt: 0x3f1c0789, dl));
6200 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6201 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6202 N2: getF32Constant(DAG, Flt: 0x3f011300, dl));
6203 } else if (LimitFloatPrecision <= 12) {
6204 // For floating-point precision of 12:
6205 //
6206 // Log10ofMantissa =
6207 // -0.64831180f +
6208 // (0.91751397f +
6209 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
6210 //
6211 // error 0.00019228036, which is better than 12 bits
6212 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6213 N2: getF32Constant(DAG, Flt: 0x3d431f31, dl));
6214 SDValue t1 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0,
6215 N2: getF32Constant(DAG, Flt: 0x3ea21fb2, dl));
6216 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6217 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
6218 N2: getF32Constant(DAG, Flt: 0x3f6ae232, dl));
6219 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6220 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t4,
6221 N2: getF32Constant(DAG, Flt: 0x3f25f7c3, dl));
6222 } else { // LimitFloatPrecision <= 18
6223 // For floating-point precision of 18:
6224 //
6225 // Log10ofMantissa =
6226 // -0.84299375f +
6227 // (1.5327582f +
6228 // (-1.0688956f +
6229 // (0.49102474f +
6230 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
6231 //
6232 // error 0.0000037995730, which is better than 18 bits
6233 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6234 N2: getF32Constant(DAG, Flt: 0x3c5d51ce, dl));
6235 SDValue t1 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0,
6236 N2: getF32Constant(DAG, Flt: 0x3e00685a, dl));
6237 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6238 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
6239 N2: getF32Constant(DAG, Flt: 0x3efb6798, dl));
6240 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6241 SDValue t5 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t4,
6242 N2: getF32Constant(DAG, Flt: 0x3f88d192, dl));
6243 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6244 SDValue t7 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
6245 N2: getF32Constant(DAG, Flt: 0x3fc4316c, dl));
6246 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
6247 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t8,
6248 N2: getF32Constant(DAG, Flt: 0x3f57ce70, dl));
6249 }
6250
6251 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: Log10ofMantissa);
6252 }
6253
6254 // No special expansion.
6255 return DAG.getNode(Opcode: ISD::FLOG10, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6256}
6257
6258/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
6259/// limited-precision mode.
6260static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
6261 const TargetLowering &TLI, SDNodeFlags Flags) {
6262 if (Op.getValueType() == MVT::f32 &&
6263 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
6264 return getLimitedPrecisionExp2(t0: Op, dl, DAG);
6265
6266 // No special expansion.
6267 return DAG.getNode(Opcode: ISD::FEXP2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6268}
6269
6270/// visitPow - Lower a pow intrinsic. Handles the special sequences for
6271/// limited-precision mode with x == 10.0f.
6272static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
6273 SelectionDAG &DAG, const TargetLowering &TLI,
6274 SDNodeFlags Flags) {
6275 bool IsExp10 = false;
6276 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
6277 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
6278 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
6279 APFloat Ten(10.0f);
6280 IsExp10 = LHSC->isExactlyValue(V: Ten);
6281 }
6282 }
6283
6284 // TODO: What fast-math-flags should be set on the FMUL node?
6285 if (IsExp10) {
6286 // Put the exponent in the right bit position for later addition to the
6287 // final result:
6288 //
6289 // #define LOG2OF10 3.3219281f
6290 // t0 = Op * LOG2OF10;
6291 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: RHS,
6292 N2: getF32Constant(DAG, Flt: 0x40549a78, dl));
6293 return getLimitedPrecisionExp2(t0, dl, DAG);
6294 }
6295
6296 // No special expansion.
6297 return DAG.getNode(Opcode: ISD::FPOW, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags);
6298}
6299
6300/// ExpandPowI - Expand a llvm.powi intrinsic.
6301static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
6302 SelectionDAG &DAG) {
6303 // If RHS is a constant, we can expand this out to a multiplication tree if
6304 // it's beneficial on the target, otherwise we end up lowering to a call to
6305 // __powidf2 (for example).
6306 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
6307 unsigned Val = RHSC->getSExtValue();
6308
6309 // powi(x, 0) -> 1.0
6310 if (Val == 0)
6311 return DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType());
6312
6313 if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
6314 Exponent: Val, OptForSize: DAG.shouldOptForSize())) {
6315 // Get the exponent as a positive value.
6316 if ((int)Val < 0)
6317 Val = -Val;
6318 // We use the simple binary decomposition method to generate the multiply
6319 // sequence. There are more optimal ways to do this (for example,
6320 // powi(x,15) generates one more multiply than it should), but this has
6321 // the benefit of being both really simple and much better than a libcall.
6322 SDValue Res; // Logically starts equal to 1.0
6323 SDValue CurSquare = LHS;
6324 // TODO: Intrinsics should have fast-math-flags that propagate to these
6325 // nodes.
6326 while (Val) {
6327 if (Val & 1) {
6328 if (Res.getNode())
6329 Res =
6330 DAG.getNode(Opcode: ISD::FMUL, DL, VT: Res.getValueType(), N1: Res, N2: CurSquare);
6331 else
6332 Res = CurSquare; // 1.0*CurSquare.
6333 }
6334
6335 CurSquare = DAG.getNode(Opcode: ISD::FMUL, DL, VT: CurSquare.getValueType(),
6336 N1: CurSquare, N2: CurSquare);
6337 Val >>= 1;
6338 }
6339
6340 // If the original was negative, invert the result, producing 1/(x*x*x).
6341 if (RHSC->getSExtValue() < 0)
6342 Res = DAG.getNode(Opcode: ISD::FDIV, DL, VT: LHS.getValueType(),
6343 N1: DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType()), N2: Res);
6344 return Res;
6345 }
6346 }
6347
6348 // Otherwise, expand to a libcall.
6349 return DAG.getNode(Opcode: ISD::FPOWI, DL, VT: LHS.getValueType(), N1: LHS, N2: RHS);
6350}
6351
6352static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
6353 SDValue LHS, SDValue RHS, SDValue Scale,
6354 SelectionDAG &DAG, const TargetLowering &TLI) {
6355 EVT VT = LHS.getValueType();
6356 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
6357 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
6358 LLVMContext &Ctx = *DAG.getContext();
6359
6360 // If the type is legal but the operation isn't, this node might survive all
6361 // the way to operation legalization. If we end up there and we do not have
6362 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
6363 // node.
6364
6365 // Coax the legalizer into expanding the node during type legalization instead
6366 // by bumping the size by one bit. This will force it to Promote, enabling the
6367 // early expansion and avoiding the need to expand later.
6368
6369 // We don't have to do this if Scale is 0; that can always be expanded, unless
6370 // it's a saturating signed operation. Those can experience true integer
6371 // division overflow, a case which we must avoid.
6372
6373 // FIXME: We wouldn't have to do this (or any of the early
6374 // expansion/promotion) if it was possible to expand a libcall of an
6375 // illegal type during operation legalization. But it's not, so things
6376 // get a bit hacky.
6377 unsigned ScaleInt = Scale->getAsZExtVal();
6378 if ((ScaleInt > 0 || (Saturating && Signed)) &&
6379 (TLI.isTypeLegal(VT) ||
6380 (VT.isVector() && TLI.isTypeLegal(VT: VT.getVectorElementType())))) {
6381 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
6382 Op: Opcode, VT, Scale: ScaleInt);
6383 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
6384 EVT PromVT;
6385 if (VT.isScalarInteger())
6386 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits() + 1);
6387 else if (VT.isVector()) {
6388 PromVT = VT.getVectorElementType();
6389 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PromVT.getSizeInBits() + 1);
6390 PromVT = EVT::getVectorVT(Context&: Ctx, VT: PromVT, EC: VT.getVectorElementCount());
6391 } else
6392 llvm_unreachable("Wrong VT for DIVFIX?");
6393 LHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL, VT: PromVT);
6394 RHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL, VT: PromVT);
6395 EVT ShiftTy = TLI.getShiftAmountTy(LHSTy: PromVT, DL: DAG.getDataLayout());
6396 // For saturating operations, we need to shift up the LHS to get the
6397 // proper saturation width, and then shift down again afterwards.
6398 if (Saturating)
6399 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: PromVT, N1: LHS,
6400 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
6401 SDValue Res = DAG.getNode(Opcode, DL, VT: PromVT, N1: LHS, N2: RHS, N3: Scale);
6402 if (Saturating)
6403 Res = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: PromVT, N1: Res,
6404 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
6405 return DAG.getZExtOrTrunc(Op: Res, DL, VT);
6406 }
6407 }
6408
6409 return DAG.getNode(Opcode, DL, VT, N1: LHS, N2: RHS, N3: Scale);
6410}
6411
6412// getUnderlyingArgRegs - Find underlying registers used for a truncated,
6413// bitcasted, or split argument. Returns a list of <Register, size in bits>
6414static void
6415getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
6416 const SDValue &N) {
6417 switch (N.getOpcode()) {
6418 case ISD::CopyFromReg: {
6419 SDValue Op = N.getOperand(i: 1);
6420 Regs.emplace_back(Args: cast<RegisterSDNode>(Val&: Op)->getReg(),
6421 Args: Op.getValueType().getSizeInBits());
6422 return;
6423 }
6424 case ISD::BITCAST:
6425 case ISD::AssertZext:
6426 case ISD::AssertSext:
6427 case ISD::TRUNCATE:
6428 getUnderlyingArgRegs(Regs, N: N.getOperand(i: 0));
6429 return;
6430 case ISD::BUILD_PAIR:
6431 case ISD::BUILD_VECTOR:
6432 case ISD::CONCAT_VECTORS:
6433 for (SDValue Op : N->op_values())
6434 getUnderlyingArgRegs(Regs, N: Op);
6435 return;
6436 default:
6437 return;
6438 }
6439}
6440
6441/// If the DbgValueInst is a dbg_value of a function argument, create the
6442/// corresponding DBG_VALUE machine instruction for it now. At the end of
6443/// instruction selection, they will be inserted to the entry BB.
6444/// We don't currently support this for variadic dbg_values, as they shouldn't
6445/// appear for function arguments or in the prologue.
6446bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6447 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6448 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6449 const Argument *Arg = dyn_cast<Argument>(Val: V);
6450 if (!Arg)
6451 return false;
6452
6453 MachineFunction &MF = DAG.getMachineFunction();
6454 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6455
6456 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6457 // we've been asked to pursue.
6458 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6459 bool Indirect) {
6460 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6461 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6462 // pointing at the VReg, which will be patched up later.
6463 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_INSTR_REF);
6464 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
6465 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6466 /* isKill */ false, /* isDead */ false,
6467 /* isUndef */ false, /* isEarlyClobber */ false,
6468 /* SubReg */ 0, /* isDebug */ true)});
6469
6470 auto *NewDIExpr = FragExpr;
6471 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6472 // the DIExpression.
6473 if (Indirect)
6474 NewDIExpr = DIExpression::prepend(Expr: FragExpr, Flags: DIExpression::DerefBefore);
6475 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
6476 NewDIExpr = DIExpression::prependOpcodes(Expr: NewDIExpr, Ops);
6477 return BuildMI(MF, DL, MCID: Inst, IsIndirect: false, MOs, Variable, Expr: NewDIExpr);
6478 } else {
6479 // Create a completely standard DBG_VALUE.
6480 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_VALUE);
6481 return BuildMI(MF, DL, MCID: Inst, IsIndirect: Indirect, Reg, Variable, Expr: FragExpr);
6482 }
6483 };
6484
6485 if (Kind == FuncArgumentDbgValueKind::Value) {
6486 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6487 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6488 // the entry block.
6489 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6490 if (!IsInEntryBlock)
6491 return false;
6492
6493 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6494 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6495 // variable that also is a param.
6496 //
6497 // Although, if we are at the top of the entry block already, we can still
6498 // emit using ArgDbgValue. This might catch some situations when the
6499 // dbg.value refers to an argument that isn't used in the entry block, so
6500 // any CopyToReg node would be optimized out and the only way to express
6501 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6502 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6503 // we should only emit as ArgDbgValue if the Variable is an argument to the
6504 // current function, and the dbg.value intrinsic is found in the entry
6505 // block.
6506 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6507 !DL->getInlinedAt();
6508 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6509 if (!IsInPrologue && !VariableIsFunctionInputArg)
6510 return false;
6511
6512 // Here we assume that a function argument on IR level only can be used to
6513 // describe one input parameter on source level. If we for example have
6514 // source code like this
6515 //
6516 // struct A { long x, y; };
6517 // void foo(struct A a, long b) {
6518 // ...
6519 // b = a.x;
6520 // ...
6521 // }
6522 //
6523 // and IR like this
6524 //
6525 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6526 // entry:
6527 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6528 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6529 // call void @llvm.dbg.value(metadata i32 %b, "b",
6530 // ...
6531 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6532 // ...
6533 //
6534 // then the last dbg.value is describing a parameter "b" using a value that
6535 // is an argument. But since we already has used %a1 to describe a parameter
6536 // we should not handle that last dbg.value here (that would result in an
6537 // incorrect hoisting of the DBG_VALUE to the function entry).
6538 // Notice that we allow one dbg.value per IR level argument, to accommodate
6539 // for the situation with fragments above.
6540 // If there is no node for the value being handled, we return true to skip
6541 // the normal generation of debug info, as it would kill existing debug
6542 // info for the parameter in case of duplicates.
6543 if (VariableIsFunctionInputArg) {
6544 unsigned ArgNo = Arg->getArgNo();
6545 if (ArgNo >= FuncInfo.DescribedArgs.size())
6546 FuncInfo.DescribedArgs.resize(N: ArgNo + 1, t: false);
6547 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(Idx: ArgNo))
6548 return !NodeMap[V].getNode();
6549 FuncInfo.DescribedArgs.set(ArgNo);
6550 }
6551 }
6552
6553 bool IsIndirect = false;
6554 std::optional<MachineOperand> Op;
6555 // Some arguments' frame index is recorded during argument lowering.
6556 int FI = FuncInfo.getArgumentFrameIndex(A: Arg);
6557 if (FI != std::numeric_limits<int>::max())
6558 Op = MachineOperand::CreateFI(Idx: FI);
6559
6560 SmallVector<std::pair<Register, TypeSize>, 8> ArgRegsAndSizes;
6561 if (!Op && N.getNode()) {
6562 getUnderlyingArgRegs(Regs&: ArgRegsAndSizes, N);
6563 Register Reg;
6564 if (ArgRegsAndSizes.size() == 1)
6565 Reg = ArgRegsAndSizes.front().first;
6566
6567 if (Reg && Reg.isVirtual()) {
6568 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6569 Register PR = RegInfo.getLiveInPhysReg(VReg: Reg);
6570 if (PR)
6571 Reg = PR;
6572 }
6573 if (Reg) {
6574 Op = MachineOperand::CreateReg(Reg, isDef: false);
6575 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6576 }
6577 }
6578
6579 if (!Op && N.getNode()) {
6580 // Check if frame index is available.
6581 SDValue LCandidate = peekThroughBitcasts(V: N);
6582 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(Val: LCandidate.getNode()))
6583 if (FrameIndexSDNode *FINode =
6584 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
6585 Op = MachineOperand::CreateFI(Idx: FINode->getIndex());
6586 }
6587
6588 if (!Op) {
6589 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6590 auto splitMultiRegDbgValue =
6591 [&](ArrayRef<std::pair<Register, TypeSize>> SplitRegs) -> bool {
6592 unsigned Offset = 0;
6593 for (const auto &[Reg, RegSizeInBits] : SplitRegs) {
6594 // FIXME: Scalable sizes are not supported in fragment expressions.
6595 if (RegSizeInBits.isScalable())
6596 return false;
6597
6598 // If the expression is already a fragment, the current register
6599 // offset+size might extend beyond the fragment. In this case, only
6600 // the register bits that are inside the fragment are relevant.
6601 int RegFragmentSizeInBits = RegSizeInBits.getFixedValue();
6602 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6603 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6604 // The register is entirely outside the expression fragment,
6605 // so is irrelevant for debug info.
6606 if (Offset >= ExprFragmentSizeInBits)
6607 break;
6608 // The register is partially outside the expression fragment, only
6609 // the low bits within the fragment are relevant for debug info.
6610 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6611 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6612 }
6613 }
6614
6615 auto FragmentExpr = DIExpression::createFragmentExpression(
6616 Expr, OffsetInBits: Offset, SizeInBits: RegFragmentSizeInBits);
6617 Offset += RegSizeInBits.getFixedValue();
6618 // If a valid fragment expression cannot be created, the variable's
6619 // correct value cannot be determined and so it is set as poison.
6620 if (!FragmentExpr) {
6621 SDDbgValue *SDV = DAG.getConstantDbgValue(
6622 Var: Variable, Expr, C: PoisonValue::get(T: V->getType()), DL, O: SDNodeOrder);
6623 DAG.AddDbgValue(DB: SDV, isParameter: false);
6624 continue;
6625 }
6626 MachineInstr *NewMI = MakeVRegDbgValue(
6627 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6628 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6629 }
6630
6631 return true;
6632 };
6633
6634 // Check if ValueMap has reg number.
6635 DenseMap<const Value *, Register>::const_iterator
6636 VMI = FuncInfo.ValueMap.find(Val: V);
6637 if (VMI != FuncInfo.ValueMap.end()) {
6638 const auto &TLI = DAG.getTargetLoweringInfo();
6639 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6640 V->getType(), std::nullopt);
6641 if (RFV.occupiesMultipleRegs())
6642 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6643
6644 Op = MachineOperand::CreateReg(Reg: VMI->second, isDef: false);
6645 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6646 } else if (ArgRegsAndSizes.size() > 1) {
6647 // This was split due to the calling convention, and no virtual register
6648 // mapping exists for the value.
6649 return splitMultiRegDbgValue(ArgRegsAndSizes);
6650 }
6651 }
6652
6653 if (!Op)
6654 return false;
6655
6656 assert(Variable->isValidLocationForIntrinsic(DL) &&
6657 "Expected inlined-at fields to agree");
6658 MachineInstr *NewMI = nullptr;
6659
6660 if (Op->isReg())
6661 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6662 else
6663 NewMI = BuildMI(MF, DL, MCID: TII->get(Opcode: TargetOpcode::DBG_VALUE), IsIndirect: true, MOs: *Op,
6664 Variable, Expr);
6665
6666 // Otherwise, use ArgDbgValues.
6667 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6668 return true;
6669}
6670
6671/// Return the appropriate SDDbgValue based on N.
6672SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6673 DILocalVariable *Variable,
6674 DIExpression *Expr,
6675 const DebugLoc &dl,
6676 unsigned DbgSDNodeOrder) {
6677 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
6678 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6679 // stack slot locations.
6680 //
6681 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6682 // debug values here after optimization:
6683 //
6684 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6685 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6686 //
6687 // Both describe the direct values of their associated variables.
6688 return DAG.getFrameIndexDbgValue(Var: Variable, Expr, FI: FISDN->getIndex(),
6689 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6690 }
6691 return DAG.getDbgValue(Var: Variable, Expr, N: N.getNode(), R: N.getResNo(),
6692 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6693}
6694
6695static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6696 switch (Intrinsic) {
6697 case Intrinsic::smul_fix:
6698 return ISD::SMULFIX;
6699 case Intrinsic::umul_fix:
6700 return ISD::UMULFIX;
6701 case Intrinsic::smul_fix_sat:
6702 return ISD::SMULFIXSAT;
6703 case Intrinsic::umul_fix_sat:
6704 return ISD::UMULFIXSAT;
6705 case Intrinsic::sdiv_fix:
6706 return ISD::SDIVFIX;
6707 case Intrinsic::udiv_fix:
6708 return ISD::UDIVFIX;
6709 case Intrinsic::sdiv_fix_sat:
6710 return ISD::SDIVFIXSAT;
6711 case Intrinsic::udiv_fix_sat:
6712 return ISD::UDIVFIXSAT;
6713 default:
6714 llvm_unreachable("Unhandled fixed point intrinsic");
6715 }
6716}
6717
6718/// Given a @llvm.call.preallocated.setup, return the corresponding
6719/// preallocated call.
6720static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6721 assert(cast<CallBase>(PreallocatedSetup)
6722 ->getCalledFunction()
6723 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6724 "expected call_preallocated_setup Value");
6725 for (const auto *U : PreallocatedSetup->users()) {
6726 auto *UseCall = cast<CallBase>(Val: U);
6727 const Function *Fn = UseCall->getCalledFunction();
6728 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6729 return UseCall;
6730 }
6731 }
6732 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6733}
6734
6735/// If DI is a debug value with an EntryValue expression, lower it using the
6736/// corresponding physical register of the associated Argument value
6737/// (guaranteed to exist by the verifier).
6738bool SelectionDAGBuilder::visitEntryValueDbgValue(
6739 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6740 DIExpression *Expr, DebugLoc DbgLoc) {
6741 if (!Expr->isEntryValue() || !hasSingleElement(C&: Values))
6742 return false;
6743
6744 // These properties are guaranteed by the verifier.
6745 const Argument *Arg = cast<Argument>(Val: Values[0]);
6746 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6747
6748 auto ArgIt = FuncInfo.ValueMap.find(Val: Arg);
6749 if (ArgIt == FuncInfo.ValueMap.end()) {
6750 LLVM_DEBUG(
6751 dbgs() << "Dropping dbg.value: expression is entry_value but "
6752 "couldn't find an associated register for the Argument\n");
6753 return true;
6754 }
6755 Register ArgVReg = ArgIt->getSecond();
6756
6757 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6758 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6759 SDDbgValue *SDV = DAG.getVRegDbgValue(
6760 Var: Variable, Expr, VReg: PhysReg, IsIndirect: false /*IsIndidrect*/, DL: DbgLoc, O: SDNodeOrder);
6761 DAG.AddDbgValue(DB: SDV, isParameter: false /*treat as dbg.declare byval parameter*/);
6762 return true;
6763 }
6764 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6765 "couldn't find a physical register\n");
6766 return true;
6767}
6768
6769/// Lower the call to the specified intrinsic function.
6770void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6771 unsigned Intrinsic) {
6772 SDLoc sdl = getCurSDLoc();
6773 switch (Intrinsic) {
6774 case Intrinsic::experimental_convergence_anchor:
6775 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_ANCHOR, DL: sdl, VT: MVT::Untyped));
6776 break;
6777 case Intrinsic::experimental_convergence_entry:
6778 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_ENTRY, DL: sdl, VT: MVT::Untyped));
6779 break;
6780 case Intrinsic::experimental_convergence_loop: {
6781 auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
6782 auto *Token = Bundle->Inputs[0].get();
6783 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_LOOP, DL: sdl, VT: MVT::Untyped,
6784 Operand: getValue(V: Token)));
6785 break;
6786 }
6787 }
6788}
6789
6790void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6791 unsigned IntrinsicID) {
6792 // For now, we're only lowering an 'add' histogram.
6793 // We can add others later, e.g. saturating adds, min/max.
6794 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6795 "Tried to lower unsupported histogram type");
6796 SDLoc sdl = getCurSDLoc();
6797 Value *Ptr = I.getOperand(i_nocapture: 0);
6798 SDValue Inc = getValue(V: I.getOperand(i_nocapture: 1));
6799 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
6800
6801 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6802 DataLayout TargetDL = DAG.getDataLayout();
6803 EVT VT = Inc.getValueType();
6804 Align Alignment = DAG.getEVTAlign(MemoryVT: VT);
6805
6806 const MDNode *Ranges = getRangeMetadata(I);
6807
6808 SDValue Root = DAG.getRoot();
6809 SDValue Base;
6810 SDValue Index;
6811 SDValue Scale;
6812 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
6813 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
6814
6815 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6816
6817 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6818 PtrInfo: MachinePointerInfo(AS),
6819 F: MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
6820 Size: MemoryLocation::UnknownSize, BaseAlignment: Alignment,
6821 Metadata: MMOMetadata(I.getAAMetadata(), Ranges));
6822
6823 if (!UniformBase) {
6824 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6825 Index = getValue(V: Ptr);
6826 Scale =
6827 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6828 }
6829
6830 EVT IdxVT = Index.getValueType();
6831
6832 // Avoid using e.g. i32 as index type when the increment must be performed
6833 // on i64's.
6834 bool MustExtendIndex = VT.getScalarSizeInBits() > IdxVT.getScalarSizeInBits();
6835 EVT EltTy = MustExtendIndex ? VT : IdxVT.getVectorElementType();
6836 if (MustExtendIndex || TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
6837 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
6838 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
6839 }
6840
6841 SDValue ID = DAG.getTargetConstant(Val: IntrinsicID, DL: sdl, VT: MVT::i32);
6842
6843 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6844 SDValue Histogram = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT: VT, dl: sdl,
6845 Ops, MMO, IndexType: ISD::SIGNED_SCALED);
6846
6847 setValue(V: &I, NewN: Histogram);
6848 DAG.setRoot(Histogram);
6849}
6850
6851void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6852 unsigned Intrinsic) {
6853 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6854 "Tried lowering invalid vector extract last");
6855 SDLoc sdl = getCurSDLoc();
6856 const DataLayout &Layout = DAG.getDataLayout();
6857 SDValue Data = getValue(V: I.getOperand(i_nocapture: 0));
6858 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
6859
6860 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6861 EVT ResVT = TLI.getValueType(DL: Layout, Ty: I.getType());
6862
6863 EVT ExtVT = TLI.getVectorIdxTy(DL: Layout);
6864 SDValue Idx = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL: sdl, VT: ExtVT, Operand: Mask);
6865 SDValue Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: sdl, VT: ResVT, N1: Data, N2: Idx);
6866
6867 Value *Default = I.getOperand(i_nocapture: 2);
6868 if (!isa<PoisonValue>(Val: Default) && !isa<UndefValue>(Val: Default)) {
6869 SDValue PassThru = getValue(V: Default);
6870 EVT BoolVT = Mask.getValueType().getScalarType();
6871 SDValue AnyActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: sdl, VT: BoolVT, Operand: Mask);
6872 Result = DAG.getSelect(DL: sdl, VT: ResVT, Cond: AnyActive, LHS: Result, RHS: PassThru);
6873 }
6874
6875 setValue(V: &I, NewN: Result);
6876}
6877
6878/// Lower the call to the specified intrinsic function.
6879void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6880 unsigned Intrinsic) {
6881 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6882 SDLoc sdl = getCurSDLoc();
6883 DebugLoc dl = getCurDebugLoc();
6884 SDValue Res;
6885
6886 SDNodeFlags Flags;
6887 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
6888 Flags.copyFMF(FPMO: *FPOp);
6889
6890 switch (Intrinsic) {
6891 default:
6892 // By default, turn this into a target intrinsic node.
6893 visitTargetIntrinsic(I, Intrinsic);
6894 return;
6895 case Intrinsic::vscale: {
6896 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6897 setValue(V: &I, NewN: DAG.getVScale(DL: sdl, VT, MulImm: APInt(VT.getSizeInBits(), 1)));
6898 return;
6899 }
6900 case Intrinsic::vastart: visitVAStart(I); return;
6901 case Intrinsic::vaend: visitVAEnd(I); return;
6902 case Intrinsic::vacopy: visitVACopy(I); return;
6903 case Intrinsic::returnaddress:
6904 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::RETURNADDR, DL: sdl,
6905 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
6906 Operand: getValue(V: I.getArgOperand(i: 0))));
6907 return;
6908 case Intrinsic::addressofreturnaddress:
6909 setValue(V: &I,
6910 NewN: DAG.getNode(Opcode: ISD::ADDROFRETURNADDR, DL: sdl,
6911 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6912 return;
6913 case Intrinsic::sponentry:
6914 setValue(V: &I,
6915 NewN: DAG.getNode(Opcode: ISD::SPONENTRY, DL: sdl,
6916 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6917 return;
6918 case Intrinsic::frameaddress:
6919 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FRAMEADDR, DL: sdl,
6920 VT: TLI.getFrameIndexTy(DL: DAG.getDataLayout()),
6921 Operand: getValue(V: I.getArgOperand(i: 0))));
6922 return;
6923 case Intrinsic::read_volatile_register:
6924 case Intrinsic::read_register: {
6925 Value *Reg = I.getArgOperand(i: 0);
6926 SDValue Chain = getRoot();
6927 SDValue RegName =
6928 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6929 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6930 Res = DAG.getNode(Opcode: ISD::READ_REGISTER, DL: sdl,
6931 VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), N1: Chain, N2: RegName);
6932 setValue(V: &I, NewN: Res);
6933 DAG.setRoot(Res.getValue(R: 1));
6934 return;
6935 }
6936 case Intrinsic::write_register: {
6937 Value *Reg = I.getArgOperand(i: 0);
6938 Value *RegValue = I.getArgOperand(i: 1);
6939 SDValue Chain = getRoot();
6940 SDValue RegName =
6941 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6942 DAG.setRoot(DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL: sdl, VT: MVT::Other, N1: Chain,
6943 N2: RegName, N3: getValue(V: RegValue)));
6944 return;
6945 }
6946 case Intrinsic::write_volatile_register: {
6947 Value *Reg = I.getArgOperand(i: 0);
6948 Value *RegValue = I.getArgOperand(i: 1);
6949 SDValue Chain = getRoot();
6950 const MDNode *MD = cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata());
6951 SDValue RegName = DAG.getMDNode(MD);
6952 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: RegValue->getType());
6953 SDValue WriteChain = DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL: sdl, VT: MVT::Other,
6954 N1: Chain, N2: RegName, N3: getValue(V: RegValue));
6955 // FAKE_USE of the physical register marks it live after the WRITE_REGISTER,
6956 // preventing the backend from dead-eliminating the write. This is
6957 // preferred over READ_REGISTER, which would emit extra register copies
6958 // (e.g. fmov xN, dN for FP/SIMD registers).
6959 const MDString *RegStr = cast<MDString>(Val: MD->getOperand(I: 0));
6960 LLT Ty = VT.isSimple() ? getLLTForMVT(Ty: VT.getSimpleVT()) : LLT();
6961 const MachineFunction &MF = DAG.getMachineFunction();
6962 Register PhysReg =
6963 TLI.getRegisterByName(RegName: RegStr->getString().data(), Ty, MF);
6964 if (PhysReg.isValid()) {
6965 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6966 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: PhysReg);
6967 MVT RegVT = *TRI->legalclasstypes_begin(RC: *RC);
6968 DAG.setRoot(DAG.getNode(Opcode: ISD::FAKE_USE, DL: sdl, VT: MVT::Other,
6969 Ops: {WriteChain, DAG.getRegister(Reg: PhysReg, VT: RegVT)}));
6970 } else {
6971 DAG.setRoot(WriteChain);
6972 }
6973 return;
6974 }
6975 case Intrinsic::memcpy:
6976 case Intrinsic::memcpy_inline: {
6977 const auto &MCI = cast<MemCpyInst>(Val: I);
6978 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6979 SDValue Src = getValue(V: I.getArgOperand(i: 1));
6980 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6981 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6982 "memcpy_inline needs constant size");
6983 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6984 Align DstAlign = MCI.getDestAlign().valueOrOne();
6985 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6986 bool isVol = MCI.isVolatile();
6987 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6988 SDValue MC = DAG.getMemcpy(Chain: Root, dl: sdl, Dst, Src, Size, DstAlign, SrcAlign,
6989 isVol, AlwaysInline: MCI.isForceInlined(), CI: &I, OverrideTailCall: std::nullopt,
6990 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
6991 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)),
6992 AAInfo: I.getAAMetadata(), BatchAA);
6993 updateDAGForMaybeTailCall(MaybeTC: MC);
6994 return;
6995 }
6996 case Intrinsic::memset:
6997 case Intrinsic::memset_inline: {
6998 const auto &MSII = cast<MemSetInst>(Val: I);
6999 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
7000 SDValue Value = getValue(V: I.getArgOperand(i: 1));
7001 SDValue Size = getValue(V: I.getArgOperand(i: 2));
7002 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
7003 "memset_inline needs constant size");
7004 // @llvm.memset defines 0 and 1 to both mean no alignment.
7005 Align DstAlign = MSII.getDestAlign().valueOrOne();
7006 bool isVol = MSII.isVolatile();
7007 SDValue Root = isVol ? getRoot() : getMemoryRoot();
7008 SDValue MC = DAG.getMemset(
7009 Chain: Root, dl: sdl, Dst, Src: Value, Size, Alignment: DstAlign, isVol, AlwaysInline: MSII.isForceInlined(),
7010 CI: &I, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)), AAInfo: I.getAAMetadata());
7011 updateDAGForMaybeTailCall(MaybeTC: MC);
7012 return;
7013 }
7014 case Intrinsic::memmove: {
7015 const auto &MMI = cast<MemMoveInst>(Val: I);
7016 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7017 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7018 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7019 // @llvm.memmove defines 0 and 1 to both mean no alignment.
7020 Align DstAlign = MMI.getDestAlign().valueOrOne();
7021 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
7022 bool isVol = MMI.isVolatile();
7023 SDValue Root = isVol ? getRoot() : getMemoryRoot();
7024 SDValue MM = DAG.getMemmove(
7025 Chain: Root, dl: sdl, Dst: Op1, Src: Op2, Size: Op3, DstAlign, SrcAlign, isVol, CI: &I,
7026 /* OverrideTailCall */ std::nullopt,
7027 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
7028 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)), AAInfo: I.getAAMetadata(), BatchAA);
7029 updateDAGForMaybeTailCall(MaybeTC: MM);
7030 return;
7031 }
7032 case Intrinsic::memcpy_element_unordered_atomic: {
7033 auto &MI = cast<AnyMemCpyInst>(Val: I);
7034 SDValue Dst = getValue(V: MI.getRawDest());
7035 SDValue Src = getValue(V: MI.getRawSource());
7036 SDValue Length = getValue(V: MI.getLength());
7037
7038 Type *LengthTy = MI.getLength()->getType();
7039 unsigned ElemSz = MI.getElementSizeInBytes();
7040 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
7041 SDValue MC =
7042 DAG.getAtomicMemcpy(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
7043 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
7044 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
7045 updateDAGForMaybeTailCall(MaybeTC: MC);
7046 return;
7047 }
7048 case Intrinsic::memmove_element_unordered_atomic: {
7049 auto &MI = cast<AnyMemMoveInst>(Val: I);
7050 SDValue Dst = getValue(V: MI.getRawDest());
7051 SDValue Src = getValue(V: MI.getRawSource());
7052 SDValue Length = getValue(V: MI.getLength());
7053
7054 Type *LengthTy = MI.getLength()->getType();
7055 unsigned ElemSz = MI.getElementSizeInBytes();
7056 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
7057 SDValue MC =
7058 DAG.getAtomicMemmove(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
7059 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
7060 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
7061 updateDAGForMaybeTailCall(MaybeTC: MC);
7062 return;
7063 }
7064 case Intrinsic::memset_element_unordered_atomic: {
7065 auto &MI = cast<AnyMemSetInst>(Val: I);
7066 SDValue Dst = getValue(V: MI.getRawDest());
7067 SDValue Val = getValue(V: MI.getValue());
7068 SDValue Length = getValue(V: MI.getLength());
7069
7070 Type *LengthTy = MI.getLength()->getType();
7071 unsigned ElemSz = MI.getElementSizeInBytes();
7072 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
7073 SDValue MC =
7074 DAG.getAtomicMemset(Chain: getRoot(), dl: sdl, Dst, Value: Val, Size: Length, SizeTy: LengthTy, ElemSz,
7075 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()));
7076 updateDAGForMaybeTailCall(MaybeTC: MC);
7077 return;
7078 }
7079 case Intrinsic::call_preallocated_setup: {
7080 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: &I);
7081 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
7082 SDValue Res = DAG.getNode(Opcode: ISD::PREALLOCATED_SETUP, DL: sdl, VT: MVT::Other,
7083 N1: getRoot(), N2: SrcValue);
7084 setValue(V: &I, NewN: Res);
7085 DAG.setRoot(Res);
7086 return;
7087 }
7088 case Intrinsic::call_preallocated_arg: {
7089 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: I.getOperand(i_nocapture: 0));
7090 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
7091 SDValue Ops[3];
7092 Ops[0] = getRoot();
7093 Ops[1] = SrcValue;
7094 Ops[2] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 1)), DL: sdl,
7095 VT: MVT::i32); // arg index
7096 SDValue Res = DAG.getNode(
7097 Opcode: ISD::PREALLOCATED_ARG, DL: sdl,
7098 VTList: DAG.getVTList(VT1: TLI.getPointerTy(DL: DAG.getDataLayout()), VT2: MVT::Other), Ops);
7099 setValue(V: &I, NewN: Res);
7100 DAG.setRoot(Res.getValue(R: 1));
7101 return;
7102 }
7103
7104 case Intrinsic::eh_typeid_for: {
7105 // Find the type id for the given typeinfo.
7106 GlobalValue *GV = ExtractTypeInfo(V: I.getArgOperand(i: 0));
7107 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(TI: GV);
7108 Res = DAG.getConstant(Val: TypeID, DL: sdl, VT: MVT::i32);
7109 setValue(V: &I, NewN: Res);
7110 return;
7111 }
7112
7113 case Intrinsic::eh_return_i32:
7114 case Intrinsic::eh_return_i64:
7115 DAG.getMachineFunction().setCallsEHReturn(true);
7116 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_RETURN, DL: sdl,
7117 VT: MVT::Other,
7118 N1: getControlRoot(),
7119 N2: getValue(V: I.getArgOperand(i: 0)),
7120 N3: getValue(V: I.getArgOperand(i: 1))));
7121 return;
7122 case Intrinsic::eh_unwind_init:
7123 DAG.getMachineFunction().setCallsUnwindInit(true);
7124 return;
7125 case Intrinsic::eh_dwarf_cfa:
7126 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EH_DWARF_CFA, DL: sdl,
7127 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
7128 Operand: getValue(V: I.getArgOperand(i: 0))));
7129 return;
7130 case Intrinsic::eh_sjlj_callsite: {
7131 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 0));
7132 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
7133
7134 FuncInfo.setCurrentCallSite(CI->getZExtValue());
7135 return;
7136 }
7137 case Intrinsic::eh_sjlj_functioncontext: {
7138 // Get and store the index of the function context.
7139 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7140 AllocaInst *FnCtx =
7141 cast<AllocaInst>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
7142 int FI = FuncInfo.StaticAllocaMap[FnCtx];
7143 MFI.setFunctionContextIndex(FI);
7144 return;
7145 }
7146 case Intrinsic::eh_sjlj_setjmp: {
7147 SDValue Ops[2];
7148 Ops[0] = getRoot();
7149 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7150 SDValue Op = DAG.getNode(Opcode: ISD::EH_SJLJ_SETJMP, DL: sdl,
7151 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
7152 setValue(V: &I, NewN: Op.getValue(R: 0));
7153 DAG.setRoot(Op.getValue(R: 1));
7154 return;
7155 }
7156 case Intrinsic::eh_sjlj_longjmp:
7157 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_SJLJ_LONGJMP, DL: sdl, VT: MVT::Other,
7158 N1: getRoot(), N2: getValue(V: I.getArgOperand(i: 0))));
7159 return;
7160 case Intrinsic::eh_sjlj_setup_dispatch:
7161 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_SJLJ_SETUP_DISPATCH, DL: sdl, VT: MVT::Other,
7162 Operand: getRoot()));
7163 return;
7164 case Intrinsic::masked_gather:
7165 visitMaskedGather(I);
7166 return;
7167 case Intrinsic::masked_load:
7168 visitMaskedLoad(I);
7169 return;
7170 case Intrinsic::masked_scatter:
7171 visitMaskedScatter(I);
7172 return;
7173 case Intrinsic::masked_store:
7174 visitMaskedStore(I);
7175 return;
7176 case Intrinsic::masked_expandload:
7177 visitMaskedLoad(I, IsExpanding: true /* IsExpanding */);
7178 return;
7179 case Intrinsic::masked_compressstore:
7180 visitMaskedStore(I, IsCompressing: true /* IsCompressing */);
7181 return;
7182 case Intrinsic::speculative_load:
7183 visitSpeculativeLoad(I);
7184 return;
7185 case Intrinsic::powi:
7186 setValue(V: &I, NewN: ExpandPowI(DL: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
7187 RHS: getValue(V: I.getArgOperand(i: 1)), DAG));
7188 return;
7189 case Intrinsic::log:
7190 setValue(V: &I, NewN: expandLog(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7191 return;
7192 case Intrinsic::log2:
7193 setValue(V: &I,
7194 NewN: expandLog2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7195 return;
7196 case Intrinsic::log10:
7197 setValue(V: &I,
7198 NewN: expandLog10(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7199 return;
7200 case Intrinsic::exp:
7201 setValue(V: &I, NewN: expandExp(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7202 return;
7203 case Intrinsic::exp2:
7204 setValue(V: &I,
7205 NewN: expandExp2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7206 return;
7207 case Intrinsic::pow:
7208 setValue(V: &I, NewN: expandPow(dl: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
7209 RHS: getValue(V: I.getArgOperand(i: 1)), DAG, TLI, Flags));
7210 return;
7211 case Intrinsic::sqrt:
7212 case Intrinsic::fabs:
7213 case Intrinsic::sin:
7214 case Intrinsic::cos:
7215 case Intrinsic::tan:
7216 case Intrinsic::asin:
7217 case Intrinsic::acos:
7218 case Intrinsic::atan:
7219 case Intrinsic::sinh:
7220 case Intrinsic::cosh:
7221 case Intrinsic::tanh:
7222 case Intrinsic::exp10:
7223 case Intrinsic::floor:
7224 case Intrinsic::ceil:
7225 case Intrinsic::trunc:
7226 case Intrinsic::rint:
7227 case Intrinsic::nearbyint:
7228 case Intrinsic::round:
7229 case Intrinsic::roundeven:
7230 case Intrinsic::canonicalize: {
7231 unsigned Opcode;
7232 // clang-format off
7233 switch (Intrinsic) {
7234 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7235 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
7236 case Intrinsic::fabs: Opcode = ISD::FABS; break;
7237 case Intrinsic::sin: Opcode = ISD::FSIN; break;
7238 case Intrinsic::cos: Opcode = ISD::FCOS; break;
7239 case Intrinsic::tan: Opcode = ISD::FTAN; break;
7240 case Intrinsic::asin: Opcode = ISD::FASIN; break;
7241 case Intrinsic::acos: Opcode = ISD::FACOS; break;
7242 case Intrinsic::atan: Opcode = ISD::FATAN; break;
7243 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
7244 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
7245 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
7246 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
7247 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
7248 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
7249 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
7250 case Intrinsic::rint: Opcode = ISD::FRINT; break;
7251 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
7252 case Intrinsic::round: Opcode = ISD::FROUND; break;
7253 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
7254 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
7255 }
7256 // clang-format on
7257
7258 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl,
7259 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7260 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
7261 return;
7262 }
7263 case Intrinsic::atan2:
7264 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FATAN2, DL: sdl,
7265 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7266 N1: getValue(V: I.getArgOperand(i: 0)),
7267 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7268 return;
7269 case Intrinsic::lround:
7270 case Intrinsic::llround:
7271 case Intrinsic::lrint:
7272 case Intrinsic::llrint: {
7273 unsigned Opcode;
7274 // clang-format off
7275 switch (Intrinsic) {
7276 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7277 case Intrinsic::lround: Opcode = ISD::LROUND; break;
7278 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
7279 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
7280 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
7281 }
7282 // clang-format on
7283
7284 EVT RetVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7285 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VT: RetVT,
7286 Operand: getValue(V: I.getArgOperand(i: 0))));
7287 return;
7288 }
7289 case Intrinsic::minnum:
7290 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINNUM, DL: sdl,
7291 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7292 N1: getValue(V: I.getArgOperand(i: 0)),
7293 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7294 return;
7295 case Intrinsic::maxnum:
7296 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXNUM, DL: sdl,
7297 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7298 N1: getValue(V: I.getArgOperand(i: 0)),
7299 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7300 return;
7301 case Intrinsic::minimum:
7302 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUM, DL: sdl,
7303 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7304 N1: getValue(V: I.getArgOperand(i: 0)),
7305 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7306 return;
7307 case Intrinsic::maximum:
7308 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUM, DL: sdl,
7309 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7310 N1: getValue(V: I.getArgOperand(i: 0)),
7311 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7312 return;
7313 case Intrinsic::minimumnum:
7314 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUMNUM, DL: sdl,
7315 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7316 N1: getValue(V: I.getArgOperand(i: 0)),
7317 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7318 return;
7319 case Intrinsic::maximumnum:
7320 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUMNUM, DL: sdl,
7321 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7322 N1: getValue(V: I.getArgOperand(i: 0)),
7323 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7324 return;
7325 case Intrinsic::copysign:
7326 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: sdl,
7327 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7328 N1: getValue(V: I.getArgOperand(i: 0)),
7329 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7330 return;
7331 case Intrinsic::ldexp:
7332 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FLDEXP, DL: sdl,
7333 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7334 N1: getValue(V: I.getArgOperand(i: 0)),
7335 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7336 return;
7337 case Intrinsic::modf:
7338 case Intrinsic::sincos:
7339 case Intrinsic::sincospi:
7340 case Intrinsic::frexp: {
7341 unsigned Opcode;
7342 switch (Intrinsic) {
7343 default:
7344 llvm_unreachable("unexpected intrinsic");
7345 case Intrinsic::sincos:
7346 Opcode = ISD::FSINCOS;
7347 break;
7348 case Intrinsic::sincospi:
7349 Opcode = ISD::FSINCOSPI;
7350 break;
7351 case Intrinsic::modf:
7352 Opcode = ISD::FMODF;
7353 break;
7354 case Intrinsic::frexp:
7355 Opcode = ISD::FFREXP;
7356 break;
7357 }
7358 SmallVector<EVT, 2> ValueVTs;
7359 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
7360 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
7361 setValue(
7362 V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: getValue(V: I.getArgOperand(i: 0)), Flags));
7363 return;
7364 }
7365 case Intrinsic::arithmetic_fence: {
7366 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: sdl,
7367 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7368 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
7369 return;
7370 }
7371 case Intrinsic::fma:
7372 setValue(V: &I, NewN: DAG.getNode(
7373 Opcode: ISD::FMA, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7374 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)),
7375 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7376 return;
7377#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7378 case Intrinsic::INTRINSIC:
7379#include "llvm/IR/ConstrainedOps.def"
7380 visitConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: I));
7381 return;
7382#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7383#include "llvm/IR/VPIntrinsics.def"
7384 visitVectorPredicationIntrinsic(VPIntrin: cast<VPIntrinsic>(Val: I));
7385 return;
7386 case Intrinsic::fptrunc_round: {
7387 // Get the last argument, the metadata and convert it to an integer in the
7388 // call
7389 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7390 std::optional<RoundingMode> RoundMode =
7391 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
7392
7393 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7394
7395 // Propagate fast-math-flags from IR to node(s).
7396 SDNodeFlags Flags;
7397 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
7398 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
7399
7400 SDValue Result;
7401 Result = DAG.getNode(
7402 Opcode: ISD::FPTRUNC_ROUND, DL: sdl, VT, N1: getValue(V: I.getArgOperand(i: 0)),
7403 N2: DAG.getTargetConstant(Val: (int)*RoundMode, DL: sdl, VT: MVT::i32));
7404 setValue(V: &I, NewN: Result);
7405
7406 return;
7407 }
7408 case Intrinsic::fmuladd: {
7409 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7410 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
7411 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMA, DL: sdl,
7412 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7413 N1: getValue(V: I.getArgOperand(i: 0)),
7414 N2: getValue(V: I.getArgOperand(i: 1)),
7415 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7416 } else if (TLI.isOperationLegalOrCustom(Op: ISD::FMULADD, VT)) {
7417 // TODO: Support splitting the vector.
7418 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMULADD, DL: sdl,
7419 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7420 N1: getValue(V: I.getArgOperand(i: 0)),
7421 N2: getValue(V: I.getArgOperand(i: 1)),
7422 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7423 } else {
7424 // TODO: Intrinsic calls should have fast-math-flags.
7425 SDValue Mul = DAG.getNode(
7426 Opcode: ISD::FMUL, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7427 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)), Flags);
7428 SDValue Add = DAG.getNode(Opcode: ISD::FADD, DL: sdl,
7429 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7430 N1: Mul, N2: getValue(V: I.getArgOperand(i: 2)), Flags);
7431 setValue(V: &I, NewN: Add);
7432 }
7433 return;
7434 }
7435 case Intrinsic::fptosi_sat: {
7436 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7437 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT_SAT, DL: sdl, VT,
7438 N1: getValue(V: I.getArgOperand(i: 0)),
7439 N2: DAG.getValueType(VT.getScalarType())));
7440 return;
7441 }
7442 case Intrinsic::fptoui_sat: {
7443 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7444 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT_SAT, DL: sdl, VT,
7445 N1: getValue(V: I.getArgOperand(i: 0)),
7446 N2: DAG.getValueType(VT.getScalarType())));
7447 return;
7448 }
7449 case Intrinsic::convert_from_arbitrary_fp: {
7450 // Extract format metadata and convert to semantics enum.
7451 EVT DstVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7452 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7453 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
7454 const fltSemantics *SrcSem =
7455 APFloatBase::getArbitraryFPSemantics(Format: FormatStr);
7456 if (!SrcSem) {
7457 DAG.getContext()->emitError(
7458 ErrorStr: "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7459 "'");
7460 setValue(V: &I, NewN: DAG.getPOISON(VT: DstVT));
7461 return;
7462 }
7463 APFloatBase::Semantics SemEnum = APFloatBase::SemanticsToEnum(Sem: *SrcSem);
7464
7465 SDValue IntVal = getValue(V: I.getArgOperand(i: 0));
7466
7467 // Emit ISD::CONVERT_FROM_ARBITRARY_FP node.
7468 SDValue SemConst =
7469 DAG.getTargetConstant(Val: static_cast<int>(SemEnum), DL: sdl, VT: MVT::i32);
7470 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: sdl, VT: DstVT, N1: IntVal,
7471 N2: SemConst));
7472 return;
7473 }
7474 case Intrinsic::convert_to_arbitrary_fp: {
7475 // Extract format metadata and convert to semantics enum.
7476 EVT DstVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7477 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7478 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
7479 const fltSemantics *DstSem =
7480 APFloatBase::getArbitraryFPSemantics(Format: FormatStr);
7481 if (!DstSem) {
7482 DAG.getContext()->emitError(
7483 ErrorStr: "convert_to_arbitrary_fp: not implemented format '" + FormatStr +
7484 "'");
7485 setValue(V: &I, NewN: DAG.getPOISON(VT: DstVT));
7486 return;
7487 }
7488 APFloatBase::Semantics SemEnum = APFloatBase::SemanticsToEnum(Sem: *DstSem);
7489
7490 Metadata *RoundMD =
7491 cast<MetadataAsValue>(Val: I.getArgOperand(i: 2))->getMetadata();
7492 StringRef RoundStr = cast<MDString>(Val: RoundMD)->getString();
7493 std::optional<RoundingMode> RoundMode = convertStrToRoundingMode(RoundStr);
7494 assert(RoundMode && *RoundMode != RoundingMode::Dynamic &&
7495 "Dynamic rounding mode should have been rejected by the verifier");
7496
7497 uint64_t Saturate =
7498 cast<ConstantInt>(Val: I.getArgOperand(i: 3))->getZExtValue() ? 1 : 0;
7499
7500 SDValue FloatVal = getValue(V: I.getArgOperand(i: 0));
7501
7502 SDValue SemConst =
7503 DAG.getTargetConstant(Val: static_cast<int>(SemEnum), DL: sdl, VT: MVT::i32);
7504 SDValue RoundConst =
7505 DAG.getTargetConstant(Val: static_cast<int>(*RoundMode), DL: sdl, VT: MVT::i32);
7506 SDValue SatConst = DAG.getTargetConstant(Val: Saturate, DL: sdl, VT: MVT::i32);
7507 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL: sdl, VT: DstVT, N1: FloatVal,
7508 N2: SemConst, N3: RoundConst, N4: SatConst));
7509 return;
7510 }
7511 case Intrinsic::set_rounding:
7512 Res = DAG.getNode(Opcode: ISD::SET_ROUNDING, DL: sdl, VT: MVT::Other,
7513 Ops: {getRoot(), getValue(V: I.getArgOperand(i: 0))});
7514 setValue(V: &I, NewN: Res);
7515 DAG.setRoot(Res.getValue(R: 0));
7516 return;
7517 case Intrinsic::is_fpclass: {
7518 const DataLayout DLayout = DAG.getDataLayout();
7519 EVT DestVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7520 EVT ArgVT = TLI.getValueType(DL: DLayout, Ty: I.getArgOperand(i: 0)->getType());
7521 FPClassTest Test = static_cast<FPClassTest>(
7522 cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue());
7523 MachineFunction &MF = DAG.getMachineFunction();
7524 const Function &F = MF.getFunction();
7525 SDValue Op = getValue(V: I.getArgOperand(i: 0));
7526 SDNodeFlags Flags;
7527 Flags.setNoFPExcept(
7528 !F.getAttributes().hasFnAttr(Kind: llvm::Attribute::StrictFP));
7529 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7530 // expansion can use illegal types. Making expansion early allows
7531 // legalizing these types prior to selection.
7532 if (!TLI.isOperationLegal(Op: ISD::IS_FPCLASS, VT: ArgVT) &&
7533 !TLI.isOperationCustom(Op: ISD::IS_FPCLASS, VT: ArgVT)) {
7534 SDValue Result = TLI.expandIS_FPCLASS(ResultVT: DestVT, Op, Test, Flags, DL: sdl, DAG);
7535 setValue(V: &I, NewN: Result);
7536 return;
7537 }
7538
7539 SDValue Check = DAG.getTargetConstant(Val: Test, DL: sdl, VT: MVT::i32);
7540 SDValue V = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: sdl, VT: DestVT, Ops: {Op, Check}, Flags);
7541 setValue(V: &I, NewN: V);
7542 return;
7543 }
7544 case Intrinsic::get_fpenv: {
7545 const DataLayout DLayout = DAG.getDataLayout();
7546 EVT EnvVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7547 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7548 SDValue Chain = getRoot();
7549 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7550 // and temporary storage in stack.
7551 if (TLI.isOperationLegalOrCustom(Op: ISD::GET_FPENV, VT: EnvVT)) {
7552 Res = DAG.getNode(
7553 Opcode: ISD::GET_FPENV, DL: sdl,
7554 VTList: DAG.getVTList(VT1: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
7555 VT2: MVT::Other),
7556 N: Chain);
7557 } else {
7558 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7559 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7560 auto MPI =
7561 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7562 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7563 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
7564 BaseAlignment: TempAlign);
7565 Chain = DAG.getGetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7566 Res = DAG.getLoad(VT: EnvVT, dl: sdl, Chain, Ptr: Temp, PtrInfo: MPI);
7567 }
7568 setValue(V: &I, NewN: Res);
7569 DAG.setRoot(Res.getValue(R: 1));
7570 return;
7571 }
7572 case Intrinsic::set_fpenv: {
7573 const DataLayout DLayout = DAG.getDataLayout();
7574 SDValue Env = getValue(V: I.getArgOperand(i: 0));
7575 EVT EnvVT = Env.getValueType();
7576 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7577 SDValue Chain = getRoot();
7578 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7579 // environment from memory.
7580 if (TLI.isOperationLegalOrCustom(Op: ISD::SET_FPENV, VT: EnvVT)) {
7581 Chain = DAG.getNode(Opcode: ISD::SET_FPENV, DL: sdl, VT: MVT::Other, N1: Chain, N2: Env);
7582 } else {
7583 // Allocate space in stack, copy environment bits into it and use this
7584 // memory in SET_FPENV_MEM.
7585 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7586 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7587 auto MPI =
7588 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7589 Chain = DAG.getStore(Chain, dl: sdl, Val: Env, Ptr: Temp, PtrInfo: MPI, Alignment: TempAlign,
7590 MMOFlags: MachineMemOperand::MOStore);
7591 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7592 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
7593 BaseAlignment: TempAlign);
7594 Chain = DAG.getSetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7595 }
7596 DAG.setRoot(Chain);
7597 return;
7598 }
7599 case Intrinsic::reset_fpenv:
7600 DAG.setRoot(DAG.getNode(Opcode: ISD::RESET_FPENV, DL: sdl, VT: MVT::Other, Operand: getRoot()));
7601 return;
7602 case Intrinsic::get_fpmode:
7603 Res = DAG.getNode(
7604 Opcode: ISD::GET_FPMODE, DL: sdl,
7605 VTList: DAG.getVTList(VT1: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
7606 VT2: MVT::Other),
7607 N: DAG.getRoot());
7608 setValue(V: &I, NewN: Res);
7609 DAG.setRoot(Res.getValue(R: 1));
7610 return;
7611 case Intrinsic::set_fpmode:
7612 Res = DAG.getNode(Opcode: ISD::SET_FPMODE, DL: sdl, VT: MVT::Other, N1: {DAG.getRoot()},
7613 N2: getValue(V: I.getArgOperand(i: 0)));
7614 DAG.setRoot(Res);
7615 return;
7616 case Intrinsic::reset_fpmode: {
7617 Res = DAG.getNode(Opcode: ISD::RESET_FPMODE, DL: sdl, VT: MVT::Other, Operand: getRoot());
7618 DAG.setRoot(Res);
7619 return;
7620 }
7621 case Intrinsic::pcmarker: {
7622 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
7623 DAG.setRoot(DAG.getNode(Opcode: ISD::PCMARKER, DL: sdl, VT: MVT::Other, N1: getRoot(), N2: Tmp));
7624 return;
7625 }
7626 case Intrinsic::readcyclecounter: {
7627 SDValue Op = getRoot();
7628 Res = DAG.getNode(Opcode: ISD::READCYCLECOUNTER, DL: sdl,
7629 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N: Op);
7630 setValue(V: &I, NewN: Res);
7631 DAG.setRoot(Res.getValue(R: 1));
7632 return;
7633 }
7634 case Intrinsic::readsteadycounter: {
7635 SDValue Op = getRoot();
7636 Res = DAG.getNode(Opcode: ISD::READSTEADYCOUNTER, DL: sdl,
7637 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N: Op);
7638 setValue(V: &I, NewN: Res);
7639 DAG.setRoot(Res.getValue(R: 1));
7640 return;
7641 }
7642 case Intrinsic::bitreverse:
7643 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITREVERSE, DL: sdl,
7644 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7645 Operand: getValue(V: I.getArgOperand(i: 0))));
7646 return;
7647 case Intrinsic::bswap:
7648 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BSWAP, DL: sdl,
7649 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7650 Operand: getValue(V: I.getArgOperand(i: 0))));
7651 return;
7652 case Intrinsic::cttz: {
7653 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7654 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7655 EVT Ty = Arg.getValueType();
7656 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_POISON,
7657 DL: sdl, VT: Ty, Operand: Arg));
7658 return;
7659 }
7660 case Intrinsic::ctlz: {
7661 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7662 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7663 EVT Ty = Arg.getValueType();
7664 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_POISON,
7665 DL: sdl, VT: Ty, Operand: Arg));
7666 return;
7667 }
7668 case Intrinsic::ctpop: {
7669 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7670 EVT Ty = Arg.getValueType();
7671 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CTPOP, DL: sdl, VT: Ty, Operand: Arg));
7672 return;
7673 }
7674 case Intrinsic::fshl:
7675 case Intrinsic::fshr: {
7676 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7677 SDValue X = getValue(V: I.getArgOperand(i: 0));
7678 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7679 SDValue Z = getValue(V: I.getArgOperand(i: 2));
7680 EVT VT = X.getValueType();
7681
7682 if (X == Y) {
7683 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7684 setValue(V: &I, NewN: DAG.getNode(Opcode: RotateOpcode, DL: sdl, VT, N1: X, N2: Z));
7685 } else {
7686 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7687 setValue(V: &I, NewN: DAG.getNode(Opcode: FunnelOpcode, DL: sdl, VT, N1: X, N2: Y, N3: Z));
7688 }
7689 return;
7690 }
7691 case Intrinsic::clmul: {
7692 SDValue X = getValue(V: I.getArgOperand(i: 0));
7693 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7694 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CLMUL, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7695 return;
7696 }
7697 case Intrinsic::pext: {
7698 SDValue X = getValue(V: I.getArgOperand(i: 0));
7699 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7700 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::PEXT, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7701 return;
7702 }
7703 case Intrinsic::pdep: {
7704 SDValue X = getValue(V: I.getArgOperand(i: 0));
7705 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7706 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::PDEP, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7707 return;
7708 }
7709 case Intrinsic::smulh:
7710 case Intrinsic::umulh: {
7711 auto Opc = Intrinsic == Intrinsic::smulh ? ISD::MULHS : ISD::MULHU;
7712 SDValue X = getValue(V: I.getArgOperand(i: 0));
7713 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7714 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7715 return;
7716 }
7717 case Intrinsic::sadd_sat: {
7718 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7719 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7720 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7721 return;
7722 }
7723 case Intrinsic::uadd_sat: {
7724 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7725 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7726 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7727 return;
7728 }
7729 case Intrinsic::ssub_sat: {
7730 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7731 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7732 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SSUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7733 return;
7734 }
7735 case Intrinsic::usub_sat: {
7736 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7737 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7738 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::USUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7739 return;
7740 }
7741 case Intrinsic::sshl_sat:
7742 case Intrinsic::ushl_sat: {
7743 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7744 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7745
7746 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
7747 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
7748
7749 // Coerce the shift amount to the right type if we can. This exposes the
7750 // truncate or zext to optimization early.
7751 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
7752 assert(ShiftTy.getSizeInBits() >=
7753 Log2_32_Ceil(Op1.getValueSizeInBits()) &&
7754 "Unexpected shift type");
7755 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
7756 }
7757
7758 unsigned Opc =
7759 Intrinsic == Intrinsic::sshl_sat ? ISD::SSHLSAT : ISD::USHLSAT;
7760 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7761 return;
7762 }
7763 case Intrinsic::smul_fix:
7764 case Intrinsic::umul_fix:
7765 case Intrinsic::smul_fix_sat:
7766 case Intrinsic::umul_fix_sat: {
7767 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7768 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7769 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7770 setValue(V: &I, NewN: DAG.getNode(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7771 VT: Op1.getValueType(), N1: Op1, N2: Op2, N3: Op3));
7772 return;
7773 }
7774 case Intrinsic::sdiv_fix:
7775 case Intrinsic::udiv_fix:
7776 case Intrinsic::sdiv_fix_sat:
7777 case Intrinsic::udiv_fix_sat: {
7778 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7779 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7780 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7781 setValue(V: &I, NewN: expandDivFix(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7782 LHS: Op1, RHS: Op2, Scale: Op3, DAG, TLI));
7783 return;
7784 }
7785 case Intrinsic::smax: {
7786 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7787 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7788 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7789 return;
7790 }
7791 case Intrinsic::smin: {
7792 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7793 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7794 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7795 return;
7796 }
7797 case Intrinsic::umax: {
7798 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7799 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7800 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7801 return;
7802 }
7803 case Intrinsic::umin: {
7804 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7805 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7806 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7807 return;
7808 }
7809 case Intrinsic::abs: {
7810 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7811 bool IntMinIsPoison = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->isOne();
7812 unsigned Opc = IntMinIsPoison ? ISD::ABS_MIN_POISON : ISD::ABS;
7813 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: Op1.getValueType(), Operand: Op1));
7814 return;
7815 }
7816 case Intrinsic::scmp: {
7817 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7818 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7819 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7820 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7821 break;
7822 }
7823 case Intrinsic::ucmp: {
7824 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7825 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7826 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7827 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7828 break;
7829 }
7830 case Intrinsic::stackaddress:
7831 case Intrinsic::stacksave: {
7832 unsigned SDOpcode = Intrinsic == Intrinsic::stackaddress ? ISD::STACKADDRESS
7833 : ISD::STACKSAVE;
7834 SDValue Op = getRoot();
7835 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7836 Res = DAG.getNode(Opcode: SDOpcode, DL: sdl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), N: Op);
7837 setValue(V: &I, NewN: Res);
7838 DAG.setRoot(Res.getValue(R: 1));
7839 return;
7840 }
7841 case Intrinsic::stackrestore:
7842 Res = getValue(V: I.getArgOperand(i: 0));
7843 DAG.setRoot(DAG.getNode(Opcode: ISD::STACKRESTORE, DL: sdl, VT: MVT::Other, N1: getRoot(), N2: Res));
7844 return;
7845 case Intrinsic::get_dynamic_area_offset: {
7846 SDValue Op = getRoot();
7847 EVT ResTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7848 Res = DAG.getNode(Opcode: ISD::GET_DYNAMIC_AREA_OFFSET, DL: sdl, VTList: DAG.getVTList(VT: ResTy),
7849 N: Op);
7850 DAG.setRoot(Op);
7851 setValue(V: &I, NewN: Res);
7852 return;
7853 }
7854 case Intrinsic::stackguard: {
7855 MachineFunction &MF = DAG.getMachineFunction();
7856 const Module &M = *MF.getFunction().getParent();
7857 EVT PtrTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7858 SDValue Chain = getRoot();
7859 if (TLI.useLoadStackGuardNode(M)) {
7860 Res = getLoadStackGuard(DAG, DL: sdl, Chain);
7861 Res = DAG.getPtrExtOrTrunc(Op: Res, DL: sdl, VT: PtrTy);
7862 } else {
7863 const Value *Global = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls());
7864 if (!Global) {
7865 LLVMContext &Ctx = *DAG.getContext();
7866 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
7867 setValue(V: &I, NewN: DAG.getPOISON(VT: PtrTy));
7868 return;
7869 }
7870
7871 Align Align = DAG.getDataLayout().getPrefTypeAlign(Ty: Global->getType());
7872 Res = DAG.getLoad(VT: PtrTy, dl: sdl, Chain, Ptr: getValue(V: Global),
7873 PtrInfo: MachinePointerInfo(Global, 0), Alignment: Align,
7874 MMOFlags: MachineMemOperand::MOVolatile);
7875 }
7876 // Mix the cookie with FP if enabled. Skip if using LOAD_STACK_GUARD
7877 // with post-RA mixing (AArch64 MSVCRT), as the mixing will be done during
7878 // post-RA expansion of LOAD_STACK_GUARD.
7879 if (TLI.useStackGuardMixFP() && !TLI.useLoadStackGuardNode(M))
7880 Res = TLI.emitStackGuardMixFP(DAG, Val: Res, DL: sdl);
7881 DAG.setRoot(Chain);
7882 setValue(V: &I, NewN: Res);
7883 return;
7884 }
7885 case Intrinsic::stackprotector: {
7886 // Emit code into the DAG to store the stack guard onto the stack.
7887 MachineFunction &MF = DAG.getMachineFunction();
7888 MachineFrameInfo &MFI = MF.getFrameInfo();
7889 const Module &M = *MF.getFunction().getParent();
7890 SDValue Src, Chain = getRoot();
7891
7892 if (TLI.useLoadStackGuardNode(M))
7893 Src = getLoadStackGuard(DAG, DL: sdl, Chain);
7894 else
7895 Src = getValue(V: I.getArgOperand(i: 0)); // The guard's value.
7896
7897 AllocaInst *Slot = cast<AllocaInst>(Val: I.getArgOperand(i: 1));
7898
7899 int FI = FuncInfo.StaticAllocaMap[Slot];
7900 MFI.setStackProtectorIndex(FI);
7901 EVT PtrTy = TLI.getFrameIndexTy(DL: DAG.getDataLayout());
7902
7903 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrTy);
7904
7905 // Store the stack protector onto the stack.
7906 Res = DAG.getStore(
7907 Chain, dl: sdl, Val: Src, Ptr: FIN,
7908 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI),
7909 Alignment: MaybeAlign(), MMOFlags: MachineMemOperand::MOVolatile);
7910 setValue(V: &I, NewN: Res);
7911 DAG.setRoot(Res);
7912 return;
7913 }
7914 case Intrinsic::objectsize:
7915 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7916
7917 case Intrinsic::is_constant:
7918 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7919
7920 case Intrinsic::annotation:
7921 case Intrinsic::ptr_annotation:
7922 case Intrinsic::launder_invariant_group:
7923 // Drop the intrinsic, but forward the value
7924 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
7925 return;
7926
7927 case Intrinsic::type_test:
7928 case Intrinsic::public_type_test:
7929 case Intrinsic::type_checked_load:
7930 case Intrinsic::type_checked_load_relative: {
7931 // These intrinsics are expected to be lowered by the LowerTypeTests pass
7932 // before code generation. Surviving until here usually indicates a
7933 // misconfiguration, for instance when devirtualization is enabled but LTO
7934 // does not actually run.
7935 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7936 *I.getFunction(),
7937 Intrinsic::getBaseName(id: Intrinsic) +
7938 " intrinsic must be lowered by the LowerTypeTests pass "
7939 "before code generation",
7940 sdl.getDebugLoc()));
7941
7942 // Lower the result to poison so that compilation can continue and collect
7943 // any further diagnostics.
7944 setValueToPoison(V: &I, dl: sdl);
7945 return;
7946 }
7947
7948 case Intrinsic::assume:
7949 case Intrinsic::experimental_noalias_scope_decl:
7950 case Intrinsic::var_annotation:
7951 case Intrinsic::sideeffect:
7952 // Discard annotate attributes, noalias scope declarations, assumptions, and
7953 // artificial side-effects.
7954 return;
7955
7956 case Intrinsic::codeview_annotation: {
7957 // Emit a label associated with this metadata.
7958 MachineFunction &MF = DAG.getMachineFunction();
7959 MCSymbol *Label = MF.getContext().createTempSymbol(Name: "annotation", AlwaysAddSuffix: true);
7960 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
7961 MF.addCodeViewAnnotation(Label, MD: cast<MDNode>(Val: MD));
7962 Res = DAG.getLabelNode(Opcode: ISD::ANNOTATION_LABEL, dl: sdl, Root: getRoot(), Label);
7963 DAG.setRoot(Res);
7964 return;
7965 }
7966
7967 case Intrinsic::init_trampoline: {
7968 const Function *F = cast<Function>(Val: I.getArgOperand(i: 1)->stripPointerCasts());
7969
7970 SDValue Ops[6];
7971 Ops[0] = getRoot();
7972 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7973 Ops[2] = getValue(V: I.getArgOperand(i: 1));
7974 Ops[3] = getValue(V: I.getArgOperand(i: 2));
7975 Ops[4] = DAG.getSrcValue(v: I.getArgOperand(i: 0));
7976 Ops[5] = DAG.getSrcValue(v: F);
7977
7978 Res = DAG.getNode(Opcode: ISD::INIT_TRAMPOLINE, DL: sdl, VT: MVT::Other, Ops);
7979
7980 DAG.setRoot(Res);
7981 return;
7982 }
7983 case Intrinsic::adjust_trampoline:
7984 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ADJUST_TRAMPOLINE, DL: sdl,
7985 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
7986 Operand: getValue(V: I.getArgOperand(i: 0))));
7987 return;
7988 case Intrinsic::gcroot: {
7989 assert(DAG.getMachineFunction().getFunction().hasGC() &&
7990 "only valid in functions with gc specified, enforced by Verifier");
7991 assert(GFI && "implied by previous");
7992 const Value *Alloca = I.getArgOperand(i: 0)->stripPointerCasts();
7993 const Constant *TypeMap = cast<Constant>(Val: I.getArgOperand(i: 1));
7994
7995 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(Val: getValue(V: Alloca).getNode());
7996 GFI->addStackRoot(Num: FI->getIndex(), Metadata: TypeMap);
7997 return;
7998 }
7999 case Intrinsic::gcread:
8000 case Intrinsic::gcwrite:
8001 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
8002 case Intrinsic::get_rounding:
8003 Res = DAG.getNode(Opcode: ISD::GET_ROUNDING, DL: sdl, ResultTys: {MVT::i32, MVT::Other}, Ops: getRoot());
8004 setValue(V: &I, NewN: Res);
8005 DAG.setRoot(Res.getValue(R: 1));
8006 return;
8007
8008 case Intrinsic::expect:
8009 case Intrinsic::expect_with_probability:
8010 // Just replace __builtin_expect(exp, c) and
8011 // __builtin_expect_with_probability(exp, c, p) with EXP.
8012 setValue(V: &I, NewN: getValue(V: I.getArgOperand(i: 0)));
8013 return;
8014
8015 case Intrinsic::ubsantrap:
8016 case Intrinsic::debugtrap:
8017 case Intrinsic::trap: {
8018 StringRef TrapFuncName =
8019 I.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
8020 if (TrapFuncName.empty()) {
8021 switch (Intrinsic) {
8022 case Intrinsic::trap:
8023 DAG.setRoot(DAG.getNode(Opcode: ISD::TRAP, DL: sdl, VT: MVT::Other, Operand: getRoot()));
8024 break;
8025 case Intrinsic::debugtrap:
8026 DAG.setRoot(DAG.getNode(Opcode: ISD::DEBUGTRAP, DL: sdl, VT: MVT::Other, Operand: getRoot()));
8027 break;
8028 case Intrinsic::ubsantrap:
8029 DAG.setRoot(DAG.getNode(
8030 Opcode: ISD::UBSANTRAP, DL: sdl, VT: MVT::Other, N1: getRoot(),
8031 N2: DAG.getTargetConstant(
8032 Val: cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue(), DL: sdl,
8033 VT: MVT::i32)));
8034 break;
8035 default: llvm_unreachable("unknown trap intrinsic");
8036 }
8037 DAG.addNoMergeSiteInfo(Node: DAG.getRoot().getNode(),
8038 NoMerge: I.hasFnAttr(Kind: Attribute::NoMerge));
8039 return;
8040 }
8041 TargetLowering::ArgListTy Args;
8042 if (Intrinsic == Intrinsic::ubsantrap) {
8043 Value *Arg = I.getArgOperand(i: 0);
8044 Args.emplace_back(args&: Arg, args: getValue(V: Arg));
8045 }
8046
8047 TargetLowering::CallLoweringInfo CLI(DAG);
8048 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
8049 CC: CallingConv::C, ResultType: I.getType(),
8050 Target: DAG.getExternalSymbol(Sym: TrapFuncName.data(),
8051 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
8052 ArgsList: std::move(Args));
8053 CLI.NoMerge = I.hasFnAttr(Kind: Attribute::NoMerge);
8054 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8055 DAG.setRoot(Result.second);
8056 return;
8057 }
8058
8059 case Intrinsic::allow_runtime_check:
8060 case Intrinsic::allow_ubsan_check:
8061 setValue(V: &I, NewN: getValue(V: ConstantInt::getTrue(Ty: I.getType())));
8062 return;
8063
8064 case Intrinsic::uadd_with_overflow:
8065 case Intrinsic::sadd_with_overflow:
8066 case Intrinsic::usub_with_overflow:
8067 case Intrinsic::ssub_with_overflow:
8068 case Intrinsic::umul_with_overflow:
8069 case Intrinsic::smul_with_overflow: {
8070 ISD::NodeType Op;
8071 switch (Intrinsic) {
8072 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8073 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
8074 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
8075 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
8076 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
8077 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
8078 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
8079 }
8080 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
8081 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
8082
8083 EVT ResultVT = Op1.getValueType();
8084 EVT OverflowVT = ResultVT.changeElementType(Context&: *Context, EltVT: MVT::i1);
8085
8086 SDVTList VTs = DAG.getVTList(VT1: ResultVT, VT2: OverflowVT);
8087 setValue(V: &I, NewN: DAG.getNode(Opcode: Op, DL: sdl, VTList: VTs, N1: Op1, N2: Op2));
8088 return;
8089 }
8090 case Intrinsic::prefetch: {
8091 SDValue Ops[5];
8092 unsigned rw = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
8093 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
8094 Ops[0] = DAG.getRoot();
8095 Ops[1] = getValue(V: I.getArgOperand(i: 0));
8096 Ops[2] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 1)), DL: sdl,
8097 VT: MVT::i32);
8098 Ops[3] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 2)), DL: sdl,
8099 VT: MVT::i32);
8100 Ops[4] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 3)), DL: sdl,
8101 VT: MVT::i32);
8102 SDValue Result = DAG.getMemIntrinsicNode(
8103 Opcode: ISD::PREFETCH, dl: sdl, VTList: DAG.getVTList(VT: MVT::Other), Ops,
8104 MemVT: EVT::getIntegerVT(Context&: *Context, BitWidth: 8), PtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
8105 /* align */ Alignment: std::nullopt, Flags);
8106
8107 // Chain the prefetch in parallel with any pending loads, to stay out of
8108 // the way of later optimizations.
8109 PendingLoads.push_back(Elt: Result);
8110 Result = getRoot();
8111 DAG.setRoot(Result);
8112 return;
8113 }
8114 case Intrinsic::lifetime_start:
8115 case Intrinsic::lifetime_end: {
8116 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
8117 // Stack coloring is not enabled in O0, discard region information.
8118 if (TM.getOptLevel() == CodeGenOptLevel::None)
8119 return;
8120
8121 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(Val: I.getArgOperand(i: 0));
8122 if (!LifetimeObject)
8123 return;
8124
8125 // First check that the Alloca is static, otherwise it won't have a
8126 // valid frame index.
8127 auto SI = FuncInfo.StaticAllocaMap.find(Val: LifetimeObject);
8128 if (SI == FuncInfo.StaticAllocaMap.end())
8129 return;
8130
8131 const int FrameIndex = SI->second;
8132 Res = DAG.getLifetimeNode(IsStart, dl: sdl, Chain: getRoot(), FrameIndex);
8133 DAG.setRoot(Res);
8134 return;
8135 }
8136 case Intrinsic::pseudoprobe: {
8137 auto Guid = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue();
8138 auto Index = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
8139 auto Attr = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
8140 Res = DAG.getPseudoProbeNode(Dl: sdl, Chain: getRoot(), Guid, Index, Attr);
8141 DAG.setRoot(Res);
8142 return;
8143 }
8144 case Intrinsic::invariant_start:
8145 // Discard region information.
8146 setValue(V: &I,
8147 NewN: DAG.getUNDEF(VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
8148 return;
8149 case Intrinsic::invariant_end:
8150 // Discard region information.
8151 return;
8152 case Intrinsic::clear_cache: {
8153 SDValue InputChain = DAG.getRoot();
8154 SDValue StartVal = getValue(V: I.getArgOperand(i: 0));
8155 SDValue EndVal = getValue(V: I.getArgOperand(i: 1));
8156 Res = DAG.getNode(Opcode: ISD::CLEAR_CACHE, DL: sdl, VTList: DAG.getVTList(VT: MVT::Other),
8157 Ops: {InputChain, StartVal, EndVal});
8158 setValue(V: &I, NewN: Res);
8159 DAG.setRoot(Res);
8160 return;
8161 }
8162 case Intrinsic::donothing:
8163 case Intrinsic::seh_try_begin:
8164 case Intrinsic::seh_scope_begin:
8165 case Intrinsic::seh_try_end:
8166 case Intrinsic::seh_scope_end:
8167 // ignore
8168 return;
8169 case Intrinsic::experimental_stackmap:
8170 visitStackmap(I);
8171 return;
8172 case Intrinsic::experimental_patchpoint_void:
8173 case Intrinsic::experimental_patchpoint:
8174 visitPatchpoint(CB: I);
8175 return;
8176 case Intrinsic::experimental_gc_statepoint:
8177 LowerStatepoint(I: cast<GCStatepointInst>(Val: I));
8178 return;
8179 case Intrinsic::experimental_gc_result:
8180 visitGCResult(I: cast<GCResultInst>(Val: I));
8181 return;
8182 case Intrinsic::experimental_gc_relocate:
8183 visitGCRelocate(Relocate: cast<GCRelocateInst>(Val: I));
8184 return;
8185 case Intrinsic::instrprof_cover:
8186 llvm_unreachable("instrprof failed to lower a cover");
8187 case Intrinsic::instrprof_increment:
8188 llvm_unreachable("instrprof failed to lower an increment");
8189 case Intrinsic::instrprof_timestamp:
8190 llvm_unreachable("instrprof failed to lower a timestamp");
8191 case Intrinsic::instrprof_value_profile:
8192 llvm_unreachable("instrprof failed to lower a value profiling call");
8193 case Intrinsic::instrprof_mcdc_parameters:
8194 llvm_unreachable("instrprof failed to lower mcdc parameters");
8195 case Intrinsic::instrprof_mcdc_tvbitmap_update:
8196 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
8197 case Intrinsic::localescape: {
8198 MachineFunction &MF = DAG.getMachineFunction();
8199 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
8200
8201 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
8202 // is the same on all targets.
8203 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
8204 Value *Arg = I.getArgOperand(i: Idx)->stripPointerCasts();
8205 if (isa<ConstantPointerNull>(Val: Arg))
8206 continue; // Skip null pointers. They represent a hole in index space.
8207 AllocaInst *Slot = cast<AllocaInst>(Val: Arg);
8208 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
8209 "can only escape static allocas");
8210 int FI = FuncInfo.StaticAllocaMap[Slot];
8211 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8212 FuncName: GlobalValue::dropLLVMManglingEscape(Name: MF.getName()), Idx);
8213 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: dl,
8214 MCID: TII->get(Opcode: TargetOpcode::LOCAL_ESCAPE))
8215 .addSym(Sym: FrameAllocSym)
8216 .addFrameIndex(Idx: FI);
8217 }
8218
8219 return;
8220 }
8221
8222 case Intrinsic::localrecover: {
8223 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
8224 MachineFunction &MF = DAG.getMachineFunction();
8225
8226 // Get the symbol that defines the frame offset.
8227 auto *Fn = cast<Function>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
8228 auto *Idx = cast<ConstantInt>(Val: I.getArgOperand(i: 2));
8229 unsigned IdxVal =
8230 unsigned(Idx->getLimitedValue(Limit: std::numeric_limits<int>::max()));
8231 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8232 FuncName: GlobalValue::dropLLVMManglingEscape(Name: Fn->getName()), Idx: IdxVal);
8233
8234 Value *FP = I.getArgOperand(i: 1);
8235 SDValue FPVal = getValue(V: FP);
8236 EVT PtrVT = FPVal.getValueType();
8237
8238 // Create a MCSymbol for the label to avoid any target lowering
8239 // that would make this PC relative.
8240 SDValue OffsetSym = DAG.getMCSymbol(Sym: FrameAllocSym, VT: PtrVT);
8241 SDValue OffsetVal =
8242 DAG.getNode(Opcode: ISD::LOCAL_RECOVER, DL: sdl, VT: PtrVT, Operand: OffsetSym);
8243
8244 // Add the offset to the FP.
8245 SDValue Add = DAG.getMemBasePlusOffset(Base: FPVal, Offset: OffsetVal, DL: sdl);
8246 setValue(V: &I, NewN: Add);
8247
8248 return;
8249 }
8250
8251 case Intrinsic::fake_use: {
8252 Value *V = I.getArgOperand(i: 0);
8253 SDValue Ops[2];
8254 // For Values not declared or previously used in this basic block, the
8255 // NodeMap will not have an entry, and `getValue` will assert if V has no
8256 // valid register value.
8257 auto FakeUseValue = [&]() -> SDValue {
8258 SDValue &N = NodeMap[V];
8259 if (N.getNode())
8260 return N;
8261
8262 // If there's a virtual register allocated and initialized for this
8263 // value, use it.
8264 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
8265 return copyFromReg;
8266 // FIXME: Do we want to preserve constants? It seems pointless.
8267 if (isa<Constant>(Val: V))
8268 return getValue(V);
8269 return SDValue();
8270 }();
8271 if (!FakeUseValue || FakeUseValue.isUndef())
8272 return;
8273 Ops[0] = getRoot();
8274 Ops[1] = FakeUseValue;
8275 // Also, do not translate a fake use with an undef operand, or any other
8276 // empty SDValues.
8277 if (!Ops[1] || Ops[1].isUndef())
8278 return;
8279 DAG.setRoot(DAG.getNode(Opcode: ISD::FAKE_USE, DL: sdl, VT: MVT::Other, Ops));
8280 return;
8281 }
8282
8283 case Intrinsic::reloc_none: {
8284 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
8285 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
8286 SDValue Ops[2] = {
8287 getRoot(),
8288 DAG.getTargetExternalSymbol(
8289 Sym: SymbolName.data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout()))};
8290 DAG.setRoot(DAG.getNode(Opcode: ISD::RELOC_NONE, DL: sdl, VT: MVT::Other, Ops));
8291 return;
8292 }
8293
8294 case Intrinsic::cond_loop: {
8295 SDValue InputChain = DAG.getRoot();
8296 SDValue P = getValue(V: I.getArgOperand(i: 0));
8297 Res = DAG.getNode(Opcode: ISD::COND_LOOP, DL: sdl, VTList: DAG.getVTList(VT: MVT::Other),
8298 Ops: {InputChain, P});
8299 setValue(V: &I, NewN: Res);
8300 DAG.setRoot(Res);
8301 return;
8302 }
8303
8304 case Intrinsic::eh_exceptionpointer:
8305 case Intrinsic::eh_exceptioncode: {
8306 // Get the exception pointer vreg, copy from it, and resize it to fit.
8307 const auto *CPI = cast<CatchPadInst>(Val: I.getArgOperand(i: 0));
8308 MVT PtrVT = TLI.getPointerTy(DL: DAG.getDataLayout());
8309 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(VT: PtrVT);
8310 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, RC: PtrRC);
8311 SDValue N = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: sdl, Reg: VReg, VT: PtrVT);
8312 if (Intrinsic == Intrinsic::eh_exceptioncode)
8313 N = DAG.getZExtOrTrunc(Op: N, DL: sdl, VT: MVT::i32);
8314 setValue(V: &I, NewN: N);
8315 return;
8316 }
8317 case Intrinsic::xray_customevent: {
8318 // Here we want to make sure that the intrinsic behaves as if it has a
8319 // specific calling convention.
8320 const auto &Triple = DAG.getTarget().getTargetTriple();
8321 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64 &&
8322 Triple.getArch() != Triple::hexagon)
8323 return;
8324
8325 SmallVector<SDValue, 8> Ops;
8326
8327 // We want to say that we always want the arguments in registers.
8328 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 0));
8329 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 1));
8330 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
8331 SDValue Chain = getRoot();
8332 Ops.push_back(Elt: LogEntryVal);
8333 Ops.push_back(Elt: StrSizeVal);
8334 Ops.push_back(Elt: Chain);
8335
8336 // We need to enforce the calling convention for the callsite, so that
8337 // argument ordering is enforced correctly, and that register allocation can
8338 // see that some registers may be assumed clobbered and have to preserve
8339 // them across calls to the intrinsic.
8340 MachineSDNode *MN = DAG.getMachineNode(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL,
8341 dl: sdl, VTs: NodeTys, Ops);
8342 SDValue patchableNode = SDValue(MN, 0);
8343 DAG.setRoot(patchableNode);
8344 setValue(V: &I, NewN: patchableNode);
8345 return;
8346 }
8347 case Intrinsic::xray_typedevent: {
8348 // Here we want to make sure that the intrinsic behaves as if it has a
8349 // specific calling convention.
8350 const auto &Triple = DAG.getTarget().getTargetTriple();
8351 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64 &&
8352 Triple.getArch() != Triple::hexagon)
8353 return;
8354
8355 SmallVector<SDValue, 8> Ops;
8356
8357 // We want to say that we always want the arguments in registers.
8358 // It's unclear to me how manipulating the selection DAG here forces callers
8359 // to provide arguments in registers instead of on the stack.
8360 SDValue LogTypeId = getValue(V: I.getArgOperand(i: 0));
8361 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 1));
8362 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 2));
8363 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
8364 SDValue Chain = getRoot();
8365 Ops.push_back(Elt: LogTypeId);
8366 Ops.push_back(Elt: LogEntryVal);
8367 Ops.push_back(Elt: StrSizeVal);
8368 Ops.push_back(Elt: Chain);
8369
8370 // We need to enforce the calling convention for the callsite, so that
8371 // argument ordering is enforced correctly, and that register allocation can
8372 // see that some registers may be assumed clobbered and have to preserve
8373 // them across calls to the intrinsic.
8374 MachineSDNode *MN = DAG.getMachineNode(
8375 Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, dl: sdl, VTs: NodeTys, Ops);
8376 SDValue patchableNode = SDValue(MN, 0);
8377 DAG.setRoot(patchableNode);
8378 setValue(V: &I, NewN: patchableNode);
8379 return;
8380 }
8381 case Intrinsic::experimental_deoptimize:
8382 LowerDeoptimizeCall(CI: &I);
8383 return;
8384 case Intrinsic::stepvector:
8385 visitStepVector(I);
8386 return;
8387 case Intrinsic::vector_reduce_fadd:
8388 case Intrinsic::vector_reduce_fmul:
8389 case Intrinsic::vector_reduce_add:
8390 case Intrinsic::vector_reduce_mul:
8391 case Intrinsic::vector_reduce_and:
8392 case Intrinsic::vector_reduce_or:
8393 case Intrinsic::vector_reduce_xor:
8394 case Intrinsic::vector_reduce_smax:
8395 case Intrinsic::vector_reduce_smin:
8396 case Intrinsic::vector_reduce_umax:
8397 case Intrinsic::vector_reduce_umin:
8398 case Intrinsic::vector_reduce_fmax:
8399 case Intrinsic::vector_reduce_fmin:
8400 case Intrinsic::vector_reduce_fmaximum:
8401 case Intrinsic::vector_reduce_fminimum:
8402 case Intrinsic::vector_reduce_fmaximumnum:
8403 case Intrinsic::vector_reduce_fminimumnum:
8404 visitVectorReduce(I, Intrinsic);
8405 return;
8406
8407 case Intrinsic::icall_branch_funnel: {
8408 SmallVector<SDValue, 16> Ops;
8409 Ops.push_back(Elt: getValue(V: I.getArgOperand(i: 0)));
8410
8411 int64_t Offset;
8412 auto *Base = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
8413 Ptr: I.getArgOperand(i: 1), Offset, DL: DAG.getDataLayout()));
8414 if (!Base)
8415 report_fatal_error(
8416 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
8417 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Base, DL: sdl, VT: MVT::i64, offset: 0));
8418
8419 struct BranchFunnelTarget {
8420 int64_t Offset;
8421 SDValue Target;
8422 };
8423 SmallVector<BranchFunnelTarget, 8> Targets;
8424
8425 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
8426 auto *ElemBase = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
8427 Ptr: I.getArgOperand(i: Op), Offset, DL: DAG.getDataLayout()));
8428 if (ElemBase != Base)
8429 report_fatal_error(reason: "all llvm.icall.branch.funnel operands must refer "
8430 "to the same GlobalValue");
8431
8432 SDValue Val = getValue(V: I.getArgOperand(i: Op + 1));
8433 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
8434 if (!GA)
8435 report_fatal_error(
8436 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
8437 Targets.push_back(Elt: {.Offset: Offset, .Target: DAG.getTargetGlobalAddress(
8438 GV: GA->getGlobal(), DL: sdl, VT: Val.getValueType(),
8439 offset: GA->getOffset())});
8440 }
8441 llvm::sort(C&: Targets,
8442 Comp: [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
8443 return T1.Offset < T2.Offset;
8444 });
8445
8446 for (auto &T : Targets) {
8447 Ops.push_back(Elt: DAG.getTargetConstant(Val: T.Offset, DL: sdl, VT: MVT::i32));
8448 Ops.push_back(Elt: T.Target);
8449 }
8450
8451 Ops.push_back(Elt: DAG.getRoot()); // Chain
8452 SDValue N(DAG.getMachineNode(Opcode: TargetOpcode::ICALL_BRANCH_FUNNEL, dl: sdl,
8453 VT: MVT::Other, Ops),
8454 0);
8455 DAG.setRoot(N);
8456 setValue(V: &I, NewN: N);
8457 HasTailCall = true;
8458 return;
8459 }
8460
8461 case Intrinsic::wasm_landingpad_index:
8462 // Information this intrinsic contained has been transferred to
8463 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
8464 // delete it now.
8465 return;
8466
8467 case Intrinsic::aarch64_settag:
8468 case Intrinsic::aarch64_settag_zero: {
8469 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8470 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
8471 SDValue Val = TSI.EmitTargetCodeForSetTag(
8472 DAG, dl: sdl, Chain: getRoot(), Addr: getValue(V: I.getArgOperand(i: 0)),
8473 Size: getValue(V: I.getArgOperand(i: 1)), DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
8474 ZeroData: ZeroMemory);
8475 DAG.setRoot(Val);
8476 setValue(V: &I, NewN: Val);
8477 return;
8478 }
8479 case Intrinsic::amdgcn_cs_chain: {
8480 // At this point we don't care if it's amdgpu_cs_chain or
8481 // amdgpu_cs_chain_preserve.
8482 CallingConv::ID CC = CallingConv::AMDGPU_CS_Chain;
8483
8484 Type *RetTy = I.getType();
8485 assert(RetTy->isVoidTy() && "Should not return");
8486
8487 SDValue Callee = getValue(V: I.getOperand(i_nocapture: 0));
8488
8489 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
8490 // We'll also tack the value of the EXEC mask at the end.
8491 TargetLowering::ArgListTy Args;
8492 Args.reserve(n: 3);
8493
8494 for (unsigned Idx : {2, 3, 1}) {
8495 TargetLowering::ArgListEntry Arg(getValue(V: I.getOperand(i_nocapture: Idx)),
8496 I.getOperand(i_nocapture: Idx)->getType());
8497 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8498 Args.push_back(x: Arg);
8499 }
8500
8501 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
8502 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
8503 Args[2].IsInReg = true; // EXEC should be inreg
8504
8505 // Forward the flags and any additional arguments.
8506 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
8507 TargetLowering::ArgListEntry Arg(getValue(V: I.getOperand(i_nocapture: Idx)),
8508 I.getOperand(i_nocapture: Idx)->getType());
8509 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8510 Args.push_back(x: Arg);
8511 }
8512
8513 TargetLowering::CallLoweringInfo CLI(DAG);
8514 CLI.setDebugLoc(getCurSDLoc())
8515 .setChain(getRoot())
8516 .setCallee(CC, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
8517 .setNoReturn(true)
8518 .setTailCall(true)
8519 .setConvergent(I.isConvergent());
8520 CLI.CB = &I;
8521 std::pair<SDValue, SDValue> Result =
8522 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
8523 (void)Result;
8524 assert(!Result.first.getNode() && !Result.second.getNode() &&
8525 "Should've lowered as tail call");
8526
8527 HasTailCall = true;
8528 return;
8529 }
8530 case Intrinsic::amdgcn_call_whole_wave: {
8531 TargetLowering::ArgListTy Args;
8532 bool isTailCall = I.isTailCall();
8533
8534 // The first argument is the callee. Skip it when assembling the call args.
8535 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
8536 TargetLowering::ArgListEntry Arg(getValue(V: I.getArgOperand(i: Idx)),
8537 I.getArgOperand(i: Idx)->getType());
8538 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8539
8540 // If we have an explicit sret argument that is an Instruction, (i.e., it
8541 // might point to function-local memory), we can't meaningfully tail-call.
8542 if (Arg.IsSRet && isa<Instruction>(Val: I.getArgOperand(i: Idx)))
8543 isTailCall = false;
8544
8545 Args.push_back(x: Arg);
8546 }
8547
8548 SDValue ConvControlToken;
8549 if (auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
8550 auto *Token = Bundle->Inputs[0].get();
8551 ConvControlToken = getValue(V: Token);
8552 }
8553
8554 TargetLowering::CallLoweringInfo CLI(DAG);
8555 CLI.setDebugLoc(getCurSDLoc())
8556 .setChain(getRoot())
8557 .setCallee(CC: CallingConv::AMDGPU_Gfx_WholeWave, ResultType: I.getType(),
8558 Target: getValue(V: I.getArgOperand(i: 0)), ArgsList: std::move(Args))
8559 .setTailCall(isTailCall && canTailCall(CB: I))
8560 .setIsPreallocated(
8561 I.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
8562 .setConvergent(I.isConvergent())
8563 .setConvergenceControlToken(ConvControlToken);
8564 CLI.CB = &I;
8565
8566 std::pair<SDValue, SDValue> Result =
8567 lowerInvokable(CLI, /*EHPadBB=*/nullptr);
8568
8569 if (Result.first.getNode())
8570 setValue(V: &I, NewN: Result.first);
8571 return;
8572 }
8573 case Intrinsic::ptrmask: {
8574 SDValue Ptr = getValue(V: I.getOperand(i_nocapture: 0));
8575 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
8576
8577 // On arm64_32, pointers are 32 bits when stored in memory, but
8578 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
8579 // match the index type, but the pointer is 64 bits, so the mask must be
8580 // zero-extended up to 64 bits to match the pointer.
8581 EVT PtrVT =
8582 TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
8583 EVT MemVT =
8584 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
8585 assert(PtrVT == Ptr.getValueType());
8586 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
8587 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
8588 // 128-bit, so we have to pad the mask with ones for unused bits.
8589 auto HighOnes = DAG.getNode(
8590 Opcode: ISD::SHL, DL: sdl, VT: PtrVT, N1: DAG.getAllOnesConstant(DL: sdl, VT: PtrVT),
8591 N2: DAG.getShiftAmountConstant(Val: Mask.getValueType().getFixedSizeInBits(),
8592 VT: PtrVT, DL: sdl));
8593 Mask = DAG.getNode(Opcode: ISD::OR, DL: sdl, VT: PtrVT,
8594 N1: DAG.getZExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT), N2: HighOnes);
8595 } else if (Mask.getValueType() != PtrVT)
8596 Mask = DAG.getPtrExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT);
8597
8598 assert(Mask.getValueType() == PtrVT);
8599 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::AND, DL: sdl, VT: PtrVT, N1: Ptr, N2: Mask));
8600 return;
8601 }
8602 case Intrinsic::threadlocal_address: {
8603 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
8604 return;
8605 }
8606 case Intrinsic::get_active_lane_mask: {
8607 EVT CCVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8608 SDValue Index = getValue(V: I.getOperand(i_nocapture: 0));
8609 SDValue TripCount = getValue(V: I.getOperand(i_nocapture: 1));
8610 EVT ElementVT = Index.getValueType();
8611
8612 if (!TLI.shouldExpandGetActiveLaneMask(VT: CCVT, OpVT: ElementVT)) {
8613 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: sdl, VT: CCVT, N1: Index,
8614 N2: TripCount));
8615 return;
8616 }
8617
8618 EVT VecTy = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElementVT,
8619 EC: CCVT.getVectorElementCount());
8620
8621 SDValue VectorIndex = DAG.getSplat(VT: VecTy, DL: sdl, Op: Index);
8622 SDValue VectorTripCount = DAG.getSplat(VT: VecTy, DL: sdl, Op: TripCount);
8623 SDValue VectorStep = DAG.getStepVector(DL: sdl, ResVT: VecTy);
8624 SDValue VectorInduction = DAG.getNode(
8625 Opcode: ISD::UADDSAT, DL: sdl, VT: VecTy, N1: VectorIndex, N2: VectorStep);
8626 SDValue SetCC = DAG.getSetCC(DL: sdl, VT: CCVT, LHS: VectorInduction,
8627 RHS: VectorTripCount, Cond: ISD::CondCode::SETULT);
8628 setValue(V: &I, NewN: SetCC);
8629 return;
8630 }
8631 case Intrinsic::mask_beforefirst: {
8632 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
8633 setValue(V: &I,
8634 NewN: DAG.getNode(Opcode: ISD::MASK_BEFOREFIRST, DL: sdl, VT: Op.getValueType(), Operand: Op));
8635 return;
8636 }
8637 case Intrinsic::experimental_get_vector_length: {
8638 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8639 "Expected positive VF");
8640 unsigned VF = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 1))->getZExtValue();
8641 bool IsScalable = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->isOne();
8642
8643 SDValue Count = getValue(V: I.getOperand(i_nocapture: 0));
8644 EVT CountVT = Count.getValueType();
8645
8646 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8647 visitTargetIntrinsic(I, Intrinsic);
8648 return;
8649 }
8650
8651 // Expand to a umin between the trip count and the maximum elements the type
8652 // can hold.
8653 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8654
8655 // Extend the trip count to at least the result VT.
8656 if (CountVT.bitsLT(VT)) {
8657 Count = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: sdl, VT, Operand: Count);
8658 CountVT = VT;
8659 }
8660
8661 SDValue MaxEVL = DAG.getElementCount(DL: sdl, VT: CountVT,
8662 EC: ElementCount::get(MinVal: VF, Scalable: IsScalable));
8663
8664 SDValue UMin = DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: CountVT, N1: Count, N2: MaxEVL);
8665 // Clip to the result type if needed.
8666 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: sdl, VT, Operand: UMin);
8667
8668 setValue(V: &I, NewN: Trunc);
8669 return;
8670 }
8671 case Intrinsic::vector_partial_reduce_add: {
8672 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8673 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8674 setValue(V: &I,
8675 NewN: DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_UMLA, DL: sdl, VT: Acc.getValueType(), N1: Acc,
8676 N2: Input, N3: DAG.getConstant(Val: 1, DL: sdl, VT: Input.getValueType())));
8677 return;
8678 }
8679 case Intrinsic::vector_partial_reduce_fadd: {
8680 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8681 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8682 setValue(V: &I, NewN: DAG.getNode(
8683 Opcode: ISD::PARTIAL_REDUCE_FMLA, DL: sdl, VT: Acc.getValueType(), N1: Acc,
8684 N2: Input, N3: DAG.getConstantFP(Val: 1.0, DL: sdl, VT: Input.getValueType())));
8685 return;
8686 }
8687 case Intrinsic::experimental_cttz_elts: {
8688 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
8689 EVT OpVT = Op.getValueType();
8690 EVT RetTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8691 bool ZeroIsPoison =
8692 !cast<ConstantSDNode>(Val: getValue(V: I.getOperand(i_nocapture: 1)))->isZero();
8693 if (OpVT.getVectorElementType() != MVT::i1) {
8694 // Compare the input vector elements to zero & use to count trailing
8695 // zeros.
8696 SDValue AllZero = DAG.getConstant(Val: 0, DL: sdl, VT: OpVT);
8697 EVT I1OpVT = OpVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1);
8698 Op = DAG.getSetCC(DL: sdl, VT: I1OpVT, LHS: Op, RHS: AllZero, Cond: ISD::SETNE);
8699 }
8700 setValue(V: &I, NewN: DAG.getNode(Opcode: ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
8701 : ISD::CTTZ_ELTS,
8702 DL: sdl, VT: RetTy, Operand: Op));
8703 return;
8704 }
8705 case Intrinsic::vector_insert: {
8706 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8707 SDValue SubVec = getValue(V: I.getOperand(i_nocapture: 1));
8708 SDValue Index = getValue(V: I.getOperand(i_nocapture: 2));
8709
8710 // The intrinsic's index type is i64, but the SDNode requires an index type
8711 // suitable for the target. Convert the index as required.
8712 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8713 if (Index.getValueType() != VectorIdxTy)
8714 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8715
8716 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8717 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: SubVec,
8718 N3: Index));
8719 return;
8720 }
8721 case Intrinsic::vector_extract: {
8722 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8723 SDValue Index = getValue(V: I.getOperand(i_nocapture: 1));
8724 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8725
8726 // The intrinsic's index type is i64, but the SDNode requires an index type
8727 // suitable for the target. Convert the index as required.
8728 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8729 if (Index.getValueType() != VectorIdxTy)
8730 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8731
8732 setValue(V: &I,
8733 NewN: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: Index));
8734 return;
8735 }
8736 case Intrinsic::experimental_vector_match: {
8737 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
8738 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
8739 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
8740 EVT ResVT = Mask.getValueType();
8741 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL: sdl, VT: ResVT, N1: Op1, N2: Op2, N3: Mask));
8742 return;
8743 }
8744 case Intrinsic::vector_reverse:
8745 visitVectorReverse(I);
8746 return;
8747 case Intrinsic::vector_splice_left:
8748 case Intrinsic::vector_splice_right:
8749 visitVectorSplice(I);
8750 return;
8751 case Intrinsic::callbr_landingpad:
8752 visitCallBrLandingPad(I);
8753 return;
8754 case Intrinsic::vector_interleave2:
8755 visitVectorInterleave(I, Factor: 2);
8756 return;
8757 case Intrinsic::vector_interleave3:
8758 visitVectorInterleave(I, Factor: 3);
8759 return;
8760 case Intrinsic::vector_interleave4:
8761 visitVectorInterleave(I, Factor: 4);
8762 return;
8763 case Intrinsic::vector_interleave5:
8764 visitVectorInterleave(I, Factor: 5);
8765 return;
8766 case Intrinsic::vector_interleave6:
8767 visitVectorInterleave(I, Factor: 6);
8768 return;
8769 case Intrinsic::vector_interleave7:
8770 visitVectorInterleave(I, Factor: 7);
8771 return;
8772 case Intrinsic::vector_interleave8:
8773 visitVectorInterleave(I, Factor: 8);
8774 return;
8775 case Intrinsic::vector_deinterleave2:
8776 visitVectorDeinterleave(I, Factor: 2);
8777 return;
8778 case Intrinsic::vector_deinterleave3:
8779 visitVectorDeinterleave(I, Factor: 3);
8780 return;
8781 case Intrinsic::vector_deinterleave4:
8782 visitVectorDeinterleave(I, Factor: 4);
8783 return;
8784 case Intrinsic::vector_deinterleave5:
8785 visitVectorDeinterleave(I, Factor: 5);
8786 return;
8787 case Intrinsic::vector_deinterleave6:
8788 visitVectorDeinterleave(I, Factor: 6);
8789 return;
8790 case Intrinsic::vector_deinterleave7:
8791 visitVectorDeinterleave(I, Factor: 7);
8792 return;
8793 case Intrinsic::vector_deinterleave8:
8794 visitVectorDeinterleave(I, Factor: 8);
8795 return;
8796 case Intrinsic::vector_repeat: {
8797 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8798 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8799 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL: sdl, VT: ResultVT, Operand: Vec));
8800 return;
8801 }
8802 case Intrinsic::experimental_vector_compress:
8803 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: sdl,
8804 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
8805 N1: getValue(V: I.getArgOperand(i: 0)),
8806 N2: getValue(V: I.getArgOperand(i: 1)),
8807 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
8808 return;
8809 case Intrinsic::experimental_convergence_anchor:
8810 case Intrinsic::experimental_convergence_entry:
8811 case Intrinsic::experimental_convergence_loop:
8812 visitConvergenceControl(I, Intrinsic);
8813 return;
8814 case Intrinsic::experimental_vector_histogram_add: {
8815 visitVectorHistogram(I, IntrinsicID: Intrinsic);
8816 return;
8817 }
8818 case Intrinsic::experimental_vector_extract_last_active: {
8819 visitVectorExtractLastActive(I, Intrinsic);
8820 return;
8821 }
8822 case Intrinsic::loop_dependence_war_mask:
8823 setValue(V: &I,
8824 NewN: DAG.getNode(Opcode: ISD::LOOP_DEPENDENCE_WAR_MASK, DL: sdl,
8825 VT: EVT::getEVT(Ty: I.getType()), N1: getValue(V: I.getOperand(i_nocapture: 0)),
8826 N2: getValue(V: I.getOperand(i_nocapture: 1)), N3: getValue(V: I.getOperand(i_nocapture: 2)),
8827 N4: DAG.getConstant(Val: 0, DL: sdl, VT: MVT::i64)));
8828 return;
8829 case Intrinsic::loop_dependence_raw_mask:
8830 setValue(V: &I,
8831 NewN: DAG.getNode(Opcode: ISD::LOOP_DEPENDENCE_RAW_MASK, DL: sdl,
8832 VT: EVT::getEVT(Ty: I.getType()), N1: getValue(V: I.getOperand(i_nocapture: 0)),
8833 N2: getValue(V: I.getOperand(i_nocapture: 1)), N3: getValue(V: I.getOperand(i_nocapture: 2)),
8834 N4: DAG.getConstant(Val: 0, DL: sdl, VT: MVT::i64)));
8835 return;
8836 case Intrinsic::masked_udiv:
8837 setValue(V: &I,
8838 NewN: DAG.getNode(Opcode: ISD::MASKED_UDIV, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8839 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8840 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8841 return;
8842 case Intrinsic::masked_sdiv:
8843 setValue(V: &I,
8844 NewN: DAG.getNode(Opcode: ISD::MASKED_SDIV, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8845 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8846 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8847 return;
8848 case Intrinsic::masked_urem:
8849 setValue(V: &I,
8850 NewN: DAG.getNode(Opcode: ISD::MASKED_UREM, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8851 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8852 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8853 return;
8854 case Intrinsic::masked_srem:
8855 setValue(V: &I,
8856 NewN: DAG.getNode(Opcode: ISD::MASKED_SREM, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8857 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8858 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8859 return;
8860 }
8861}
8862
8863void SelectionDAGBuilder::pushFPOpOutChain(SDValue Result,
8864 fp::ExceptionBehavior EB) {
8865 assert(Result.getNode()->getNumValues() == 2);
8866 SDValue OutChain = Result.getValue(R: 1);
8867 assert(OutChain.getValueType() == MVT::Other);
8868
8869 // Instead of updating the root immediately, push the produced chain to the
8870 // appropriate list, deferring the update until the root is requested. In this
8871 // case, the nodes from the lists are chained using TokenFactor, indicating
8872 // that the operations are independent.
8873 //
8874 // In particular, the root is updated before any call that might access the
8875 // floating-point environment, except for constrained intrinsics.
8876 switch (EB) {
8877 case fp::ExceptionBehavior::ebMayTrap:
8878 case fp::ExceptionBehavior::ebIgnore:
8879 PendingConstrainedFP.push_back(Elt: OutChain);
8880 break;
8881 case fp::ExceptionBehavior::ebStrict:
8882 PendingConstrainedFPStrict.push_back(Elt: OutChain);
8883 break;
8884 }
8885}
8886
8887void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8888 const ConstrainedFPIntrinsic &FPI) {
8889 SDLoc sdl = getCurSDLoc();
8890
8891 // We do not need to serialize constrained FP intrinsics against
8892 // each other or against (nonvolatile) loads, so they can be
8893 // chained like loads.
8894 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
8895 SDValue Chain = getFPOperationRoot(EB);
8896 SmallVector<SDValue, 4> Opers;
8897 Opers.push_back(Elt: Chain);
8898 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8899 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: I)));
8900
8901 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8902 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: FPI.getType());
8903 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
8904
8905 SDNodeFlags Flags;
8906 if (EB == fp::ExceptionBehavior::ebIgnore)
8907 Flags.setNoFPExcept(true);
8908
8909 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &FPI))
8910 Flags.copyFMF(FPMO: *FPOp);
8911
8912 unsigned Opcode;
8913 switch (FPI.getIntrinsicID()) {
8914 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8915#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8916 case Intrinsic::INTRINSIC: \
8917 Opcode = ISD::STRICT_##DAGN; \
8918 break;
8919#include "llvm/IR/ConstrainedOps.def"
8920 case Intrinsic::experimental_constrained_fmuladd: {
8921 Opcode = ISD::STRICT_FMA;
8922 // Break fmuladd into fmul and fadd.
8923 if (!TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
8924 Opers.pop_back();
8925 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8926 pushFPOpOutChain(Result: Mul, EB);
8927 Opcode = ISD::STRICT_FADD;
8928 Opers.clear();
8929 Opers.push_back(Elt: Mul.getValue(R: 1));
8930 Opers.push_back(Elt: Mul.getValue(R: 0));
8931 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: 2)));
8932 }
8933 break;
8934 }
8935 }
8936
8937 // A few strict DAG nodes carry additional operands that are not
8938 // set up by the default code above.
8939 switch (Opcode) {
8940 default: break;
8941 case ISD::STRICT_FP_ROUND:
8942 Opers.push_back(
8943 Elt: DAG.getTargetConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
8944 break;
8945 case ISD::STRICT_FSETCC:
8946 case ISD::STRICT_FSETCCS: {
8947 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
8948 ISD::CondCode Condition = getFCmpCondCode(Pred: FPCmp->getPredicate());
8949 if (DAG.isKnownNeverNaN(Op: Opers[1]) && DAG.isKnownNeverNaN(Op: Opers[2]))
8950 Condition = getFCmpCodeWithoutNaN(CC: Condition);
8951 Opers.push_back(Elt: DAG.getCondCode(Cond: Condition));
8952 break;
8953 }
8954 }
8955
8956 SDValue Result = DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8957 pushFPOpOutChain(Result, EB);
8958
8959 SDValue FPResult = Result.getValue(R: 0);
8960 setValue(V: &FPI, NewN: FPResult);
8961}
8962
8963static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8964 std::optional<unsigned> ResOPC;
8965 switch (VPIntrin.getIntrinsicID()) {
8966 case Intrinsic::vp_cttz_elts: {
8967 bool IsZeroPoison = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8968 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_POISON : ISD::VP_CTTZ_ELTS;
8969 break;
8970 }
8971#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8972 case Intrinsic::VPID: \
8973 ResOPC = ISD::VPSD; \
8974 break;
8975#include "llvm/IR/VPIntrinsics.def"
8976 }
8977
8978 if (!ResOPC)
8979 llvm_unreachable(
8980 "Inconsistency: no SDNode available for this VPIntrinsic!");
8981
8982 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8983 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8984 if (VPIntrin.getFastMathFlags().allowReassoc())
8985 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8986 : ISD::VP_REDUCE_FMUL;
8987 }
8988
8989 return *ResOPC;
8990}
8991
8992void SelectionDAGBuilder::visitVPLoad(
8993 const VPIntrinsic &VPIntrin, EVT VT,
8994 const SmallVectorImpl<SDValue> &OpValues) {
8995 SDLoc DL = getCurSDLoc();
8996 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8997 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8998 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8999 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
9000 SDValue LD;
9001 // Do not serialize variable-length loads of constant memory with
9002 // anything.
9003 if (!Alignment)
9004 Alignment = DAG.getEVTAlign(MemoryVT: VT);
9005 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
9006 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
9007 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9008 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9009 MachineMemOperand::Flags MMOFlags =
9010 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9011 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9012 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
9013 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment,
9014 Metadata: MMOMetadata(AAInfo, Ranges));
9015 LD = DAG.getLoadVP(VT, dl: DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
9016 MMO, IsExpanding: false /*IsExpanding */);
9017 if (AddToChain)
9018 PendingLoads.push_back(Elt: LD.getValue(R: 1));
9019 setValue(V: &VPIntrin, NewN: LD);
9020}
9021
9022void SelectionDAGBuilder::visitVPLoadFF(
9023 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,
9024 const SmallVectorImpl<SDValue> &OpValues) {
9025 assert(OpValues.size() == 3 && "Unexpected number of operands");
9026 SDLoc DL = getCurSDLoc();
9027 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
9028 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9029 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9030 const MDNode *Ranges = VPIntrin.getMetadata(KindID: LLVMContext::MD_range);
9031 SDValue LD;
9032 // Do not serialize variable-length loads of constant memory with
9033 // anything.
9034 if (!Alignment)
9035 Alignment = DAG.getEVTAlign(MemoryVT: VT);
9036 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
9037 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
9038 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9039 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9040 PtrInfo: MachinePointerInfo(PtrOperand), F: MachineMemOperand::MOLoad,
9041 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment,
9042 Metadata: MMOMetadata(AAInfo, Ranges));
9043 LD = DAG.getLoadFFVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
9044 MMO);
9045 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EVLVT, Operand: LD.getValue(R: 1));
9046 if (AddToChain)
9047 PendingLoads.push_back(Elt: LD.getValue(R: 2));
9048 setValue(V: &VPIntrin, NewN: DAG.getMergeValues(Ops: {LD.getValue(R: 0), Trunc}, dl: DL));
9049}
9050
9051void SelectionDAGBuilder::visitVPGather(
9052 const VPIntrinsic &VPIntrin, EVT VT,
9053 const SmallVectorImpl<SDValue> &OpValues) {
9054 SDLoc DL = getCurSDLoc();
9055 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9056 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
9057 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9058 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9059 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
9060 SDValue LD;
9061 if (!Alignment)
9062 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9063 unsigned AS =
9064 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9065 MachineMemOperand::Flags MMOFlags =
9066 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9067 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9068 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9069 BaseAlignment: *Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
9070 SDValue Base, Index, Scale;
9071 bool UniformBase =
9072 getUniformBase(Ptr: PtrOperand, Base, Index, Scale, SDB: this, CurBB: VPIntrin.getParent(),
9073 ElemSize: VT.getScalarStoreSize());
9074 if (!UniformBase) {
9075 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9076 Index = getValue(V: PtrOperand);
9077 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9078 }
9079 EVT IdxVT = Index.getValueType();
9080 EVT EltTy = IdxVT.getVectorElementType();
9081 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
9082 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
9083 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
9084 }
9085 LD = DAG.getGatherVP(
9086 VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), VT, dl: DL,
9087 Ops: {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
9088 IndexType: ISD::SIGNED_SCALED);
9089 PendingLoads.push_back(Elt: LD.getValue(R: 1));
9090 setValue(V: &VPIntrin, NewN: LD);
9091}
9092
9093void SelectionDAGBuilder::visitVPStore(
9094 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9095 SDLoc DL = getCurSDLoc();
9096 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
9097 EVT VT = OpValues[0].getValueType();
9098 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9099 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9100 SDValue ST;
9101 if (!Alignment)
9102 Alignment = DAG.getEVTAlign(MemoryVT: VT);
9103 SDValue Ptr = OpValues[1];
9104 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
9105 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9106 MachineMemOperand::Flags MMOFlags =
9107 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9108 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9109 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
9110 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, Metadata: AAInfo);
9111 ST = DAG.getStoreVP(Chain: getMemoryRoot(), dl: DL, Val: OpValues[0], Ptr, Offset,
9112 Mask: OpValues[2], EVL: OpValues[3], MemVT: VT, MMO, AM: ISD::UNINDEXED,
9113 /* IsTruncating */ false, /*IsCompressing*/ false);
9114 DAG.setRoot(ST);
9115 setValue(V: &VPIntrin, NewN: ST);
9116}
9117
9118void SelectionDAGBuilder::visitVPScatter(
9119 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9120 SDLoc DL = getCurSDLoc();
9121 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9122 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
9123 EVT VT = OpValues[0].getValueType();
9124 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9125 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9126 SDValue ST;
9127 if (!Alignment)
9128 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9129 unsigned AS =
9130 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9131 MachineMemOperand::Flags MMOFlags =
9132 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9133 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9134 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9135 BaseAlignment: *Alignment, Metadata: AAInfo);
9136 SDValue Base, Index, Scale;
9137 bool UniformBase =
9138 getUniformBase(Ptr: PtrOperand, Base, Index, Scale, SDB: this, CurBB: VPIntrin.getParent(),
9139 ElemSize: VT.getScalarStoreSize());
9140 if (!UniformBase) {
9141 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9142 Index = getValue(V: PtrOperand);
9143 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9144 }
9145 EVT IdxVT = Index.getValueType();
9146 EVT EltTy = IdxVT.getVectorElementType();
9147 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
9148 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
9149 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
9150 }
9151 ST = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT, dl: DL,
9152 Ops: {getMemoryRoot(), OpValues[0], Base, Index, Scale,
9153 OpValues[2], OpValues[3]},
9154 MMO, IndexType: ISD::SIGNED_SCALED);
9155 DAG.setRoot(ST);
9156 setValue(V: &VPIntrin, NewN: ST);
9157}
9158
9159void SelectionDAGBuilder::visitVPStridedLoad(
9160 const VPIntrinsic &VPIntrin, EVT VT,
9161 const SmallVectorImpl<SDValue> &OpValues) {
9162 SDLoc DL = getCurSDLoc();
9163 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
9164 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9165 if (!Alignment)
9166 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9167 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9168 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
9169 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
9170 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
9171 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9172 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9173 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9174 MachineMemOperand::Flags MMOFlags =
9175 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9176 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9177 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9178 BaseAlignment: *Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
9179
9180 SDValue LD = DAG.getStridedLoadVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Stride: OpValues[1],
9181 Mask: OpValues[2], EVL: OpValues[3], MMO,
9182 IsExpanding: false /*IsExpanding*/);
9183
9184 if (AddToChain)
9185 PendingLoads.push_back(Elt: LD.getValue(R: 1));
9186 setValue(V: &VPIntrin, NewN: LD);
9187}
9188
9189void SelectionDAGBuilder::visitVPStridedStore(
9190 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9191 SDLoc DL = getCurSDLoc();
9192 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
9193 EVT VT = OpValues[0].getValueType();
9194 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9195 if (!Alignment)
9196 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9197 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9198 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9199 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9200 MachineMemOperand::Flags MMOFlags =
9201 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9202 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9203 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9204 BaseAlignment: *Alignment, Metadata: AAInfo);
9205
9206 SDValue ST = DAG.getStridedStoreVP(
9207 Chain: getMemoryRoot(), DL, Val: OpValues[0], Ptr: OpValues[1],
9208 Offset: DAG.getPOISON(VT: OpValues[1].getValueType()), Stride: OpValues[2], Mask: OpValues[3],
9209 EVL: OpValues[4], MemVT: VT, MMO, AM: ISD::UNINDEXED, /*IsTruncating*/ false,
9210 /*IsCompressing*/ false);
9211
9212 DAG.setRoot(ST);
9213 setValue(V: &VPIntrin, NewN: ST);
9214}
9215
9216void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
9217 const VPIntrinsic &VPIntrin) {
9218 SDLoc DL = getCurSDLoc();
9219 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
9220
9221 auto IID = VPIntrin.getIntrinsicID();
9222
9223 SmallVector<EVT, 4> ValueVTs;
9224 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9225 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: VPIntrin.getType(), ValueVTs);
9226 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
9227
9228 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IntrinsicID: IID);
9229
9230 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
9231 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
9232 "Unexpected target EVL type");
9233
9234 // Request operands.
9235 SmallVector<SDValue, 7> OpValues;
9236 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
9237 auto Op = getValue(V: VPIntrin.getArgOperand(i: I));
9238 if (I == EVLParamPos)
9239 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EVLParamVT, Operand: Op);
9240 OpValues.push_back(Elt: Op);
9241 }
9242
9243 switch (Opcode) {
9244 default: {
9245 SDNodeFlags SDFlags;
9246 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &VPIntrin))
9247 SDFlags.copyFMF(FPMO: *FPMO);
9248 SDValue Result = DAG.getNode(Opcode, DL, VTList: VTs, Ops: OpValues, Flags: SDFlags);
9249 setValue(V: &VPIntrin, NewN: Result);
9250 break;
9251 }
9252 case ISD::VP_LOAD:
9253 visitVPLoad(VPIntrin, VT: ValueVTs[0], OpValues);
9254 break;
9255 case ISD::VP_LOAD_FF:
9256 visitVPLoadFF(VPIntrin, VT: ValueVTs[0], EVLVT: ValueVTs[1], OpValues);
9257 break;
9258 case ISD::VP_GATHER:
9259 visitVPGather(VPIntrin, VT: ValueVTs[0], OpValues);
9260 break;
9261 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9262 visitVPStridedLoad(VPIntrin, VT: ValueVTs[0], OpValues);
9263 break;
9264 case ISD::VP_STORE:
9265 visitVPStore(VPIntrin, OpValues);
9266 break;
9267 case ISD::VP_SCATTER:
9268 visitVPScatter(VPIntrin, OpValues);
9269 break;
9270 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9271 visitVPStridedStore(VPIntrin, OpValues);
9272 break;
9273 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
9274 case ISD::VP_CTTZ_ELTS: {
9275 SDValue Result =
9276 DAG.getNode(Opcode, DL, VTList: VTs, Ops: {OpValues[0], OpValues[2], OpValues[3]});
9277 setValue(V: &VPIntrin, NewN: Result);
9278 break;
9279 }
9280 }
9281}
9282
9283SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
9284 const BasicBlock *EHPadBB,
9285 MCSymbol *&BeginLabel) {
9286 MachineFunction &MF = DAG.getMachineFunction();
9287
9288 // Skip emitting EH_LABEL on targets whose exception tables don't reference
9289 // them (32-bit x86 SEH, Wasm).
9290 if (!MF.getContext().getAsmInfo().usesPerInvokeEHLabels()) {
9291 BeginLabel = nullptr;
9292 return Chain;
9293 }
9294
9295 // Insert a label before the invoke call to mark the try range. This can be
9296 // used to detect deletion of the invoke via the MachineModuleInfo.
9297 BeginLabel = MF.getContext().createTempSymbol();
9298
9299 // For SjLj, keep track of which landing pads go with which invokes
9300 // so as to maintain the ordering of pads in the LSDA.
9301 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
9302 if (CallSiteIndex) {
9303 MF.setCallSiteBeginLabel(BeginLabel, Site: CallSiteIndex);
9304 LPadToCallSiteMap[FuncInfo.getMBB(BB: EHPadBB)].push_back(Elt: CallSiteIndex);
9305
9306 // Now that the call site is handled, stop tracking it.
9307 FuncInfo.setCurrentCallSite(0);
9308 }
9309
9310 return DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: BeginLabel);
9311}
9312
9313SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
9314 const BasicBlock *EHPadBB,
9315 MCSymbol *BeginLabel) {
9316 // No labels were emitted.
9317 if (!BeginLabel)
9318 return Chain;
9319
9320 MachineFunction &MF = DAG.getMachineFunction();
9321
9322 // Insert a label at the end of the invoke call to mark the try range. This
9323 // can be used to detect deletion of the invoke via the MachineModuleInfo.
9324 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
9325 Chain = DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: EndLabel);
9326
9327 // Inform MachineModuleInfo of range.
9328 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
9329 // There is a platform (e.g. wasm) that uses funclet style IR but does not
9330 // actually use outlined funclets and their LSDA info style.
9331 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
9332 assert(II && "II should've been set");
9333 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
9334 EHInfo->addIPToStateRange(II, InvokeBegin: BeginLabel, InvokeEnd: EndLabel);
9335 } else if (!isScopedEHPersonality(Pers)) {
9336 assert(EHPadBB);
9337 MF.addInvoke(LandingPad: FuncInfo.getMBB(BB: EHPadBB), BeginLabel, EndLabel);
9338 }
9339
9340 return Chain;
9341}
9342
9343std::pair<SDValue, SDValue>
9344SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
9345 const BasicBlock *EHPadBB) {
9346 MCSymbol *BeginLabel = nullptr;
9347
9348 if (EHPadBB) {
9349 // Both PendingLoads and PendingExports must be flushed here;
9350 // this call might not return.
9351 (void)getRoot();
9352 DAG.setRoot(lowerStartEH(Chain: getControlRoot(), EHPadBB, BeginLabel));
9353 CLI.setChain(getRoot());
9354 }
9355
9356 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9357 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
9358
9359 assert((CLI.IsTailCall || Result.second.getNode()) &&
9360 "Non-null chain expected with non-tail call!");
9361 assert((Result.second.getNode() || !Result.first.getNode()) &&
9362 "Null value expected with tail call!");
9363
9364 if (!Result.second.getNode()) {
9365 // As a special case, a null chain means that a tail call has been emitted
9366 // and the DAG root is already updated.
9367 HasTailCall = true;
9368
9369 // Since there's no actual continuation from this block, nothing can be
9370 // relying on us setting vregs for them.
9371 PendingExports.clear();
9372 } else {
9373 DAG.setRoot(Result.second);
9374 }
9375
9376 if (EHPadBB) {
9377 DAG.setRoot(lowerEndEH(Chain: getRoot(), II: cast_or_null<InvokeInst>(Val: CLI.CB), EHPadBB,
9378 BeginLabel));
9379 Result.second = getRoot();
9380 }
9381
9382 return Result;
9383}
9384
9385bool SelectionDAGBuilder::canTailCall(const CallBase &CB) const {
9386 bool isMustTailCall = CB.isMustTailCall();
9387
9388 // Avoid emitting tail calls in functions with the disable-tail-calls
9389 // attribute.
9390 const Function *Caller = CB.getParent()->getParent();
9391 if (!isMustTailCall &&
9392 Caller->getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
9393 return false;
9394
9395 // We can't tail call inside a function with a swifterror argument. Lowering
9396 // does not support this yet. It would have to move into the swifterror
9397 // register before the call.
9398 if (DAG.hasSwiftErrorArg())
9399 return false;
9400
9401 // Check if target-independent constraints permit a tail call here.
9402 // Target-dependent constraints are checked within TLI->LowerCallTo.
9403 return isInTailCallPosition(Call: CB, TM: DAG.getTarget());
9404}
9405
9406void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
9407 bool isTailCall, bool isMustTailCall,
9408 const BasicBlock *EHPadBB,
9409 const TargetLowering::PtrAuthInfo *PAI) {
9410 auto &DL = DAG.getDataLayout();
9411 FunctionType *FTy = CB.getFunctionType();
9412 Type *RetTy = CB.getType();
9413
9414 TargetLowering::ArgListTy Args;
9415 Args.reserve(n: CB.arg_size());
9416
9417 const Value *SwiftErrorVal = nullptr;
9418 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9419
9420 if (isTailCall)
9421 isTailCall = canTailCall(CB);
9422
9423 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
9424 const Value *V = *I;
9425
9426 // Skip empty types
9427 if (V->getType()->isEmptyTy())
9428 continue;
9429
9430 SDValue ArgNode = getValue(V);
9431 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
9432 Entry.setAttributes(Call: &CB, ArgIdx: I - CB.arg_begin());
9433
9434 // Use swifterror virtual register as input to the call.
9435 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
9436 SwiftErrorVal = V;
9437 // We find the virtual register for the actual swifterror argument.
9438 // Instead of using the Value, we use the virtual register instead.
9439 Entry.Node =
9440 DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
9441 VT: EVT(TLI.getPointerTy(DL)));
9442 }
9443
9444 Args.push_back(x: Entry);
9445
9446 // If we have an explicit sret argument that is an Instruction, (i.e., it
9447 // might point to function-local memory), we can't meaningfully tail-call.
9448 if (Entry.IsSRet && isa<Instruction>(Val: V))
9449 isTailCall = false;
9450 }
9451
9452 // If call site has a cfguardtarget operand bundle, create and add an
9453 // additional ArgListEntry.
9454 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_cfguardtarget)) {
9455 Value *V = Bundle->Inputs[0];
9456 TargetLowering::ArgListEntry Entry(V, getValue(V));
9457 Entry.IsCFGuardTarget = true;
9458 Args.push_back(x: Entry);
9459 }
9460
9461 // Disable tail calls if there is an swifterror argument. Targets have not
9462 // been updated to support tail calls.
9463 if (TLI.supportSwiftError() && SwiftErrorVal)
9464 isTailCall = false;
9465
9466 ConstantInt *CFIType = nullptr;
9467 if (CB.isIndirectCall()) {
9468 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi)) {
9469 if (!TLI.supportKCFIBundles())
9470 report_fatal_error(
9471 reason: "Target doesn't support calls with kcfi operand bundles.");
9472 CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
9473 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
9474 }
9475 }
9476
9477 SDValue ConvControlToken;
9478 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
9479 auto *Token = Bundle->Inputs[0].get();
9480 ConvControlToken = getValue(V: Token);
9481 }
9482
9483 GlobalValue *DeactivationSymbol = nullptr;
9484 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
9485 DeactivationSymbol = cast<GlobalValue>(Val: Bundle->Inputs[0].get());
9486 }
9487
9488 TargetLowering::CallLoweringInfo CLI(DAG);
9489 CLI.setDebugLoc(getCurSDLoc())
9490 .setChain(getRoot())
9491 .setCallee(ResultType: RetTy, FTy, Target: Callee, ArgsList: std::move(Args), Call: CB)
9492 .setTailCall(isTailCall)
9493 .setConvergent(CB.isConvergent())
9494 .setIsPreallocated(
9495 CB.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
9496 .setCFIType(CFIType)
9497 .setConvergenceControlToken(ConvControlToken)
9498 .setDeactivationSymbol(DeactivationSymbol);
9499
9500 // Set the pointer authentication info if we have it.
9501 if (PAI) {
9502 if (!TLI.supportPtrAuthBundles())
9503 report_fatal_error(
9504 reason: "This target doesn't support calls with ptrauth operand bundles.");
9505 CLI.setPtrAuth(*PAI);
9506 }
9507
9508 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9509
9510 if (Result.first.getNode()) {
9511 Result.first = lowerRangeToAssertZExt(DAG, I: CB, Op: Result.first);
9512 Result.first = lowerNoFPClassToAssertNoFPClass(DAG, I: CB, Op: Result.first);
9513 setValue(V: &CB, NewN: Result.first);
9514 }
9515
9516 // The last element of CLI.InVals has the SDValue for swifterror return.
9517 // Here we copy it to a virtual register and update SwiftErrorMap for
9518 // book-keeping.
9519 if (SwiftErrorVal && TLI.supportSwiftError()) {
9520 // Get the last element of InVals.
9521 SDValue Src = CLI.InVals.back();
9522 Register VReg =
9523 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
9524 SDValue CopyNode = CLI.DAG.getCopyToReg(Chain: Result.second, dl: CLI.DL, Reg: VReg, N: Src);
9525 DAG.setRoot(CopyNode);
9526 }
9527}
9528
9529static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
9530 SelectionDAGBuilder &Builder) {
9531 // Check to see if this load can be trivially constant folded, e.g. if the
9532 // input is from a string literal.
9533 if (const Constant *LoadInput = dyn_cast<Constant>(Val: PtrVal)) {
9534 // Cast pointer to the type we really want to load.
9535 Type *LoadTy =
9536 Type::getIntNTy(C&: PtrVal->getContext(), N: LoadVT.getScalarSizeInBits());
9537 if (LoadVT.isVector())
9538 LoadTy = FixedVectorType::get(ElementType: LoadTy, NumElts: LoadVT.getVectorNumElements());
9539 if (const Constant *LoadCst =
9540 ConstantFoldLoadFromConstPtr(C: const_cast<Constant *>(LoadInput),
9541 Ty: LoadTy, DL: Builder.DAG.getDataLayout()))
9542 return Builder.getValue(V: LoadCst);
9543 }
9544
9545 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9546 // still constant memory, the input chain can be the entry node.
9547 SDValue Root;
9548 bool ConstantMemory = false;
9549
9550 // Do not serialize (non-volatile) loads of constant memory with anything.
9551 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(P: PtrVal)) {
9552 Root = Builder.DAG.getEntryNode();
9553 ConstantMemory = true;
9554 } else {
9555 // Do not serialize non-volatile loads against each other.
9556 Root = Builder.DAG.getRoot();
9557 }
9558
9559 SDValue Ptr = Builder.getValue(V: PtrVal);
9560 SDValue LoadVal =
9561 Builder.DAG.getLoad(VT: LoadVT, dl: Builder.getCurSDLoc(), Chain: Root, Ptr,
9562 PtrInfo: MachinePointerInfo(PtrVal), Alignment: Align(1));
9563
9564 if (!ConstantMemory)
9565 Builder.PendingLoads.push_back(Elt: LoadVal.getValue(R: 1));
9566 return LoadVal;
9567}
9568
9569/// Record the value for an instruction that produces an integer result,
9570/// converting the type where necessary.
9571void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9572 SDValue Value,
9573 bool IsSigned) {
9574 EVT VT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9575 Ty: I.getType(), AllowUnknown: true);
9576 Value = DAG.getExtOrTrunc(IsSigned, Op: Value, DL: getCurSDLoc(), VT);
9577 setValue(V: &I, NewN: Value);
9578}
9579
9580/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9581/// true and lower it. Otherwise return false, and it will be lowered like a
9582/// normal call.
9583/// The caller already checked that \p I calls the appropriate LibFunc with a
9584/// correct prototype.
9585bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9586 const Value *LHS = I.getArgOperand(i: 0), *RHS = I.getArgOperand(i: 1);
9587 const Value *Size = I.getArgOperand(i: 2);
9588 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(Val: getValue(V: Size));
9589 if (CSize && CSize->getZExtValue() == 0) {
9590 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9591 Ty: I.getType(), AllowUnknown: true);
9592 setValue(V: &I, NewN: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: CallVT));
9593 return true;
9594 }
9595
9596 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9597 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9598 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: LHS), Op2: getValue(V: RHS),
9599 Op3: getValue(V: Size), CI: &I);
9600 if (Res.first.getNode()) {
9601 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9602 PendingLoads.push_back(Elt: Res.second);
9603 return true;
9604 }
9605
9606 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9607 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9608 if (!CSize || !isOnlyUsedInZeroEqualityComparison(CtxI: &I))
9609 return false;
9610
9611 // If the target has a fast compare for the given size, it will return a
9612 // preferred load type for that size. Require that the load VT is legal and
9613 // that the target supports unaligned loads of that type. Otherwise, return
9614 // INVALID.
9615 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9616 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9617 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9618 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9619 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9620 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9621 // TODO: Check alignment of src and dest ptrs.
9622 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9623 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9624 if (!TLI.isTypeLegal(VT: LVT) ||
9625 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: SrcAS) ||
9626 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: DstAS))
9627 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
9628 }
9629
9630 return LVT;
9631 };
9632
9633 // This turns into unaligned loads. We only do this if the target natively
9634 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9635 // we'll only produce a small number of byte loads.
9636 MVT LoadVT;
9637 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9638 switch (NumBitsToCompare) {
9639 default:
9640 return false;
9641 case 16:
9642 LoadVT = MVT::i16;
9643 break;
9644 case 32:
9645 LoadVT = MVT::i32;
9646 break;
9647 case 64:
9648 case 128:
9649 case 256:
9650 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9651 break;
9652 }
9653
9654 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9655 return false;
9656
9657 SDValue LoadL = getMemCmpLoad(PtrVal: LHS, LoadVT, Builder&: *this);
9658 SDValue LoadR = getMemCmpLoad(PtrVal: RHS, LoadVT, Builder&: *this);
9659
9660 // Bitcast to a wide integer type if the loads are vectors.
9661 if (LoadVT.isVector()) {
9662 EVT CmpVT = EVT::getIntegerVT(Context&: LHS->getContext(), BitWidth: LoadVT.getSizeInBits());
9663 LoadL = DAG.getBitcast(VT: CmpVT, V: LoadL);
9664 LoadR = DAG.getBitcast(VT: CmpVT, V: LoadR);
9665 }
9666
9667 SDValue Cmp = DAG.getSetCC(DL: getCurSDLoc(), VT: MVT::i1, LHS: LoadL, RHS: LoadR, Cond: ISD::SETNE);
9668 processIntegerCallValue(I, Value: Cmp, IsSigned: false);
9669 return true;
9670}
9671
9672/// See if we can lower a memchr call into an optimized form. If so, return
9673/// true and lower it. Otherwise return false, and it will be lowered like a
9674/// normal call.
9675/// The caller already checked that \p I calls the appropriate LibFunc with a
9676/// correct prototype.
9677bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9678 const Value *Src = I.getArgOperand(i: 0);
9679 const Value *Char = I.getArgOperand(i: 1);
9680 const Value *Length = I.getArgOperand(i: 2);
9681
9682 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9683 std::pair<SDValue, SDValue> Res =
9684 TSI.EmitTargetCodeForMemchr(DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(),
9685 Src: getValue(V: Src), Char: getValue(V: Char), Length: getValue(V: Length),
9686 SrcPtrInfo: MachinePointerInfo(Src));
9687 if (Res.first.getNode()) {
9688 setValue(V: &I, NewN: Res.first);
9689 PendingLoads.push_back(Elt: Res.second);
9690 return true;
9691 }
9692
9693 return false;
9694}
9695
9696/// See if we can lower a memccpy call into an optimized form. If so, return
9697/// true and lower it, otherwise return false and it will be lowered like a
9698/// normal call.
9699/// The caller already checked that \p I calls the appropriate LibFunc with a
9700/// correct prototype.
9701bool SelectionDAGBuilder::visitMemCCpyCall(const CallInst &I) {
9702 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9703 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemccpy(
9704 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Dst: getValue(V: I.getArgOperand(i: 0)),
9705 Src: getValue(V: I.getArgOperand(i: 1)), C: getValue(V: I.getArgOperand(i: 2)),
9706 Size: getValue(V: I.getArgOperand(i: 3)), CI: &I);
9707
9708 if (Res.first) {
9709 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9710 PendingLoads.push_back(Elt: Res.second);
9711 return true;
9712 }
9713 return false;
9714}
9715
9716/// See if we can lower a mempcpy call into an optimized form. If so, return
9717/// true and lower it. Otherwise return false, and it will be lowered like a
9718/// normal call.
9719/// The caller already checked that \p I calls the appropriate LibFunc with a
9720/// correct prototype.
9721bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9722 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
9723 SDValue Src = getValue(V: I.getArgOperand(i: 1));
9724 SDValue Size = getValue(V: I.getArgOperand(i: 2));
9725
9726 Align DstAlign = DAG.InferPtrAlign(Ptr: Dst).valueOrOne();
9727 Align SrcAlign = DAG.InferPtrAlign(Ptr: Src).valueOrOne();
9728
9729 SDLoc sdl = getCurSDLoc();
9730
9731 // In the mempcpy context we need to pass in a false value for isTailCall
9732 // because the return pointer needs to be adjusted by the size of
9733 // the copied memory.
9734 SDValue Root = getMemoryRoot();
9735 SDValue MC = DAG.getMemcpy(
9736 Chain: Root, dl: sdl, Dst, Src, Size, DstAlign, SrcAlign, isVol: false, AlwaysInline: false,
9737 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
9738 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)), AAInfo: I.getAAMetadata());
9739 assert(MC.getNode() != nullptr &&
9740 "** memcpy should not be lowered as TailCall in mempcpy context **");
9741 DAG.setRoot(MC);
9742
9743 // Check if Size needs to be truncated or extended.
9744 Size = DAG.getSExtOrTrunc(Op: Size, DL: sdl, VT: Dst.getValueType());
9745
9746 // Adjust return pointer to point just past the last dst byte.
9747 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Base: Dst, Offset: Size, DL: sdl);
9748 setValue(V: &I, NewN: DstPlusSize);
9749 return true;
9750}
9751
9752/// See if we can lower a strcpy call into an optimized form. If so, return
9753/// true and lower it, otherwise return false and it will be lowered like a
9754/// normal call.
9755/// The caller already checked that \p I calls the appropriate LibFunc with a
9756/// correct prototype.
9757bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9758 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9759
9760 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9761 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcpy(
9762 DAG, DL: getCurSDLoc(), Chain: getRoot(), Dest: getValue(V: Arg0), Src: getValue(V: Arg1),
9763 DestPtrInfo: MachinePointerInfo(Arg0), SrcPtrInfo: MachinePointerInfo(Arg1), isStpcpy, CI: &I);
9764 if (Res.first.getNode()) {
9765 setValue(V: &I, NewN: Res.first);
9766 DAG.setRoot(Res.second);
9767 return true;
9768 }
9769
9770 return false;
9771}
9772
9773/// See if we can lower a strcmp call into an optimized form. If so, return
9774/// true and lower it, otherwise return false and it will be lowered like a
9775/// normal call.
9776/// The caller already checked that \p I calls the appropriate LibFunc with a
9777/// correct prototype.
9778bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9779 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9780
9781 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9782 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcmp(
9783 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: Arg0), Op2: getValue(V: Arg1),
9784 Op1PtrInfo: MachinePointerInfo(Arg0), Op2PtrInfo: MachinePointerInfo(Arg1), CI: &I);
9785 if (Res.first.getNode()) {
9786 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9787 PendingLoads.push_back(Elt: Res.second);
9788 return true;
9789 }
9790
9791 return false;
9792}
9793
9794/// See if we can lower a strlen call into an optimized form. If so, return
9795/// true and lower it, otherwise return false and it will be lowered like a
9796/// normal call.
9797/// The caller already checked that \p I calls the appropriate LibFunc with a
9798/// correct prototype.
9799bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9800 const Value *Arg0 = I.getArgOperand(i: 0);
9801
9802 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9803 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(
9804 DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(), Src: getValue(V: Arg0), CI: &I);
9805 if (Res.first.getNode()) {
9806 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9807 PendingLoads.push_back(Elt: Res.second);
9808 return true;
9809 }
9810
9811 return false;
9812}
9813
9814/// See if we can lower a strnlen call into an optimized form. If so, return
9815/// true and lower it, otherwise return false and it will be lowered like a
9816/// normal call.
9817/// The caller already checked that \p I calls the appropriate LibFunc with a
9818/// correct prototype.
9819bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9820 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9821
9822 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9823 std::pair<SDValue, SDValue> Res =
9824 TSI.EmitTargetCodeForStrnlen(DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(),
9825 Src: getValue(V: Arg0), MaxLength: getValue(V: Arg1),
9826 SrcPtrInfo: MachinePointerInfo(Arg0));
9827 if (Res.first.getNode()) {
9828 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9829 PendingLoads.push_back(Elt: Res.second);
9830 return true;
9831 }
9832
9833 return false;
9834}
9835
9836/// See if we can lower a Strstr call into an optimized form. If so, return
9837/// true and lower it, otherwise return false and it will be lowered like a
9838/// normal call.
9839/// The caller already checked that \p I calls the appropriate LibFunc with a
9840/// correct prototype.
9841bool SelectionDAGBuilder::visitStrstrCall(const CallInst &I) {
9842 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9843 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9844 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrstr(
9845 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: Arg0), Op2: getValue(V: Arg1), CI: &I);
9846 if (Res.first) {
9847 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9848 PendingLoads.push_back(Elt: Res.second);
9849 return true;
9850 }
9851 return false;
9852}
9853
9854/// See if we can lower a unary floating-point operation into an SDNode with
9855/// the specified Opcode. If so, return true and lower it, otherwise return
9856/// false and it will be lowered like a normal call.
9857/// The caller already checked that \p I calls the appropriate LibFunc with a
9858/// correct prototype.
9859bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9860 unsigned Opcode) {
9861 // We already checked this call's prototype; verify it doesn't modify errno.
9862 // Do not perform optimizations for call sites that require strict
9863 // floating-point semantics.
9864 if (!I.onlyReadsMemory() || I.isStrictFP())
9865 return false;
9866
9867 SDNodeFlags Flags;
9868 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9869
9870 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
9871 setValue(V: &I,
9872 NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Tmp.getValueType(), Operand: Tmp, Flags));
9873 return true;
9874}
9875
9876/// See if we can lower a binary floating-point operation into an SDNode with
9877/// the specified Opcode. If so, return true and lower it. Otherwise return
9878/// false, and it will be lowered like a normal call.
9879/// The caller already checked that \p I calls the appropriate LibFunc with a
9880/// correct prototype.
9881bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9882 unsigned Opcode) {
9883 // We already checked this call's prototype; verify it doesn't modify errno.
9884 // Do not perform optimizations for call sites that require strict
9885 // floating-point semantics.
9886 if (!I.onlyReadsMemory() || I.isStrictFP())
9887 return false;
9888
9889 SDNodeFlags Flags;
9890 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9891
9892 SDValue Tmp0 = getValue(V: I.getArgOperand(i: 0));
9893 SDValue Tmp1 = getValue(V: I.getArgOperand(i: 1));
9894 EVT VT = Tmp0.getValueType();
9895 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT, N1: Tmp0, N2: Tmp1, Flags));
9896 return true;
9897}
9898
9899void SelectionDAGBuilder::visitCall(const CallInst &I) {
9900 // Handle inline assembly differently.
9901 if (I.isInlineAsm()) {
9902 visitInlineAsm(Call: I);
9903 return;
9904 }
9905
9906 diagnoseDontCall(CI: I);
9907
9908 if (Function *F = I.getCalledFunction()) {
9909 if (F->isDeclaration()) {
9910 // Is this an LLVM intrinsic?
9911 if (unsigned IID = F->getIntrinsicID()) {
9912 visitIntrinsicCall(I, Intrinsic: IID);
9913 return;
9914 }
9915 }
9916
9917 // Check for well-known libc/libm calls. If the function is internal, it
9918 // can't be a library call. Don't do the check if marked as nobuiltin for
9919 // some reason.
9920 // This code should not handle libcalls that are already canonicalized to
9921 // intrinsics by the middle-end.
9922 LibFunc Func = !I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName()
9923 ? LibInfo->getLibFunc(FDecl: *F)
9924 : NotLibFunc;
9925 if (LibInfo->hasOptimizedCodeGen(F: Func)) {
9926 switch (Func) {
9927 default: break;
9928 case LibFunc_bcmp:
9929 if (visitMemCmpBCmpCall(I))
9930 return;
9931 break;
9932 case LibFunc_copysign:
9933 case LibFunc_copysignf:
9934 case LibFunc_copysignl:
9935 // We already checked this call's prototype; verify it doesn't modify
9936 // errno.
9937 if (I.onlyReadsMemory()) {
9938 SDValue LHS = getValue(V: I.getArgOperand(i: 0));
9939 SDValue RHS = getValue(V: I.getArgOperand(i: 1));
9940 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: getCurSDLoc(),
9941 VT: LHS.getValueType(), N1: LHS, N2: RHS));
9942 return;
9943 }
9944 break;
9945 case LibFunc_sin:
9946 case LibFunc_sinf:
9947 case LibFunc_sinl:
9948 if (visitUnaryFloatCall(I, Opcode: ISD::FSIN))
9949 return;
9950 break;
9951 case LibFunc_cos:
9952 case LibFunc_cosf:
9953 case LibFunc_cosl:
9954 if (visitUnaryFloatCall(I, Opcode: ISD::FCOS))
9955 return;
9956 break;
9957 case LibFunc_tan:
9958 case LibFunc_tanf:
9959 case LibFunc_tanl:
9960 if (visitUnaryFloatCall(I, Opcode: ISD::FTAN))
9961 return;
9962 break;
9963 case LibFunc_asin:
9964 case LibFunc_asinf:
9965 case LibFunc_asinl:
9966 if (visitUnaryFloatCall(I, Opcode: ISD::FASIN))
9967 return;
9968 break;
9969 case LibFunc_acos:
9970 case LibFunc_acosf:
9971 case LibFunc_acosl:
9972 if (visitUnaryFloatCall(I, Opcode: ISD::FACOS))
9973 return;
9974 break;
9975 case LibFunc_atan:
9976 case LibFunc_atanf:
9977 case LibFunc_atanl:
9978 if (visitUnaryFloatCall(I, Opcode: ISD::FATAN))
9979 return;
9980 break;
9981 case LibFunc_atan2:
9982 case LibFunc_atan2f:
9983 case LibFunc_atan2l:
9984 if (visitBinaryFloatCall(I, Opcode: ISD::FATAN2))
9985 return;
9986 break;
9987 case LibFunc_sinh:
9988 case LibFunc_sinhf:
9989 case LibFunc_sinhl:
9990 if (visitUnaryFloatCall(I, Opcode: ISD::FSINH))
9991 return;
9992 break;
9993 case LibFunc_cosh:
9994 case LibFunc_coshf:
9995 case LibFunc_coshl:
9996 if (visitUnaryFloatCall(I, Opcode: ISD::FCOSH))
9997 return;
9998 break;
9999 case LibFunc_tanh:
10000 case LibFunc_tanhf:
10001 case LibFunc_tanhl:
10002 if (visitUnaryFloatCall(I, Opcode: ISD::FTANH))
10003 return;
10004 break;
10005 case LibFunc_sqrt:
10006 case LibFunc_sqrtf:
10007 case LibFunc_sqrtl:
10008 case LibFunc_sqrt_finite:
10009 case LibFunc_sqrtf_finite:
10010 case LibFunc_sqrtl_finite:
10011 if (visitUnaryFloatCall(I, Opcode: ISD::FSQRT))
10012 return;
10013 break;
10014 case LibFunc_log2:
10015 case LibFunc_log2f:
10016 case LibFunc_log2l:
10017 if (visitUnaryFloatCall(I, Opcode: ISD::FLOG2))
10018 return;
10019 break;
10020 case LibFunc_exp2:
10021 case LibFunc_exp2f:
10022 case LibFunc_exp2l:
10023 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP2))
10024 return;
10025 break;
10026 case LibFunc_exp10:
10027 case LibFunc_exp10f:
10028 case LibFunc_exp10l:
10029 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP10))
10030 return;
10031 break;
10032 case LibFunc_ldexp:
10033 case LibFunc_ldexpf:
10034 case LibFunc_ldexpl:
10035 if (visitBinaryFloatCall(I, Opcode: ISD::FLDEXP))
10036 return;
10037 break;
10038 case LibFunc_strstr:
10039 if (visitStrstrCall(I))
10040 return;
10041 break;
10042 case LibFunc_memcmp:
10043 if (visitMemCmpBCmpCall(I))
10044 return;
10045 break;
10046 case LibFunc_memccpy:
10047 if (visitMemCCpyCall(I))
10048 return;
10049 break;
10050 case LibFunc_mempcpy:
10051 if (visitMemPCpyCall(I))
10052 return;
10053 break;
10054 case LibFunc_memchr:
10055 if (visitMemChrCall(I))
10056 return;
10057 break;
10058 case LibFunc_strcpy:
10059 if (visitStrCpyCall(I, isStpcpy: false))
10060 return;
10061 break;
10062 case LibFunc_stpcpy:
10063 if (visitStrCpyCall(I, isStpcpy: true))
10064 return;
10065 break;
10066 case LibFunc_strcmp:
10067 if (visitStrCmpCall(I))
10068 return;
10069 break;
10070 case LibFunc_strlen:
10071 if (visitStrLenCall(I))
10072 return;
10073 break;
10074 case LibFunc_strnlen:
10075 if (visitStrNLenCall(I))
10076 return;
10077 break;
10078 }
10079 }
10080 }
10081
10082 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
10083 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), /*EHPadBB=*/nullptr);
10084 return;
10085 }
10086
10087 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
10088 // have to do anything here to lower funclet bundles.
10089 // CFGuardTarget bundles are lowered in LowerCallTo.
10090 failForInvalidBundles(
10091 I, Name: "calls",
10092 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
10093 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
10094 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
10095 LLVMContext::OB_convergencectrl, LLVMContext::OB_deactivation_symbol});
10096
10097 SDValue Callee = getValue(V: I.getCalledOperand());
10098
10099 if (I.hasDeoptState())
10100 LowerCallSiteWithDeoptBundle(Call: &I, Callee, EHPadBB: nullptr);
10101 else
10102 // Check if we can potentially perform a tail call. More detailed checking
10103 // is be done within LowerCallTo, after more information about the call is
10104 // known.
10105 LowerCallTo(CB: I, Callee, isTailCall: I.isTailCall(), isMustTailCall: I.isMustTailCall());
10106}
10107
10108void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(
10109 const CallBase &CB, const BasicBlock *EHPadBB) {
10110 auto PAB = CB.getOperandBundle(Name: "ptrauth");
10111 const Value *CalleeV = CB.getCalledOperand();
10112
10113 // Gather the call ptrauth data from the operand bundle:
10114 // [ i32 <key>, i64 <discriminator> ]
10115 const auto *Key = cast<ConstantInt>(Val: PAB->Inputs[0]);
10116 const Value *Discriminator = PAB->Inputs[1];
10117
10118 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
10119 assert(Discriminator->getType()->isIntegerTy(64) &&
10120 "Invalid ptrauth discriminator");
10121
10122 // Look through ptrauth constants to find the raw callee.
10123 // Do a direct unauthenticated call if we found it and everything matches.
10124 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CalleeV))
10125 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
10126 DL: DAG.getDataLayout()))
10127 return LowerCallTo(CB, Callee: getValue(V: CalleeCPA->getPointer()), isTailCall: CB.isTailCall(),
10128 isMustTailCall: CB.isMustTailCall(), EHPadBB);
10129
10130 // Functions should never be ptrauth-called directly.
10131 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
10132
10133 // Otherwise, do an authenticated indirect call.
10134 TargetLowering::PtrAuthInfo PAI = {.Key: Key->getZExtValue(),
10135 .Discriminator: getValue(V: Discriminator)};
10136
10137 LowerCallTo(CB, Callee: getValue(V: CalleeV), isTailCall: CB.isTailCall(), isMustTailCall: CB.isMustTailCall(),
10138 EHPadBB, PAI: &PAI);
10139}
10140
10141namespace {
10142
10143/// AsmOperandInfo - This contains information for each constraint that we are
10144/// lowering.
10145class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
10146public:
10147 /// CallOperand - If this is the result output operand or a clobber
10148 /// this is null, otherwise it is the incoming operand to the CallInst.
10149 /// This gets modified as the asm is processed.
10150 SDValue CallOperand;
10151
10152 /// AssignedRegs - If this is a register or register class operand, this
10153 /// contains the set of register corresponding to the operand.
10154 RegsForValue AssignedRegs;
10155
10156 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
10157 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
10158 }
10159
10160 /// Whether or not this operand accesses memory
10161 bool hasMemory(const TargetLowering &TLI) const {
10162 // Indirect operand accesses access memory.
10163 if (isIndirect)
10164 return true;
10165
10166 for (const auto &Code : Codes)
10167 if (TLI.getConstraintType(Constraint: Code) == TargetLowering::C_Memory)
10168 return true;
10169
10170 return false;
10171 }
10172};
10173
10174
10175} // end anonymous namespace
10176
10177/// Make sure that the output operand \p OpInfo and its corresponding input
10178/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
10179/// out).
10180static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
10181 SDISelAsmOperandInfo &MatchingOpInfo,
10182 SelectionDAG &DAG) {
10183 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
10184 return;
10185
10186 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
10187 const auto &TLI = DAG.getTargetLoweringInfo();
10188
10189 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
10190 TLI.getRegForInlineAsmConstraint(TRI, Constraint: OpInfo.ConstraintCode,
10191 VT: OpInfo.ConstraintVT);
10192 std::pair<unsigned, const TargetRegisterClass *> InputRC =
10193 TLI.getRegForInlineAsmConstraint(TRI, Constraint: MatchingOpInfo.ConstraintCode,
10194 VT: MatchingOpInfo.ConstraintVT);
10195 const bool OutOpIsIntOrFP =
10196 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
10197 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
10198 MatchingOpInfo.ConstraintVT.isFloatingPoint();
10199 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
10200 // FIXME: error out in a more elegant fashion
10201 report_fatal_error(reason: "Unsupported asm: input constraint"
10202 " with a matching output constraint of"
10203 " incompatible type!");
10204 }
10205 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
10206}
10207
10208/// Get a direct memory input to behave well as an indirect operand.
10209/// This may introduce stores, hence the need for a \p Chain.
10210/// \return The (possibly updated) chain.
10211static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
10212 SDISelAsmOperandInfo &OpInfo,
10213 SelectionDAG &DAG) {
10214 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10215
10216 // If we don't have an indirect input, put it in the constpool if we can,
10217 // otherwise spill it to a stack slot.
10218 // TODO: This isn't quite right. We need to handle these according to
10219 // the addressing mode that the constraint wants. Also, this may take
10220 // an additional register for the computation and we don't want that
10221 // either.
10222
10223 // If the operand is a float, integer, or vector constant, spill to a
10224 // constant pool entry to get its address.
10225 const Value *OpVal = OpInfo.CallOperandVal;
10226 if (isa<ConstantFP>(Val: OpVal) || isa<ConstantInt>(Val: OpVal) ||
10227 isa<ConstantVector>(Val: OpVal) || isa<ConstantDataVector>(Val: OpVal)) {
10228 OpInfo.CallOperand = DAG.getConstantPool(
10229 C: cast<Constant>(Val: OpVal), VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
10230 return Chain;
10231 }
10232
10233 // Otherwise, create a stack slot and emit a store to it before the asm.
10234 Type *Ty = OpVal->getType();
10235 auto &DL = DAG.getDataLayout();
10236 TypeSize TySize = DL.getTypeAllocSize(Ty);
10237 MachineFunction &MF = DAG.getMachineFunction();
10238 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
10239 int StackID = 0;
10240 if (TySize.isScalable())
10241 StackID = TFI->getStackIDForScalableVectors();
10242 int SSFI = MF.getFrameInfo().CreateStackObject(Size: TySize.getKnownMinValue(),
10243 Alignment: DL.getPrefTypeAlign(Ty), isSpillSlot: false,
10244 Alloca: nullptr, ID: StackID);
10245 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: TLI.getFrameIndexTy(DL));
10246 Chain = DAG.getTruncStore(Chain, dl: Location, Val: OpInfo.CallOperand, Ptr: StackSlot,
10247 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: SSFI),
10248 SVT: TLI.getMemValueType(DL, Ty));
10249 OpInfo.CallOperand = StackSlot;
10250
10251 return Chain;
10252}
10253
10254/// GetRegistersForValue - Assign registers (virtual or physical) for the
10255/// specified operand. We prefer to assign virtual registers, to allow the
10256/// register allocator to handle the assignment process. However, if the asm
10257/// uses features that we can't model on machineinstrs, we have SDISel do the
10258/// allocation. This produces generally horrible, but correct, code.
10259///
10260/// OpInfo describes the operand
10261/// RefOpInfo describes the matching operand if any, the operand otherwise
10262static std::optional<unsigned>
10263getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
10264 SDISelAsmOperandInfo &OpInfo,
10265 SDISelAsmOperandInfo &RefOpInfo) {
10266 LLVMContext &Context = *DAG.getContext();
10267 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10268
10269 MachineFunction &MF = DAG.getMachineFunction();
10270 SmallVector<Register, 4> Regs;
10271 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10272
10273 // No work to do for memory/address operands.
10274 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10275 OpInfo.ConstraintType == TargetLowering::C_Address)
10276 return std::nullopt;
10277
10278 // If this is a constraint for a single physreg, or a constraint for a
10279 // register class, find it.
10280 unsigned AssignedReg;
10281 const TargetRegisterClass *RC;
10282 std::tie(args&: AssignedReg, args&: RC) = TLI.getRegForInlineAsmConstraint(
10283 TRI: &TRI, Constraint: RefOpInfo.ConstraintCode, VT: RefOpInfo.ConstraintVT);
10284 // RC is unset only on failure. Return immediately.
10285 if (!RC)
10286 return std::nullopt;
10287
10288 // Get the actual register value type. This is important, because the user
10289 // may have asked for (e.g.) the AX register in i32 type. We need to
10290 // remember that AX is actually i16 to get the right extension.
10291 const MVT RegVT = *TRI.legalclasstypes_begin(RC: *RC);
10292
10293 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10294 // If this is an FP operand in an integer register (or visa versa), or more
10295 // generally if the operand value disagrees with the register class we plan
10296 // to stick it in, fix the operand type.
10297 //
10298 // If this is an input value, the bitcast to the new type is done now.
10299 // Bitcast for output value is done at the end of visitInlineAsm().
10300 if ((OpInfo.Type == InlineAsm::isOutput ||
10301 OpInfo.Type == InlineAsm::isInput) &&
10302 !TRI.isTypeLegalForClass(RC: *RC, T: OpInfo.ConstraintVT)) {
10303 // Try to convert to the first EVT that the reg class contains. If the
10304 // types are identical size, use a bitcast to convert (e.g. two differing
10305 // vector types). Note: output bitcast is done at the end of
10306 // visitInlineAsm().
10307 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10308 // Exclude indirect inputs while they are unsupported because the code
10309 // to perform the load is missing and thus OpInfo.CallOperand still
10310 // refers to the input address rather than the pointed-to value.
10311 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
10312 OpInfo.CallOperand =
10313 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: RegVT, Operand: OpInfo.CallOperand);
10314 OpInfo.ConstraintVT = RegVT;
10315 // If the operand is an FP value and we want it in integer registers,
10316 // use the corresponding integer type. This turns an f64 value into
10317 // i64, which can be passed with two i32 values on a 32-bit machine.
10318 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10319 MVT VT = MVT::getIntegerVT(BitWidth: OpInfo.ConstraintVT.getSizeInBits());
10320 if (OpInfo.Type == InlineAsm::isInput)
10321 OpInfo.CallOperand =
10322 DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: OpInfo.CallOperand);
10323 OpInfo.ConstraintVT = VT;
10324 }
10325 }
10326 }
10327
10328 // No need to allocate a matching input constraint since the constraint it's
10329 // matching to has already been allocated.
10330 if (OpInfo.isMatchingInputConstraint())
10331 return std::nullopt;
10332
10333 EVT ValueVT = OpInfo.ConstraintVT;
10334 if (OpInfo.ConstraintVT == MVT::Other)
10335 ValueVT = RegVT;
10336
10337 // Initialize NumRegs.
10338 unsigned NumRegs = 1;
10339 if (OpInfo.ConstraintVT != MVT::Other)
10340 NumRegs = TLI.getNumRegisters(Context, VT: OpInfo.ConstraintVT, RegisterVT: RegVT);
10341
10342 // If this is a constraint for a specific physical register, like {r17},
10343 // assign it now.
10344
10345 // If this associated to a specific register, initialize iterator to correct
10346 // place. If virtual, make sure we have enough registers
10347
10348 // Initialize iterator if necessary
10349 TargetRegisterClass::iterator I = RC->begin();
10350 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10351
10352 // Do not check for single registers.
10353 if (AssignedReg) {
10354 I = std::find(first: I, last: RC->end(), val: AssignedReg);
10355 if (I == RC->end()) {
10356 // RC does not contain the selected register, which indicates a
10357 // mismatch between the register and the required type/bitwidth.
10358 return {AssignedReg};
10359 }
10360 }
10361
10362 for (; NumRegs; --NumRegs, ++I) {
10363 assert(I != RC->end() && "Ran out of registers to allocate!");
10364 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RegClass: RC);
10365 Regs.push_back(Elt: R);
10366 }
10367
10368 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
10369 return std::nullopt;
10370}
10371
10372static unsigned
10373findMatchingInlineAsmOperand(unsigned OperandNo,
10374 const std::vector<SDValue> &AsmNodeOperands) {
10375 // Scan until we find the definition we already emitted of this operand.
10376 unsigned CurOp = InlineAsm::Op_FirstOperand;
10377 for (; OperandNo; --OperandNo) {
10378 // Advance to the next operand.
10379 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10380 const InlineAsm::Flag F(OpFlag);
10381 assert(
10382 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
10383 "Skipped past definitions?");
10384 CurOp += F.getNumOperandRegisters() + 1;
10385 }
10386 return CurOp;
10387}
10388
10389namespace {
10390
10391class ExtraFlags {
10392 unsigned Flags = 0;
10393
10394public:
10395 explicit ExtraFlags(const CallBase &Call) {
10396 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10397 if (IA->hasSideEffects())
10398 Flags |= InlineAsm::Extra_HasSideEffects;
10399 if (IA->isAlignStack())
10400 Flags |= InlineAsm::Extra_IsAlignStack;
10401 if (IA->canThrow())
10402 Flags |= InlineAsm::Extra_MayUnwind;
10403 if (Call.isConvergent())
10404 Flags |= InlineAsm::Extra_IsConvergent;
10405 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
10406 }
10407
10408 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
10409 // Ideally, we would only check against memory constraints. However, the
10410 // meaning of an Other constraint can be target-specific and we can't easily
10411 // reason about it. Therefore, be conservative and set MayLoad/MayStore
10412 // for Other constraints as well.
10413 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10414 OpInfo.ConstraintType == TargetLowering::C_Other) {
10415 if (OpInfo.Type == InlineAsm::isInput)
10416 Flags |= InlineAsm::Extra_MayLoad;
10417 else if (OpInfo.Type == InlineAsm::isOutput)
10418 Flags |= InlineAsm::Extra_MayStore;
10419 else if (OpInfo.Type == InlineAsm::isClobber)
10420 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
10421 }
10422 }
10423
10424 unsigned get() const { return Flags; }
10425};
10426
10427} // end anonymous namespace
10428
10429static bool isFunction(SDValue Op) {
10430 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
10431 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
10432 auto Fn = dyn_cast_or_null<Function>(Val: GA->getGlobal());
10433
10434 // In normal "call dllimport func" instruction (non-inlineasm) it force
10435 // indirect access by specifing call opcode. And usually specially print
10436 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
10437 // not do in this way now. (In fact, this is similar with "Data Access"
10438 // action). So here we ignore dllimport function.
10439 if (Fn && !Fn->hasDLLImportStorageClass())
10440 return true;
10441 }
10442 }
10443 return false;
10444}
10445
10446namespace {
10447
10448struct ConstraintDecisionInfo {
10449 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
10450 std::vector<SDValue> AsmNodeOperands;
10451 SDValue Glue, Chain;
10452 bool HasSideEffect = false;
10453 MCSymbol *BeginLabel = nullptr;
10454
10455 SmallVector<char> Buffer;
10456 raw_svector_ostream ErrorMsg;
10457
10458 ConstraintDecisionInfo() : ErrorMsg(Buffer) {}
10459};
10460
10461} // end anonymous namespace
10462
10463/// Construct operand info objects.
10464static bool
10465constructOperandInfo(ConstraintDecisionInfo &Info,
10466 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10467 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10468 ExtraFlags &ExtraInfo) {
10469 for (auto &T : TargetConstraints) {
10470 Info.ConstraintOperands.push_back(Elt: SDISelAsmOperandInfo(T));
10471 SDISelAsmOperandInfo &OpInfo = Info.ConstraintOperands.back();
10472
10473 if (OpInfo.CallOperandVal)
10474 OpInfo.CallOperand = Builder.getValue(V: OpInfo.CallOperandVal);
10475
10476 if (!Info.HasSideEffect)
10477 Info.HasSideEffect = OpInfo.hasMemory(TLI);
10478
10479 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
10480 // FIXME: Could we compute this on OpInfo rather than T?
10481
10482 // Compute the constraint code and ConstraintType to use.
10483 TLI.ComputeConstraintToUse(OpInfo&: T, Op: SDValue());
10484
10485 if (T.ConstraintType == TargetLowering::C_Immediate && OpInfo.CallOperand &&
10486 !isa<ConstantSDNode>(Val: OpInfo.CallOperand)) {
10487 // We've delayed emitting a diagnostic like the "n" constraint because
10488 // inlining could cause an integer showing up.
10489 Info.ErrorMsg << "constraint '" << T.ConstraintCode
10490 << "' expects an integer constant expression";
10491 return true;
10492 }
10493
10494 ExtraInfo.update(OpInfo: T);
10495 }
10496
10497 return false;
10498}
10499
10500/// Compute which constraint option to use for each operand.
10501static void
10502computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
10503 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10504 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10505 const TargetMachine &TM, SelectionDAG &DAG) {
10506 const auto *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10507 SmallVector<StringRef, 4> AsmStrs;
10508 IA->collectAsmStrs(AsmStrs);
10509
10510 int OpNo = -1;
10511 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10512 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10513 OpNo++;
10514
10515 // If this is an output operand with a matching input operand, look up the
10516 // matching input. If their types mismatch, e.g. one is an integer, the
10517 // other is floating point, or their sizes are different, flag it as an
10518 // error.
10519 if (OpInfo.hasMatchingInput()) {
10520 SDISelAsmOperandInfo &Input =
10521 Info.ConstraintOperands[OpInfo.MatchingInput];
10522 patchMatchingInput(OpInfo, MatchingOpInfo&: Input, DAG);
10523 }
10524
10525 // Compute the constraint code and ConstraintType to use.
10526 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
10527
10528 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10529 OpInfo.Type == InlineAsm::isClobber) ||
10530 OpInfo.ConstraintType == TargetLowering::C_Address)
10531 continue;
10532
10533 // In Linux PIC model, there are 4 cases about value/label addressing:
10534 //
10535 // 1: Function call or Label jmp inside the module.
10536 // 2: Data access (such as global variable, static variable) inside module.
10537 // 3: Function call or Label jmp outside the module.
10538 // 4: Data access (such as global variable) outside the module.
10539 //
10540 // Due to current llvm inline asm architecture designed to not "recognize"
10541 // the asm code, there are quite troubles for us to treat mem addressing
10542 // differently for same value/adress used in different instuctions.
10543 // For example, in pic model, call a func may in plt way or direclty
10544 // pc-related, but lea/mov a function adress may use got.
10545 //
10546 // Here we try to "recognize" function call for the case 1 and case 3 in
10547 // inline asm. And try to adjust the constraint for them.
10548 //
10549 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10550 // label, so here we don't handle jmp function label now, but we need to
10551 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10552 if (OpInfo.isIndirect && isFunction(Op: OpInfo.CallOperand) &&
10553 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10554 TM.getCodeModel() != CodeModel::Large) {
10555 OpInfo.isIndirect = false;
10556 OpInfo.ConstraintType = TargetLowering::C_Address;
10557 }
10558
10559 // If this is a memory input, and if the operand is not indirect, do what we
10560 // need to provide an address for the memory input.
10561 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10562 !OpInfo.isIndirect) {
10563 assert((OpInfo.isMultipleAlternative ||
10564 (OpInfo.Type == InlineAsm::isInput)) &&
10565 "Can only indirectify direct input operands!");
10566
10567 // Memory operands really want the address of the value.
10568 Info.Chain = getAddressForMemoryInput(Chain: Info.Chain, Location: Builder.getCurSDLoc(),
10569 OpInfo, DAG);
10570
10571 // There is no longer a Value* corresponding to this operand.
10572 OpInfo.CallOperandVal = nullptr;
10573
10574 // It is now an indirect operand.
10575 OpInfo.isIndirect = true;
10576 }
10577 }
10578}
10579
10580/// Prepare DAG-level operands. As part of this, assign virtual and physical
10581/// registers for inputs and output.
10582static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info,
10583 const CallBase &Call,
10584 SelectionDAGBuilder &Builder,
10585 const TargetLowering &TLI,
10586 SelectionDAG &DAG) {
10587 SDLoc DL = Builder.getCurSDLoc();
10588 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10589 // Assign Registers.
10590 SDISelAsmOperandInfo &RefOpInfo =
10591 OpInfo.isMatchingInputConstraint()
10592 ? Info.ConstraintOperands[OpInfo.getMatchedOperand()]
10593 : OpInfo;
10594 const auto RegError = getRegistersForValue(DAG, DL, OpInfo, RefOpInfo);
10595 if (RegError) {
10596 const MachineFunction &MF = DAG.getMachineFunction();
10597 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10598 const char *RegName = TRI.getName(RegNo: *RegError);
10599 Info.ErrorMsg << "register '" << RegName << "' allocated for constraint '"
10600 << OpInfo.ConstraintCode
10601 << "' does not match required type";
10602 return true;
10603 }
10604
10605 auto DetectWriteToReservedRegister = [&]() {
10606 const MachineFunction &MF = DAG.getMachineFunction();
10607 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10608
10609 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10610 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, PhysReg: Reg)) {
10611 Info.ErrorMsg << "write to reserved register '"
10612 << TRI.getRegAsmName(Reg) << "'";
10613 return true;
10614 }
10615 }
10616
10617 return false;
10618 };
10619 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10620 (OpInfo.Type == InlineAsm::isInput &&
10621 !OpInfo.isMatchingInputConstraint())) &&
10622 "Only address as input operand is allowed.");
10623
10624 switch (OpInfo.Type) {
10625 case InlineAsm::isOutput:
10626 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10627 const InlineAsm::ConstraintCode ConstraintID =
10628 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10629 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10630 "Failed to convert memory constraint code to constraint id.");
10631
10632 // Add information to the INLINEASM node to know about this output.
10633 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
10634 OpFlags.setMemConstraint(ConstraintID);
10635 Info.AsmNodeOperands.push_back(
10636 x: DAG.getTargetConstant(Val: OpFlags, DL, VT: MVT::i32));
10637 Info.AsmNodeOperands.push_back(x: OpInfo.CallOperand);
10638 } else {
10639 // Otherwise, this outputs to a register (directly for C_Register /
10640 // C_RegisterClass, and a target-defined fashion for
10641 // C_Immediate/C_Other). Find a register that we can use.
10642 if (OpInfo.AssignedRegs.Regs.empty()) {
10643 Info.ErrorMsg << "could not allocate output register for "
10644 << "constraint '" << OpInfo.ConstraintCode << "'";
10645 return true;
10646 }
10647
10648 if (DetectWriteToReservedRegister())
10649 return true;
10650
10651 // Add information to the INLINEASM node to know that this register is
10652 // set.
10653 OpInfo.AssignedRegs.AddInlineAsmOperands(
10654 Code: OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10655 : InlineAsm::Kind::RegDef,
10656 HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10657 }
10658 break;
10659
10660 case InlineAsm::isInput:
10661 case InlineAsm::isLabel: {
10662 SDValue InOperandVal = OpInfo.CallOperand;
10663
10664 if (OpInfo.isMatchingInputConstraint()) {
10665 // If this is required to match an output register we have already set,
10666 // just use its register.
10667 auto CurOp = findMatchingInlineAsmOperand(OperandNo: OpInfo.getMatchedOperand(),
10668 AsmNodeOperands: Info.AsmNodeOperands);
10669 InlineAsm::Flag Flag(Info.AsmNodeOperands[CurOp]->getAsZExtVal());
10670 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10671 if (OpInfo.isIndirect) {
10672 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10673 Info.ErrorMsg << "inline asm not supported yet: cannot handle "
10674 << "tied indirect register inputs";
10675 return true;
10676 }
10677
10678 SmallVector<Register, 4> Regs;
10679 MachineFunction &MF = DAG.getMachineFunction();
10680 MachineRegisterInfo &MRI = MF.getRegInfo();
10681 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10682 auto *R = cast<RegisterSDNode>(Val&: Info.AsmNodeOperands[CurOp + 1]);
10683 Register TiedReg = R->getReg();
10684 MVT RegVT = R->getSimpleValueType(ResNo: 0);
10685 const TargetRegisterClass *RC =
10686 TiedReg.isVirtual() ? MRI.getRegClass(Reg: TiedReg)
10687 : RegVT != MVT::Untyped ? TLI.getRegClassFor(VT: RegVT)
10688 : TRI.getMinimalPhysRegClass(Reg: TiedReg);
10689 for (unsigned I = 0, E = Flag.getNumOperandRegisters(); I != E; ++I)
10690 Regs.push_back(Elt: MRI.createVirtualRegister(RegClass: RC));
10691
10692 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10693
10694 // Use the produced MatchedRegs object to
10695 MatchedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl: DL, Chain&: Info.Chain,
10696 Glue: &Info.Glue, V: &Call);
10697 MatchedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::RegUse, HasMatching: true,
10698 MatchingIdx: OpInfo.getMatchedOperand(), dl: DL, DAG,
10699 Ops&: Info.AsmNodeOperands);
10700 break;
10701 }
10702
10703 assert(Flag.isMemKind() && "Unknown matching constraint!");
10704 assert(Flag.getNumOperandRegisters() == 1 &&
10705 "Unexpected number of operands");
10706
10707 // Add information to the INLINEASM node to know about this input.
10708 // See InlineAsm.h isUseOperandTiedToDef.
10709 Flag.clearMemConstraint();
10710 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10711 Info.AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10712 Val: Flag, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10713 Info.AsmNodeOperands.push_back(x: Info.AsmNodeOperands[CurOp + 1]);
10714 break;
10715 }
10716
10717 // Treat indirect 'X' constraint as memory.
10718 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10719 OpInfo.isIndirect)
10720 OpInfo.ConstraintType = TargetLowering::C_Memory;
10721
10722 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10723 OpInfo.ConstraintType == TargetLowering::C_Other) {
10724 std::vector<SDValue> Ops;
10725 TLI.LowerAsmOperandForConstraint(Op: InOperandVal, Constraint: OpInfo.ConstraintCode,
10726 Ops, DAG);
10727 if (Ops.empty()) {
10728 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10729 if (isa<ConstantSDNode>(Val: InOperandVal)) {
10730 Info.ErrorMsg << "value out of range for constraint '"
10731 << OpInfo.ConstraintCode << "'";
10732 return true;
10733 }
10734
10735 Info.ErrorMsg << "invalid operand for inline asm constraint '"
10736 << OpInfo.ConstraintCode << "'";
10737 return true;
10738 }
10739
10740 // Add information to the INLINEASM node to know about this input.
10741 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10742 Info.AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10743 Val: ResOpType, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10744 llvm::append_range(C&: Info.AsmNodeOperands, R&: Ops);
10745 break;
10746 }
10747
10748 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10749 assert((OpInfo.isIndirect ||
10750 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10751 "Operand must be indirect to be a mem!");
10752 assert(InOperandVal.getValueType() ==
10753 TLI.getPointerTy(DAG.getDataLayout()) &&
10754 "Memory operands expect pointer values");
10755
10756 const InlineAsm::ConstraintCode ConstraintID =
10757 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10758 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10759 "Failed to convert memory constraint code to constraint id.");
10760
10761 // Add information to the INLINEASM node to know about this input.
10762 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10763 ResOpType.setMemConstraint(ConstraintID);
10764 Info.AsmNodeOperands.push_back(
10765 x: DAG.getTargetConstant(Val: ResOpType, DL, VT: MVT::i32));
10766 Info.AsmNodeOperands.push_back(x: InOperandVal);
10767 break;
10768 }
10769
10770 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10771 const InlineAsm::ConstraintCode ConstraintID =
10772 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10773 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10774 "Failed to convert memory constraint code to constraint id.");
10775
10776 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10777
10778 SDValue AsmOp = InOperandVal;
10779 if (isFunction(Op: InOperandVal)) {
10780 auto *GA = cast<GlobalAddressSDNode>(Val&: InOperandVal);
10781 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10782 AsmOp = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL,
10783 VT: InOperandVal.getValueType(),
10784 offset: GA->getOffset());
10785 }
10786
10787 // Add information to the INLINEASM node to know about this input.
10788 ResOpType.setMemConstraint(ConstraintID);
10789
10790 Info.AsmNodeOperands.push_back(
10791 x: DAG.getTargetConstant(Val: ResOpType, DL, VT: MVT::i32));
10792 Info.AsmNodeOperands.push_back(x: AsmOp);
10793 break;
10794 }
10795
10796 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10797 OpInfo.ConstraintType != TargetLowering::C_Register) {
10798 Info.ErrorMsg << "unknown asm constraint '" << OpInfo.ConstraintCode
10799 << "'";
10800 return true;
10801 }
10802
10803 // TODO: Support this.
10804 if (OpInfo.isIndirect) {
10805 Info.ErrorMsg << "cannot handle indirect register inputs yet for "
10806 << "constraint '" << OpInfo.ConstraintCode << "'";
10807 return true;
10808 }
10809
10810 // Copy the input into the appropriate registers.
10811 if (OpInfo.AssignedRegs.Regs.empty()) {
10812 Info.ErrorMsg << "could not allocate input reg for constraint '"
10813 << OpInfo.ConstraintCode << "'";
10814 return true;
10815 }
10816
10817 if (DetectWriteToReservedRegister())
10818 return true;
10819
10820 OpInfo.AssignedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl: DL, Chain&: Info.Chain,
10821 Glue: &Info.Glue, V: &Call);
10822 OpInfo.AssignedRegs.AddInlineAsmOperands(
10823 Code: InlineAsm::Kind::RegUse, HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10824 break;
10825 }
10826
10827 case InlineAsm::isClobber:
10828 // Add the clobbered value to the operand list, so that the register
10829 // allocator is aware that the physreg got clobbered.
10830 if (!OpInfo.AssignedRegs.Regs.empty())
10831 OpInfo.AssignedRegs.AddInlineAsmOperands(
10832 Code: InlineAsm::Kind::Clobber, HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10833 break;
10834 }
10835 }
10836
10837 return false;
10838}
10839
10840/// DetermineConstraints - Find the constraints to use for inline asm operands.
10841static bool
10842determineConstraints(ConstraintDecisionInfo &Info,
10843 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10844 const CallBase &Call, SelectionDAGBuilder &Builder,
10845 const TargetLowering &TLI, const TargetMachine &TM,
10846 SelectionDAG &DAG, const BasicBlock *EHPadBB) {
10847 const auto *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10848 ExtraFlags ExtraInfo(Call);
10849
10850 // First pass: Construct operand info objects.
10851 Info.HasSideEffect = IA->hasSideEffects();
10852 if (constructOperandInfo(Info, TargetConstraints, Builder, TLI, ExtraInfo))
10853 return true;
10854
10855 // We won't need to flush pending loads if this asm doesn't touch
10856 // memory and is nonvolatile.
10857 Info.Chain = Info.HasSideEffect ? Builder.getRoot() : DAG.getRoot();
10858
10859 bool IsCallBr = isa<CallBrInst>(Val: Call);
10860 bool EmitEHLabels = isa<InvokeInst>(Val: Call);
10861 if (IsCallBr || EmitEHLabels)
10862 // If this is a callbr or invoke we need to flush pending exports since
10863 // inlineasm_br and invoke are terminators.
10864 // We need to do this before nodes are glued to the inlineasm_br node.
10865 Info.Chain = Builder.getControlRoot();
10866
10867 if (EmitEHLabels)
10868 Info.Chain = Builder.lowerStartEH(Chain: Info.Chain, EHPadBB, BeginLabel&: Info.BeginLabel);
10869
10870 // Second pass: Compute which constraint option to use.
10871 computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM, DAG);
10872
10873 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10874 Info.AsmNodeOperands.push_back(x: SDValue()); // reserve space for input chain
10875 Info.AsmNodeOperands.push_back(x: DAG.getTargetExternalSymbol(
10876 Sym: IA->getAsmString().data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout())));
10877
10878 // If we have a !srcloc metadata node associated with it, we want to attach
10879 // this to the ultimately generated inline asm machineinstr. To do this, we
10880 // pass in the third operand as this (potentially null) inline asm MDNode.
10881 const MDNode *SrcLoc = Call.getMetadata(Kind: "srcloc");
10882 Info.AsmNodeOperands.push_back(x: DAG.getMDNode(MD: SrcLoc));
10883
10884 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10885 // bits as operand 3.
10886 Info.AsmNodeOperands.push_back(
10887 x: DAG.getTargetConstant(Val: ExtraInfo.get(), DL: Builder.getCurSDLoc(),
10888 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10889
10890 // Third pass: Prepare DAG-level operands
10891 return prepareDAGLevelOperands(Info, Call, Builder, TLI, DAG);
10892}
10893
10894/// visitInlineAsm - Handle a call to an InlineAsm object.
10895void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
10896 const BasicBlock *EHPadBB) {
10897 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10898 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
10899 DL: DAG.getDataLayout(), TRI: DAG.getSubtarget().getRegisterInfo(), Call);
10900
10901 assert((!isa<InvokeInst>(Call) || EHPadBB) &&
10902 "InvokeInst must have an EHPadBB");
10903
10904 ConstraintDecisionInfo Info;
10905 if (determineConstraints(Info, TargetConstraints, Call, Builder&: *this, TLI, TM, DAG,
10906 EHPadBB))
10907 return emitInlineAsmError(Call, Message: Info.ErrorMsg.str());
10908
10909 SDValue Glue = Info.Glue;
10910 SDValue Chain = Info.Chain;
10911
10912 // Finish up input operands. Set the input chain and add the flag last.
10913 Info.AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10914 if (Glue.getNode())
10915 Info.AsmNodeOperands.push_back(x: Glue);
10916
10917 bool IsCallBr = isa<CallBrInst>(Val: Call);
10918 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10919 Chain =
10920 DAG.getNode(Opcode: ISDOpc, DL: getCurSDLoc(), VTList: DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue),
10921 Ops: Info.AsmNodeOperands);
10922 Glue = Chain.getValue(R: 1);
10923
10924 // Do additional work to generate outputs.
10925
10926 SmallVector<EVT, 1> ResultVTs;
10927 SmallVector<SDValue, 1> ResultValues;
10928 SmallVector<SDValue, 8> OutChains;
10929
10930 llvm::Type *CallResultType = Call.getType();
10931 ArrayRef<Type *> ResultTypes;
10932 if (StructType *StructResult = dyn_cast<StructType>(Val: CallResultType))
10933 ResultTypes = StructResult->elements();
10934 else if (!CallResultType->isVoidTy())
10935 ResultTypes = ArrayRef(CallResultType);
10936
10937 auto CurResultType = ResultTypes.begin();
10938 auto handleRegAssign = [&](SDValue V) {
10939 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10940 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10941 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: *CurResultType);
10942 ++CurResultType;
10943 // If the type of the inline asm call site return value is different but has
10944 // same size as the type of the asm output bitcast it. One example of this
10945 // is for vectors with different width / number of elements. This can
10946 // happen for register classes that can contain multiple different value
10947 // types. The preg or vreg allocated may not have the same VT as was
10948 // expected.
10949 //
10950 // This can also happen for a return value that disagrees with the register
10951 // class it is put in, eg. a double in a general-purpose register on a
10952 // 32-bit machine.
10953 if (ResultVT != V.getValueType() &&
10954 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10955 V = DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10956 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10957 V.getValueType().isInteger()) {
10958 // If a result value was tied to an input value, the computed result
10959 // may have a wider width than the expected result. Extract the
10960 // relevant portion.
10961 V = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10962 }
10963 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10964 ResultVTs.push_back(Elt: ResultVT);
10965 ResultValues.push_back(Elt: V);
10966 };
10967
10968 // Deal with output operands.
10969 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10970 if (OpInfo.Type == InlineAsm::isOutput) {
10971 SDValue Val;
10972 // Skip trivial output operands.
10973 if (OpInfo.AssignedRegs.Regs.empty())
10974 continue;
10975
10976 switch (OpInfo.ConstraintType) {
10977 case TargetLowering::C_Register:
10978 case TargetLowering::C_RegisterClass:
10979 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(),
10980 Chain, Glue: &Glue, V: &Call);
10981 break;
10982 case TargetLowering::C_Immediate:
10983 case TargetLowering::C_Other:
10984 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, DL: getCurSDLoc(),
10985 OpInfo, DAG);
10986 break;
10987 case TargetLowering::C_Memory:
10988 break; // Already handled.
10989 case TargetLowering::C_Address:
10990 break; // Silence warning.
10991 case TargetLowering::C_Unknown:
10992 assert(false && "Unexpected unknown constraint");
10993 }
10994
10995 // Indirect output manifest as stores. Record output chains.
10996 if (OpInfo.isIndirect) {
10997 const Value *Ptr = OpInfo.CallOperandVal;
10998 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10999 SDValue Store = DAG.getStore(Chain, dl: getCurSDLoc(), Val, Ptr: getValue(V: Ptr),
11000 PtrInfo: MachinePointerInfo(Ptr));
11001 OutChains.push_back(Elt: Store);
11002 } else {
11003 // generate CopyFromRegs to associated registers.
11004 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
11005 if (Val.getOpcode() == ISD::MERGE_VALUES) {
11006 for (const SDValue &V : Val->op_values())
11007 handleRegAssign(V);
11008 } else
11009 handleRegAssign(Val);
11010 }
11011 }
11012 }
11013
11014 // Set results.
11015 if (!ResultValues.empty()) {
11016 assert(CurResultType == ResultTypes.end() &&
11017 "Mismatch in number of ResultTypes");
11018 assert(ResultValues.size() == ResultTypes.size() &&
11019 "Mismatch in number of output operands in asm result");
11020
11021 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
11022 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
11023 setValue(V: &Call, NewN: V);
11024 }
11025
11026 // Collect store chains.
11027 if (!OutChains.empty())
11028 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: getCurSDLoc(), VT: MVT::Other, Ops: OutChains);
11029
11030 if (const auto *II = dyn_cast<InvokeInst>(Val: &Call))
11031 Chain = lowerEndEH(Chain, II, EHPadBB, BeginLabel: Info.BeginLabel);
11032
11033 // Only Update Root if inline assembly has a memory effect.
11034 if (ResultValues.empty() || Info.HasSideEffect || !OutChains.empty() ||
11035 IsCallBr || isa<InvokeInst>(Val: Call))
11036 DAG.setRoot(Chain);
11037}
11038
11039void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
11040 const Twine &Message) {
11041 LLVMContext &Ctx = *DAG.getContext();
11042 Ctx.diagnose(DI: DiagnosticInfoInlineAsm(Call, Message));
11043
11044 // Make sure we leave the DAG in a valid state
11045 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11046 SmallVector<EVT, 1> ValueVTs;
11047 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: Call.getType(), ValueVTs);
11048
11049 if (ValueVTs.empty())
11050 return;
11051
11052 SmallVector<SDValue, 1> Ops;
11053 for (const EVT &VT : ValueVTs)
11054 Ops.push_back(Elt: DAG.getUNDEF(VT));
11055
11056 setValue(V: &Call, NewN: DAG.getMergeValues(Ops, dl: getCurSDLoc()));
11057}
11058
11059void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
11060 DAG.setRoot(DAG.getNode(Opcode: ISD::VASTART, DL: getCurSDLoc(),
11061 VT: MVT::Other, N1: getRoot(),
11062 N2: getValue(V: I.getArgOperand(i: 0)),
11063 N3: DAG.getSrcValue(v: I.getArgOperand(i: 0))));
11064}
11065
11066void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
11067 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11068 const DataLayout &DL = DAG.getDataLayout();
11069 SDValue V = DAG.getVAArg(
11070 VT: TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType()), dl: getCurSDLoc(),
11071 Chain: getRoot(), Ptr: getValue(V: I.getOperand(i_nocapture: 0)), SV: DAG.getSrcValue(v: I.getOperand(i_nocapture: 0)),
11072 Align: DL.getABITypeAlign(Ty: I.getType()).value());
11073 DAG.setRoot(V.getValue(R: 1));
11074
11075 if (I.getType()->isPointerTy())
11076 V = DAG.getPtrExtOrTrunc(
11077 Op: V, DL: getCurSDLoc(), VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()));
11078 setValue(V: &I, NewN: V);
11079}
11080
11081void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
11082 DAG.setRoot(DAG.getNode(Opcode: ISD::VAEND, DL: getCurSDLoc(),
11083 VT: MVT::Other, N1: getRoot(),
11084 N2: getValue(V: I.getArgOperand(i: 0)),
11085 N3: DAG.getSrcValue(v: I.getArgOperand(i: 0))));
11086}
11087
11088void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
11089 DAG.setRoot(DAG.getNode(Opcode: ISD::VACOPY, DL: getCurSDLoc(),
11090 VT: MVT::Other, N1: getRoot(),
11091 N2: getValue(V: I.getArgOperand(i: 0)),
11092 N3: getValue(V: I.getArgOperand(i: 1)),
11093 N4: DAG.getSrcValue(v: I.getArgOperand(i: 0)),
11094 N5: DAG.getSrcValue(v: I.getArgOperand(i: 1))));
11095}
11096
11097SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
11098 const Instruction &I,
11099 SDValue Op) {
11100 std::optional<ConstantRange> CR = getRange(I);
11101
11102 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
11103 return Op;
11104
11105 APInt Hi = CR->getUnsignedMax();
11106 unsigned Bits = std::max(a: Hi.getActiveBits(),
11107 b: static_cast<unsigned>(IntegerType::MIN_INT_BITS));
11108
11109 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
11110
11111 SDLoc SL = getCurSDLoc();
11112
11113 SDValue ZExt = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: Op.getValueType(), N1: Op,
11114 N2: DAG.getValueType(SmallVT));
11115 unsigned NumVals = Op.getNode()->getNumValues();
11116 if (NumVals == 1)
11117 return ZExt;
11118
11119 SmallVector<SDValue, 4> Ops;
11120
11121 Ops.push_back(Elt: ZExt);
11122 for (unsigned I = 1; I != NumVals; ++I)
11123 Ops.push_back(Elt: Op.getValue(R: I));
11124
11125 return DAG.getMergeValues(Ops, dl: SL);
11126}
11127
11128SDValue SelectionDAGBuilder::lowerNoFPClassToAssertNoFPClass(
11129 SelectionDAG &DAG, const Instruction &I, SDValue Op) {
11130 FPClassTest Classes = getNoFPClass(I);
11131 if (Classes == fcNone)
11132 return Op;
11133
11134 SDLoc SL = getCurSDLoc();
11135 SDValue TestConst = DAG.getTargetConstant(Val: Classes, DL: SDLoc(), VT: MVT::i32);
11136
11137 if (Op.getOpcode() != ISD::MERGE_VALUES) {
11138 return DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: SL, VT: Op.getValueType(), N1: Op,
11139 N2: TestConst);
11140 }
11141
11142 SmallVector<SDValue, 8> Ops(Op.getNumOperands());
11143 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
11144 SDValue MergeOp = Op.getOperand(i: I);
11145 Ops[I] = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: SL, VT: MergeOp.getValueType(),
11146 N1: MergeOp, N2: TestConst);
11147 }
11148
11149 return DAG.getMergeValues(Ops, dl: SL);
11150}
11151
11152/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
11153/// the call being lowered.
11154///
11155/// This is a helper for lowering intrinsics that follow a target calling
11156/// convention or require stack pointer adjustment. Only a subset of the
11157/// intrinsic's operands need to participate in the calling convention.
11158void SelectionDAGBuilder::populateCallLoweringInfo(
11159 TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
11160 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
11161 AttributeSet RetAttrs, bool IsPatchPoint) {
11162 TargetLowering::ArgListTy Args;
11163 Args.reserve(n: NumArgs);
11164
11165 // Populate the argument list.
11166 // Attributes for args start at offset 1, after the return attribute.
11167 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
11168 ArgI != ArgE; ++ArgI) {
11169 const Value *V = Call->getOperand(i_nocapture: ArgI);
11170
11171 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
11172
11173 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
11174 Entry.setAttributes(Call, ArgIdx: ArgI);
11175 Args.push_back(x: Entry);
11176 }
11177
11178 CLI.setDebugLoc(getCurSDLoc())
11179 .setChain(getRoot())
11180 .setCallee(CC: Call->getCallingConv(), ResultType: ReturnTy, Target: Callee, ArgsList: std::move(Args),
11181 ResultAttrs: RetAttrs)
11182 .setDiscardResult(Call->use_empty())
11183 .setIsPatchPoint(IsPatchPoint)
11184 .setIsPreallocated(
11185 Call->countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0);
11186}
11187
11188/// Add a stack map intrinsic call's live variable operands to a stackmap
11189/// or patchpoint target node's operand list.
11190///
11191/// Constants are converted to TargetConstants purely as an optimization to
11192/// avoid constant materialization and register allocation.
11193///
11194/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
11195/// generate addess computation nodes, and so FinalizeISel can convert the
11196/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
11197/// address materialization and register allocation, but may also be required
11198/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
11199/// alloca in the entry block, then the runtime may assume that the alloca's
11200/// StackMap location can be read immediately after compilation and that the
11201/// location is valid at any point during execution (this is similar to the
11202/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
11203/// only available in a register, then the runtime would need to trap when
11204/// execution reaches the StackMap in order to read the alloca's location.
11205static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
11206 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
11207 SelectionDAGBuilder &Builder) {
11208 SelectionDAG &DAG = Builder.DAG;
11209 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
11210 SDValue Op = Builder.getValue(V: Call.getArgOperand(i: I));
11211
11212 // Things on the stack are pointer-typed, meaning that they are already
11213 // legal and can be emitted directly to target nodes.
11214 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op)) {
11215 Ops.push_back(Elt: DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: Op.getValueType()));
11216 } else {
11217 // Otherwise emit a target independent node to be legalised.
11218 Ops.push_back(Elt: Builder.getValue(V: Call.getArgOperand(i: I)));
11219 }
11220 }
11221}
11222
11223/// Lower llvm.experimental.stackmap.
11224void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
11225 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
11226 // [live variables...])
11227
11228 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
11229
11230 SDValue Chain, InGlue, Callee;
11231 SmallVector<SDValue, 32> Ops;
11232
11233 SDLoc DL = getCurSDLoc();
11234 Callee = getValue(V: CI.getCalledOperand());
11235
11236 // The stackmap intrinsic only records the live variables (the arguments
11237 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
11238 // intrinsic, this won't be lowered to a function call. This means we don't
11239 // have to worry about calling conventions and target specific lowering code.
11240 // Instead we perform the call lowering right here.
11241 //
11242 // chain, flag = CALLSEQ_START(chain, 0, 0)
11243 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
11244 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
11245 //
11246 Chain = DAG.getCALLSEQ_START(Chain: getRoot(), InSize: 0, OutSize: 0, DL);
11247 InGlue = Chain.getValue(R: 1);
11248
11249 // Add the STACKMAP operands, starting with DAG house-keeping.
11250 Ops.push_back(Elt: Chain);
11251 Ops.push_back(Elt: InGlue);
11252
11253 // Add the <id>, <numShadowBytes> operands.
11254 //
11255 // These do not require legalisation, and can be emitted directly to target
11256 // constant nodes.
11257 SDValue ID = getValue(V: CI.getArgOperand(i: 0));
11258 assert(ID.getValueType() == MVT::i64);
11259 SDValue IDConst =
11260 DAG.getTargetConstant(Val: ID->getAsZExtVal(), DL, VT: ID.getValueType());
11261 Ops.push_back(Elt: IDConst);
11262
11263 SDValue Shad = getValue(V: CI.getArgOperand(i: 1));
11264 assert(Shad.getValueType() == MVT::i32);
11265 SDValue ShadConst =
11266 DAG.getTargetConstant(Val: Shad->getAsZExtVal(), DL, VT: Shad.getValueType());
11267 Ops.push_back(Elt: ShadConst);
11268
11269 // Add the live variables.
11270 addStackMapLiveVars(Call: CI, StartIdx: 2, DL, Ops, Builder&: *this);
11271
11272 // Create the STACKMAP node.
11273 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
11274 Chain = DAG.getNode(Opcode: ISD::STACKMAP, DL, VTList: NodeTys, Ops);
11275 InGlue = Chain.getValue(R: 1);
11276
11277 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: InGlue, DL);
11278
11279 // Stackmaps don't generate values, so nothing goes into the NodeMap.
11280
11281 // Set the root to the target-lowered call chain.
11282 DAG.setRoot(Chain);
11283
11284 // Inform the Frame Information that we have a stackmap in this function.
11285 FuncInfo.MF->getFrameInfo().setHasStackMap();
11286}
11287
11288/// Lower llvm.experimental.patchpoint directly to its target opcode.
11289void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
11290 const BasicBlock *EHPadBB) {
11291 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
11292 // i32 <numBytes>,
11293 // i8* <target>,
11294 // i32 <numArgs>,
11295 // [Args...],
11296 // [live variables...])
11297
11298 CallingConv::ID CC = CB.getCallingConv();
11299 bool IsAnyRegCC = CC == CallingConv::AnyReg;
11300 bool HasDef = !CB.getType()->isVoidTy();
11301 SDLoc dl = getCurSDLoc();
11302 SDValue Callee = getValue(V: CB.getArgOperand(i: PatchPointOpers::TargetPos));
11303
11304 // Handle immediate and symbolic callees.
11305 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Val&: Callee))
11306 Callee = DAG.getIntPtrConstant(Val: ConstCallee->getZExtValue(), DL: dl,
11307 /*isTarget=*/true);
11308 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11309 Callee = DAG.getTargetGlobalAddress(GV: SymbolicCallee->getGlobal(),
11310 DL: SDLoc(SymbolicCallee),
11311 VT: SymbolicCallee->getValueType(ResNo: 0));
11312
11313 // Get the real number of arguments participating in the call <numArgs>
11314 SDValue NArgVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NArgPos));
11315 unsigned NumArgs = NArgVal->getAsZExtVal();
11316
11317 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
11318 // Intrinsics include all meta-operands up to but not including CC.
11319 unsigned NumMetaOpers = PatchPointOpers::CCPos;
11320 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
11321 "Not enough arguments provided to the patchpoint intrinsic");
11322
11323 // For AnyRegCC the arguments are lowered later on manually.
11324 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11325 Type *ReturnTy =
11326 IsAnyRegCC ? Type::getVoidTy(C&: *DAG.getContext()) : CB.getType();
11327
11328 TargetLowering::CallLoweringInfo CLI(DAG);
11329 populateCallLoweringInfo(CLI, Call: &CB, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee,
11330 ReturnTy, RetAttrs: CB.getAttributes().getRetAttrs(), IsPatchPoint: true);
11331 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
11332
11333 SDNode *CallEnd = Result.second.getNode();
11334 if (CallEnd->getOpcode() == ISD::EH_LABEL)
11335 CallEnd = CallEnd->getOperand(Num: 0).getNode();
11336 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
11337 CallEnd = CallEnd->getOperand(Num: 0).getNode();
11338
11339 /// Get a call instruction from the call sequence chain.
11340 /// Tail calls are not allowed.
11341 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
11342 "Expected a callseq node.");
11343 SDNode *Call = CallEnd->getOperand(Num: 0).getNode();
11344 bool HasGlue = Call->getGluedNode();
11345
11346 // Replace the target specific call node with the patchable intrinsic.
11347 SmallVector<SDValue, 8> Ops;
11348
11349 // Push the chain.
11350 Ops.push_back(Elt: *(Call->op_begin()));
11351
11352 // Optionally, push the glue (if any).
11353 if (HasGlue)
11354 Ops.push_back(Elt: *(Call->op_end() - 1));
11355
11356 // Push the register mask info.
11357 if (HasGlue)
11358 Ops.push_back(Elt: *(Call->op_end() - 2));
11359 else
11360 Ops.push_back(Elt: *(Call->op_end() - 1));
11361
11362 // Add the <id> and <numBytes> constants.
11363 SDValue IDVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::IDPos));
11364 Ops.push_back(Elt: DAG.getTargetConstant(Val: IDVal->getAsZExtVal(), DL: dl, VT: MVT::i64));
11365 SDValue NBytesVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NBytesPos));
11366 Ops.push_back(Elt: DAG.getTargetConstant(Val: NBytesVal->getAsZExtVal(), DL: dl, VT: MVT::i32));
11367
11368 // Add the callee.
11369 Ops.push_back(Elt: Callee);
11370
11371 // Adjust <numArgs> to account for any arguments that have been passed on the
11372 // stack instead.
11373 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
11374 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
11375 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11376 Ops.push_back(Elt: DAG.getTargetConstant(Val: NumCallRegArgs, DL: dl, VT: MVT::i32));
11377
11378 // Add the calling convention
11379 Ops.push_back(Elt: DAG.getTargetConstant(Val: (unsigned)CC, DL: dl, VT: MVT::i32));
11380
11381 // Add the arguments we omitted previously. The register allocator should
11382 // place these in any free register.
11383 if (IsAnyRegCC)
11384 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
11385 Ops.push_back(Elt: getValue(V: CB.getArgOperand(i)));
11386
11387 // Push the arguments from the call instruction.
11388 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
11389 Ops.append(in_start: Call->op_begin() + 2, in_end: e);
11390
11391 // Push live variables for the stack map.
11392 addStackMapLiveVars(Call: CB, StartIdx: NumMetaOpers + NumArgs, DL: dl, Ops, Builder&: *this);
11393
11394 SDVTList NodeTys;
11395 if (IsAnyRegCC && HasDef) {
11396 // Create the return types based on the intrinsic definition
11397 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11398 SmallVector<EVT, 3> ValueVTs;
11399 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: CB.getType(), ValueVTs);
11400 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
11401
11402 // There is always a chain and a glue type at the end
11403 ValueVTs.push_back(Elt: MVT::Other);
11404 ValueVTs.push_back(Elt: MVT::Glue);
11405 NodeTys = DAG.getVTList(VTs: ValueVTs);
11406 } else
11407 NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
11408
11409 // Replace the target specific call node with a PATCHPOINT node.
11410 SDValue PPV = DAG.getNode(Opcode: ISD::PATCHPOINT, DL: dl, VTList: NodeTys, Ops);
11411
11412 // Update the NodeMap.
11413 if (HasDef) {
11414 if (IsAnyRegCC)
11415 setValue(V: &CB, NewN: SDValue(PPV.getNode(), 0));
11416 else
11417 setValue(V: &CB, NewN: Result.first);
11418 }
11419
11420 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
11421 // call sequence. Furthermore the location of the chain and glue can change
11422 // when the AnyReg calling convention is used and the intrinsic returns a
11423 // value.
11424 if (IsAnyRegCC && HasDef) {
11425 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
11426 SDValue To[] = {PPV.getValue(R: 1), PPV.getValue(R: 2)};
11427 DAG.ReplaceAllUsesOfValuesWith(From, To, Num: 2);
11428 } else
11429 DAG.ReplaceAllUsesWith(From: Call, To: PPV.getNode());
11430 DAG.DeleteNode(N: Call);
11431
11432 // Inform the Frame Information that we have a patchpoint in this function.
11433 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11434}
11435
11436void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
11437 unsigned Intrinsic) {
11438 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11439 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
11440 SDValue Op2;
11441 if (I.arg_size() > 1)
11442 Op2 = getValue(V: I.getArgOperand(i: 1));
11443 SDLoc dl = getCurSDLoc();
11444 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
11445 SDValue Res;
11446 SDNodeFlags SDFlags;
11447 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
11448 SDFlags.copyFMF(FPMO: *FPMO);
11449
11450 switch (Intrinsic) {
11451 case Intrinsic::vector_reduce_fadd:
11452 if (SDFlags.hasAllowReassociation())
11453 Res = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Op1,
11454 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FADD, DL: dl, VT, Operand: Op2, Flags: SDFlags),
11455 Flags: SDFlags);
11456 else
11457 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FADD, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
11458 break;
11459 case Intrinsic::vector_reduce_fmul:
11460 if (SDFlags.hasAllowReassociation())
11461 Res = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Op1,
11462 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FMUL, DL: dl, VT, Operand: Op2, Flags: SDFlags),
11463 Flags: SDFlags);
11464 else
11465 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FMUL, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
11466 break;
11467 case Intrinsic::vector_reduce_add:
11468 Res = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT, Operand: Op1);
11469 break;
11470 case Intrinsic::vector_reduce_mul:
11471 Res = DAG.getNode(Opcode: ISD::VECREDUCE_MUL, DL: dl, VT, Operand: Op1);
11472 break;
11473 case Intrinsic::vector_reduce_and:
11474 Res = DAG.getNode(Opcode: ISD::VECREDUCE_AND, DL: dl, VT, Operand: Op1);
11475 break;
11476 case Intrinsic::vector_reduce_or:
11477 Res = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: dl, VT, Operand: Op1);
11478 break;
11479 case Intrinsic::vector_reduce_xor:
11480 Res = DAG.getNode(Opcode: ISD::VECREDUCE_XOR, DL: dl, VT, Operand: Op1);
11481 break;
11482 case Intrinsic::vector_reduce_smax:
11483 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMAX, DL: dl, VT, Operand: Op1);
11484 break;
11485 case Intrinsic::vector_reduce_smin:
11486 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMIN, DL: dl, VT, Operand: Op1);
11487 break;
11488 case Intrinsic::vector_reduce_umax:
11489 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL: dl, VT, Operand: Op1);
11490 break;
11491 case Intrinsic::vector_reduce_umin:
11492 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMIN, DL: dl, VT, Operand: Op1);
11493 break;
11494 case Intrinsic::vector_reduce_fmax:
11495 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAX, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11496 break;
11497 case Intrinsic::vector_reduce_fmin:
11498 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMIN, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11499 break;
11500 case Intrinsic::vector_reduce_fmaximum:
11501 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAXIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11502 break;
11503 case Intrinsic::vector_reduce_fminimum:
11504 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMINIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11505 break;
11506 case Intrinsic::vector_reduce_fmaximumnum:
11507 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAXIMUMNUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11508 break;
11509 case Intrinsic::vector_reduce_fminimumnum:
11510 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMINIMUMNUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11511 break;
11512 default:
11513 llvm_unreachable("Unhandled vector reduce intrinsic");
11514 }
11515 setValue(V: &I, NewN: Res);
11516}
11517
11518/// Returns an AttributeList representing the attributes applied to the return
11519/// value of the given call.
11520static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
11521 SmallVector<Attribute::AttrKind, 2> Attrs;
11522 if (CLI.RetSExt)
11523 Attrs.push_back(Elt: Attribute::SExt);
11524 if (CLI.RetZExt)
11525 Attrs.push_back(Elt: Attribute::ZExt);
11526 if (CLI.IsInReg)
11527 Attrs.push_back(Elt: Attribute::InReg);
11528
11529 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
11530 Kinds: Attrs);
11531}
11532
11533/// TargetLowering::LowerCallTo - This is the default LowerCallTo
11534/// implementation, which just calls LowerCall.
11535/// FIXME: When all targets are
11536/// migrated to using LowerCall, this hook should be integrated into SDISel.
11537std::pair<SDValue, SDValue>
11538TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
11539 LLVMContext &Context = CLI.RetTy->getContext();
11540
11541 // Handle the incoming return values from the call.
11542 CLI.Ins.clear();
11543 SmallVector<Type *, 4> RetOrigTys;
11544 SmallVector<TypeSize, 4> Offsets;
11545 auto &DL = CLI.DAG.getDataLayout();
11546 ComputeValueTypes(DL, Ty: CLI.OrigRetTy, Types&: RetOrigTys, Offsets: &Offsets);
11547
11548 SmallVector<EVT, 4> RetVTs;
11549 if (CLI.RetTy != CLI.OrigRetTy) {
11550 assert(RetOrigTys.size() == 1 &&
11551 "Only supported for non-aggregate returns");
11552 RetVTs.push_back(Elt: getValueType(DL, Ty: CLI.RetTy));
11553 } else {
11554 for (Type *Ty : RetOrigTys)
11555 RetVTs.push_back(Elt: getValueType(DL, Ty));
11556 }
11557
11558 if (CLI.IsPostTypeLegalization) {
11559 // If we are lowering a libcall after legalization, split the return type.
11560 SmallVector<Type *, 4> OldRetOrigTys;
11561 SmallVector<EVT, 4> OldRetVTs;
11562 SmallVector<TypeSize, 4> OldOffsets;
11563 RetOrigTys.swap(RHS&: OldRetOrigTys);
11564 RetVTs.swap(RHS&: OldRetVTs);
11565 Offsets.swap(RHS&: OldOffsets);
11566
11567 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
11568 EVT RetVT = OldRetVTs[i];
11569 uint64_t Offset = OldOffsets[i];
11570 MVT RegisterVT = getRegisterType(Context, VT: RetVT);
11571 unsigned NumRegs = getNumRegisters(Context, VT: RetVT);
11572 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
11573 RetOrigTys.append(NumInputs: NumRegs, Elt: OldRetOrigTys[i]);
11574 RetVTs.append(NumInputs: NumRegs, Elt: RegisterVT);
11575 for (unsigned j = 0; j != NumRegs; ++j)
11576 Offsets.push_back(Elt: TypeSize::getFixed(ExactSize: Offset + j * RegisterVTByteSZ));
11577 }
11578 }
11579
11580 SmallVector<ISD::OutputArg, 4> Outs;
11581 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI: *this, DL);
11582
11583 bool CanLowerReturn =
11584 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
11585 CLI.IsVarArg, Outs, Context, RetTy: CLI.RetTy);
11586
11587 SDValue DemoteStackSlot;
11588 int DemoteStackIdx = -100;
11589 if (!CanLowerReturn) {
11590 // FIXME: equivalent assert?
11591 // assert(!CS.hasInAllocaArgument() &&
11592 // "sret demotion is incompatible with inalloca");
11593 uint64_t TySize = DL.getTypeAllocSize(Ty: CLI.RetTy);
11594 Align Alignment = DL.getPrefTypeAlign(Ty: CLI.RetTy);
11595 MachineFunction &MF = CLI.DAG.getMachineFunction();
11596 DemoteStackIdx =
11597 MF.getFrameInfo().CreateStackObject(Size: TySize, Alignment, isSpillSlot: false);
11598 Type *StackSlotPtrType = PointerType::get(C&: Context, AddressSpace: DL.getAllocaAddrSpace());
11599
11600 DemoteStackSlot = CLI.DAG.getFrameIndex(FI: DemoteStackIdx, VT: getFrameIndexTy(DL));
11601 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11602 Entry.IsSRet = true;
11603 Entry.Alignment = Alignment;
11604 CLI.getArgs().insert(position: CLI.getArgs().begin(), x: Entry);
11605 CLI.NumFixedArgs += 1;
11606 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11607 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(C&: Context);
11608
11609 // sret demotion isn't compatible with tail-calls, since the sret argument
11610 // points into the callers stack frame.
11611 CLI.IsTailCall = false;
11612 } else {
11613 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11614 Ty: CLI.RetTy, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11615 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
11616 ISD::ArgFlagsTy Flags;
11617 if (NeedsRegBlock) {
11618 Flags.setInConsecutiveRegs();
11619 if (I == RetVTs.size() - 1)
11620 Flags.setInConsecutiveRegsLast();
11621 }
11622 EVT VT = RetVTs[I];
11623 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11624 unsigned NumRegs =
11625 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11626 for (unsigned i = 0; i != NumRegs; ++i) {
11627 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
11628 CLI.IsReturnValueUsed, ISD::InputArg::NoArgIndex, 0);
11629 if (CLI.RetTy->isPointerTy()) {
11630 Ret.Flags.setPointer();
11631 Ret.Flags.setPointerAddrSpace(
11632 cast<PointerType>(Val: CLI.RetTy)->getAddressSpace());
11633 }
11634 if (CLI.RetSExt)
11635 Ret.Flags.setSExt();
11636 if (CLI.RetZExt)
11637 Ret.Flags.setZExt();
11638 if (CLI.IsInReg)
11639 Ret.Flags.setInReg();
11640 CLI.Ins.push_back(Elt: Ret);
11641 }
11642 }
11643 }
11644
11645 // We push in swifterror return as the last element of CLI.Ins.
11646 ArgListTy &Args = CLI.getArgs();
11647 if (supportSwiftError()) {
11648 for (const ArgListEntry &Arg : Args) {
11649 if (Arg.IsSwiftError) {
11650 ISD::ArgFlagsTy Flags;
11651 Flags.setSwiftError();
11652 ISD::InputArg Ret(Flags, getPointerTy(DL), EVT(getPointerTy(DL)),
11653 PointerType::getUnqual(C&: Context),
11654 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
11655 CLI.Ins.push_back(Elt: Ret);
11656 }
11657 }
11658 }
11659
11660 // Handle all of the outgoing arguments.
11661 CLI.Outs.clear();
11662 CLI.OutVals.clear();
11663 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11664 SmallVector<Type *, 4> OrigArgTys;
11665 ComputeValueTypes(DL, Ty: Args[i].OrigTy, Types&: OrigArgTys);
11666 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11667 Type *FinalType = Args[i].Ty;
11668 if (Args[i].IsByVal)
11669 FinalType = Args[i].IndirectType;
11670 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11671 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11672 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
11673 ++Value) {
11674 Type *OrigArgTy = OrigArgTys[Value];
11675 Type *ArgTy = OrigArgTy;
11676 if (Args[i].Ty != Args[i].OrigTy) {
11677 assert(Value == 0 && "Only supported for non-aggregate arguments");
11678 ArgTy = Args[i].Ty;
11679 }
11680
11681 EVT VT = getValueType(DL, Ty: ArgTy);
11682 SDValue Op = SDValue(Args[i].Node.getNode(),
11683 Args[i].Node.getResNo() + Value);
11684 ISD::ArgFlagsTy Flags;
11685
11686 // Certain targets (such as MIPS), may have a different ABI alignment
11687 // for a type depending on the context. Give the target a chance to
11688 // specify the alignment it wants.
11689 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11690 Flags.setOrigAlign(OriginalAlignment);
11691
11692 if (i >= CLI.NumFixedArgs)
11693 Flags.setVarArg();
11694 if (ArgTy->isPointerTy()) {
11695 Flags.setPointer();
11696 Flags.setPointerAddrSpace(cast<PointerType>(Val: ArgTy)->getAddressSpace());
11697 }
11698 if (Args[i].IsZExt)
11699 Flags.setZExt();
11700 if (Args[i].IsSExt)
11701 Flags.setSExt();
11702 if (Args[i].IsNoExt)
11703 Flags.setNoExt();
11704 if (Args[i].IsInReg) {
11705 // If we are using vectorcall calling convention, a structure that is
11706 // passed InReg - is surely an HVA
11707 if (CLI.CallConv == CallingConv::X86_VectorCall &&
11708 isa<StructType>(Val: FinalType)) {
11709 // The first value of a structure is marked
11710 if (0 == Value)
11711 Flags.setHvaStart();
11712 Flags.setHva();
11713 }
11714 // Set InReg Flag
11715 Flags.setInReg();
11716 }
11717 if (Args[i].IsSRet)
11718 Flags.setSRet();
11719 if (Args[i].IsSwiftSelf)
11720 Flags.setSwiftSelf();
11721 if (Args[i].IsSwiftAsync)
11722 Flags.setSwiftAsync();
11723 if (Args[i].IsSwiftError)
11724 Flags.setSwiftError();
11725 if (Args[i].IsCFGuardTarget)
11726 Flags.setCFGuardTarget();
11727 if (Args[i].IsByVal)
11728 Flags.setByVal();
11729 if (Args[i].IsByRef)
11730 Flags.setByRef();
11731 if (Args[i].IsPreallocated) {
11732 Flags.setPreallocated();
11733 // Set the byval flag for CCAssignFn callbacks that don't know about
11734 // preallocated. This way we can know how many bytes we should've
11735 // allocated and how many bytes a callee cleanup function will pop. If
11736 // we port preallocated to more targets, we'll have to add custom
11737 // preallocated handling in the various CC lowering callbacks.
11738 Flags.setByVal();
11739 }
11740 if (Args[i].IsInAlloca) {
11741 Flags.setInAlloca();
11742 // Set the byval flag for CCAssignFn callbacks that don't know about
11743 // inalloca. This way we can know how many bytes we should've allocated
11744 // and how many bytes a callee cleanup function will pop. If we port
11745 // inalloca to more targets, we'll have to add custom inalloca handling
11746 // in the various CC lowering callbacks.
11747 Flags.setByVal();
11748 }
11749 Align MemAlign;
11750 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11751 unsigned FrameSize = DL.getTypeAllocSize(Ty: Args[i].IndirectType);
11752 Flags.setByValSize(FrameSize);
11753
11754 // info is not there but there are cases it cannot get right.
11755 if (auto MA = Args[i].Alignment)
11756 MemAlign = *MA;
11757 else
11758 MemAlign = getByValTypeAlignment(Ty: Args[i].IndirectType, DL);
11759 } else if (auto MA = Args[i].Alignment) {
11760 MemAlign = *MA;
11761 } else {
11762 MemAlign = OriginalAlignment;
11763 }
11764 Flags.setMemAlign(MemAlign);
11765 if (Args[i].IsNest)
11766 Flags.setNest();
11767 if (NeedsRegBlock)
11768 Flags.setInConsecutiveRegs();
11769
11770 MVT PartVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11771 unsigned NumParts =
11772 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11773 SmallVector<SDValue, 4> Parts(NumParts);
11774 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
11775
11776 if (Args[i].IsSExt)
11777 ExtendKind = ISD::SIGN_EXTEND;
11778 else if (Args[i].IsZExt)
11779 ExtendKind = ISD::ZERO_EXTEND;
11780
11781 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11782 // for now.
11783 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11784 CanLowerReturn) {
11785 assert((CLI.RetTy == Args[i].Ty ||
11786 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11787 CLI.RetTy->getPointerAddressSpace() ==
11788 Args[i].Ty->getPointerAddressSpace())) &&
11789 RetVTs.size() == NumValues && "unexpected use of 'returned'");
11790 // Before passing 'returned' to the target lowering code, ensure that
11791 // either the register MVT and the actual EVT are the same size or that
11792 // the return value and argument are extended in the same way; in these
11793 // cases it's safe to pass the argument register value unchanged as the
11794 // return register value (although it's at the target's option whether
11795 // to do so)
11796 // TODO: allow code generation to take advantage of partially preserved
11797 // registers rather than clobbering the entire register when the
11798 // parameter extension method is not compatible with the return
11799 // extension method
11800 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11801 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11802 CLI.RetZExt == Args[i].IsZExt))
11803 Flags.setReturned();
11804 }
11805
11806 getCopyToParts(DAG&: CLI.DAG, DL: CLI.DL, Val: Op, Parts: &Parts[0], NumParts, PartVT, V: CLI.CB,
11807 CallConv: CLI.CallConv, ExtendKind);
11808
11809 for (unsigned j = 0; j != NumParts; ++j) {
11810 // if it isn't first piece, alignment must be 1
11811 // For scalable vectors the scalable part is currently handled
11812 // by individual targets, so we just use the known minimum size here.
11813 ISD::OutputArg MyFlags(
11814 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
11815 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11816 if (NumParts > 1 && j == 0)
11817 MyFlags.Flags.setSplit();
11818 else if (j != 0) {
11819 MyFlags.Flags.setOrigAlign(Align(1));
11820 if (j == NumParts - 1)
11821 MyFlags.Flags.setSplitEnd();
11822 }
11823
11824 CLI.Outs.push_back(Elt: MyFlags);
11825 CLI.OutVals.push_back(Elt: Parts[j]);
11826 }
11827
11828 if (NeedsRegBlock && Value == NumValues - 1)
11829 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11830 }
11831 }
11832
11833 SmallVector<SDValue, 4> InVals;
11834 CLI.Chain = LowerCall(CLI, InVals);
11835
11836 // Update CLI.InVals to use outside of this function.
11837 CLI.InVals = InVals;
11838
11839 // Verify that the target's LowerCall behaved as expected.
11840 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11841 "LowerCall didn't return a valid chain!");
11842 assert((!CLI.IsTailCall || InVals.empty()) &&
11843 "LowerCall emitted a return value for a tail call!");
11844 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11845 "LowerCall didn't emit the correct number of values!");
11846
11847 // For a tail call, the return value is merely live-out and there aren't
11848 // any nodes in the DAG representing it. Return a special value to
11849 // indicate that a tail call has been emitted and no more Instructions
11850 // should be processed in the current block.
11851 if (CLI.IsTailCall) {
11852 CLI.DAG.setRoot(CLI.Chain);
11853 return std::make_pair(x: SDValue(), y: SDValue());
11854 }
11855
11856#ifndef NDEBUG
11857 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11858 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11859 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11860 "LowerCall emitted a value with the wrong type!");
11861 }
11862#endif
11863
11864 SmallVector<SDValue, 4> ReturnValues;
11865 if (!CanLowerReturn) {
11866 // The instruction result is the result of loading from the
11867 // hidden sret parameter.
11868 MVT PtrVT = getPointerTy(DL, AS: DL.getAllocaAddrSpace());
11869
11870 unsigned NumValues = RetVTs.size();
11871 ReturnValues.resize(N: NumValues);
11872 SmallVector<SDValue, 4> Chains(NumValues);
11873
11874 // An aggregate return value cannot wrap around the address space, so
11875 // offsets to its parts don't wrap either.
11876 MachineFunction &MF = CLI.DAG.getMachineFunction();
11877 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(ObjectIdx: DemoteStackIdx);
11878 for (unsigned i = 0; i < NumValues; ++i) {
11879 SDValue Add = CLI.DAG.getMemBasePlusOffset(
11880 Base: DemoteStackSlot, Offset: CLI.DAG.getConstant(Val: Offsets[i], DL: CLI.DL, VT: PtrVT),
11881 DL: CLI.DL, Flags: SDNodeFlags::NoUnsignedWrap);
11882 SDValue L = CLI.DAG.getLoad(
11883 VT: RetVTs[i], dl: CLI.DL, Chain: CLI.Chain, Ptr: Add,
11884 PtrInfo: MachinePointerInfo::getFixedStack(MF&: CLI.DAG.getMachineFunction(),
11885 FI: DemoteStackIdx, Offset: Offsets[i]),
11886 Alignment: HiddenSRetAlign);
11887 ReturnValues[i] = L;
11888 Chains[i] = L.getValue(R: 1);
11889 }
11890
11891 CLI.Chain = CLI.DAG.getNode(Opcode: ISD::TokenFactor, DL: CLI.DL, VT: MVT::Other, Ops: Chains);
11892 } else {
11893 // Collect the legal value parts into potentially illegal values
11894 // that correspond to the original function's return values.
11895 std::optional<ISD::NodeType> AssertOp;
11896 if (CLI.RetSExt)
11897 AssertOp = ISD::AssertSext;
11898 else if (CLI.RetZExt)
11899 AssertOp = ISD::AssertZext;
11900 unsigned CurReg = 0;
11901 for (EVT VT : RetVTs) {
11902 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11903 unsigned NumRegs =
11904 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11905
11906 ReturnValues.push_back(Elt: getCopyFromParts(
11907 DAG&: CLI.DAG, DL: CLI.DL, Parts: &InVals[CurReg], NumParts: NumRegs, PartVT: RegisterVT, ValueVT: VT, V: nullptr,
11908 InChain: CLI.Chain, CC: CLI.CallConv, AssertOp));
11909 CurReg += NumRegs;
11910 }
11911
11912 // For a function returning void, there is no return value. We can't create
11913 // such a node, so we just return a null return value in that case. In
11914 // that case, nothing will actually look at the value.
11915 if (ReturnValues.empty())
11916 return std::make_pair(x: SDValue(), y&: CLI.Chain);
11917 }
11918
11919 SDValue Res = CLI.DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: CLI.DL,
11920 VTList: CLI.DAG.getVTList(VTs: RetVTs), Ops: ReturnValues);
11921 return std::make_pair(x&: Res, y&: CLI.Chain);
11922}
11923
11924/// Places new result values for the node in Results (their number
11925/// and types must exactly match those of the original return values of
11926/// the node), or leaves Results empty, which indicates that the node is not
11927/// to be custom lowered after all.
11928void TargetLowering::LowerOperationWrapper(SDNode *N,
11929 SmallVectorImpl<SDValue> &Results,
11930 SelectionDAG &DAG) const {
11931 SDValue Res = LowerOperation(Op: SDValue(N, 0), DAG);
11932
11933 if (!Res.getNode())
11934 return;
11935
11936 // If the original node has one result, take the return value from
11937 // LowerOperation as is. It might not be result number 0.
11938 if (N->getNumValues() == 1) {
11939 Results.push_back(Elt: Res);
11940 return;
11941 }
11942
11943 // If the original node has multiple results, then the return node should
11944 // have the same number of results.
11945 assert((N->getNumValues() == Res->getNumValues()) &&
11946 "Lowering returned the wrong number of results!");
11947
11948 // Places new result values base on N result number.
11949 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11950 Results.push_back(Elt: Res.getValue(R: I));
11951}
11952
11953SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
11954 llvm_unreachable("LowerOperation not implemented for this target!");
11955}
11956
11957void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
11958 Register Reg,
11959 ISD::NodeType ExtendType) {
11960 SDValue Op = getNonRegisterValue(V);
11961 assert((Op.getOpcode() != ISD::CopyFromReg ||
11962 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11963 "Copy from a reg to the same reg!");
11964 assert(!Reg.isPhysical() && "Is a physreg");
11965
11966 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11967 // If this is an InlineAsm we have to match the registers required, not the
11968 // notional registers required by the type.
11969
11970 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11971 std::nullopt); // This is not an ABI copy.
11972 SDValue Chain = DAG.getEntryNode();
11973
11974 if (ExtendType == ISD::ANY_EXTEND) {
11975 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(Val: V);
11976 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11977 ExtendType = PreferredExtendIt->second;
11978 }
11979 RFV.getCopyToRegs(Val: Op, DAG, dl: getCurSDLoc(), Chain, Glue: nullptr, V, PreferredExtendType: ExtendType);
11980 PendingExports.push_back(Elt: Chain);
11981}
11982
11983#include "llvm/CodeGen/SelectionDAGISel.h"
11984
11985/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11986/// entry block, return true. This includes arguments used by switches, since
11987/// the switch may expand into multiple basic blocks.
11988static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11989 // With FastISel active, we may be splitting blocks, so force creation
11990 // of virtual registers for all non-dead arguments.
11991 if (FastISel)
11992 return A->use_empty();
11993
11994 const BasicBlock &Entry = A->getParent()->front();
11995 for (const User *U : A->users())
11996 if (cast<Instruction>(Val: U)->getParent() != &Entry || isa<SwitchInst>(Val: U))
11997 return false; // Use not in entry block.
11998
11999 return true;
12000}
12001
12002using ArgCopyElisionMapTy =
12003 DenseMap<const Argument *,
12004 std::pair<const AllocaInst *, const StoreInst *>>;
12005
12006/// Scan the entry block of the function in FuncInfo for arguments that look
12007/// like copies into a local alloca. Record any copied arguments in
12008/// ArgCopyElisionCandidates.
12009static void
12010findArgumentCopyElisionCandidates(const DataLayout &DL,
12011 FunctionLoweringInfo *FuncInfo,
12012 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
12013 // Record the state of every static alloca used in the entry block. Argument
12014 // allocas are all used in the entry block, so we need approximately as many
12015 // entries as we have arguments.
12016 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
12017 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
12018 unsigned NumArgs = FuncInfo->Fn->arg_size();
12019 StaticAllocas.reserve(NumEntries: NumArgs * 2);
12020
12021 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
12022 if (!V)
12023 return nullptr;
12024 V = V->stripPointerCasts();
12025 const auto *AI = dyn_cast<AllocaInst>(Val: V);
12026 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(Val: AI))
12027 return nullptr;
12028 auto Iter = StaticAllocas.insert(KV: {AI, Unknown});
12029 return &Iter.first->second;
12030 };
12031
12032 // Look for stores of arguments to static allocas. Look through bitcasts and
12033 // GEPs to handle type coercions, as long as the alloca is fully initialized
12034 // by the store. Any non-store use of an alloca escapes it and any subsequent
12035 // unanalyzed store might write it.
12036 // FIXME: Handle structs initialized with multiple stores.
12037 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
12038 // Look for stores, and handle non-store uses conservatively.
12039 const auto *SI = dyn_cast<StoreInst>(Val: &I);
12040 if (!SI) {
12041 // We will look through cast uses, so ignore them completely.
12042 if (I.isCast())
12043 continue;
12044 // Ignore debug info and pseudo op intrinsics, they don't escape or store
12045 // to allocas.
12046 if (I.isDebugOrPseudoInst())
12047 continue;
12048 // This is an unknown instruction. Assume it escapes or writes to all
12049 // static alloca operands.
12050 for (const Use &U : I.operands()) {
12051 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
12052 *Info = StaticAllocaInfo::Clobbered;
12053 }
12054 continue;
12055 }
12056
12057 // If the stored value is a static alloca, mark it as escaped.
12058 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
12059 *Info = StaticAllocaInfo::Clobbered;
12060
12061 // Check if the destination is a static alloca.
12062 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
12063 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
12064 if (!Info)
12065 continue;
12066 const AllocaInst *AI = cast<AllocaInst>(Val: Dst);
12067
12068 // Skip allocas that have been initialized or clobbered.
12069 if (*Info != StaticAllocaInfo::Unknown)
12070 continue;
12071
12072 // Check if the stored value is an argument, and that this store fully
12073 // initializes the alloca.
12074 // If the argument type has padding bits we can't directly forward a pointer
12075 // as the upper bits may contain garbage.
12076 // Don't elide copies from the same argument twice.
12077 const Value *Val = SI->getValueOperand()->stripPointerCasts();
12078 const auto *Arg = dyn_cast<Argument>(Val);
12079 std::optional<TypeSize> AllocaSize = AI->getAllocationSize(DL);
12080 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
12081 Arg->getType()->isEmptyTy() || !AllocaSize ||
12082 DL.getTypeStoreSize(Ty: Arg->getType()) != *AllocaSize ||
12083 !DL.typeSizeEqualsStoreSize(Ty: Arg->getType()) ||
12084 ArgCopyElisionCandidates.count(Val: Arg)) {
12085 *Info = StaticAllocaInfo::Clobbered;
12086 continue;
12087 }
12088
12089 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
12090 << '\n');
12091
12092 // Mark this alloca and store for argument copy elision.
12093 *Info = StaticAllocaInfo::Elidable;
12094 ArgCopyElisionCandidates.insert(KV: {Arg, {AI, SI}});
12095
12096 // Stop scanning if we've seen all arguments. This will happen early in -O0
12097 // builds, which is useful, because -O0 builds have large entry blocks and
12098 // many allocas.
12099 if (ArgCopyElisionCandidates.size() == NumArgs)
12100 break;
12101 }
12102}
12103
12104/// Try to elide argument copies from memory into a local alloca. Succeeds if
12105/// ArgVal is a load from a suitable fixed stack object.
12106static void tryToElideArgumentCopy(
12107 FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
12108 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
12109 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
12110 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
12111 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
12112 // Check if this is a load from a fixed stack object.
12113 auto *LNode = dyn_cast<LoadSDNode>(Val: ArgVals[0]);
12114 if (!LNode)
12115 return;
12116 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode());
12117 if (!FINode)
12118 return;
12119
12120 // Check that the fixed stack object is the right size and alignment.
12121 // Look at the alignment that the user wrote on the alloca instead of looking
12122 // at the stack object.
12123 auto ArgCopyIter = ArgCopyElisionCandidates.find(Val: &Arg);
12124 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
12125 const AllocaInst *AI = ArgCopyIter->second.first;
12126 int FixedIndex = FINode->getIndex();
12127 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
12128 int OldIndex = AllocaIndex;
12129 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
12130 if (MFI.getObjectSize(ObjectIdx: FixedIndex) != MFI.getObjectSize(ObjectIdx: OldIndex)) {
12131 LLVM_DEBUG(
12132 dbgs() << " argument copy elision failed due to bad fixed stack "
12133 "object size\n");
12134 return;
12135 }
12136 Align RequiredAlignment = AI->getAlign();
12137 if (MFI.getObjectAlign(ObjectIdx: FixedIndex) < RequiredAlignment) {
12138 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
12139 "greater than stack argument alignment ("
12140 << DebugStr(RequiredAlignment) << " vs "
12141 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
12142 return;
12143 }
12144
12145 // Perform the elision. Delete the old stack object and replace its only use
12146 // in the variable info map. Mark the stack object as mutable and aliased.
12147 LLVM_DEBUG({
12148 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
12149 << " Replacing frame index " << OldIndex << " with " << FixedIndex
12150 << '\n';
12151 });
12152 MFI.RemoveStackObject(ObjectIdx: OldIndex);
12153 MFI.setIsImmutableObjectIndex(ObjectIdx: FixedIndex, IsImmutable: false);
12154 MFI.setIsAliasedObjectIndex(ObjectIdx: FixedIndex, IsAliased: true);
12155 AllocaIndex = FixedIndex;
12156 ArgCopyElisionFrameIndexMap.insert(KV: {OldIndex, FixedIndex});
12157 for (SDValue ArgVal : ArgVals)
12158 Chains.push_back(Elt: ArgVal.getValue(R: 1));
12159
12160 // Avoid emitting code for the store implementing the copy.
12161 const StoreInst *SI = ArgCopyIter->second.second;
12162 ElidedArgCopyInstrs.insert(Ptr: SI);
12163
12164 // Check for uses of the argument again so that we can avoid exporting ArgVal
12165 // if it is't used by anything other than the store.
12166 for (const Value *U : Arg.users()) {
12167 if (U != SI) {
12168 ArgHasUses = true;
12169 break;
12170 }
12171 }
12172}
12173
12174void SelectionDAGISel::LowerArguments(const Function &F) {
12175 SelectionDAG &DAG = SDB->DAG;
12176 SDLoc dl = SDB->getCurSDLoc();
12177 const DataLayout &DL = DAG.getDataLayout();
12178 SmallVector<ISD::InputArg, 16> Ins;
12179
12180 // In Naked functions we aren't going to save any registers.
12181 if (F.hasFnAttribute(Kind: Attribute::Naked))
12182 return;
12183
12184 if (!FuncInfo->CanLowerReturn) {
12185 // Put in an sret pointer parameter before all the other parameters.
12186 MVT ValueVT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
12187
12188 ISD::ArgFlagsTy Flags;
12189 Flags.setSRet();
12190 MVT RegisterVT = TLI->getRegisterType(Context&: *DAG.getContext(), VT: ValueVT);
12191 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
12192 ISD::InputArg::NoArgIndex, 0);
12193 Ins.push_back(Elt: RetArg);
12194 }
12195
12196 // Look for stores of arguments to static allocas. Mark such arguments with a
12197 // flag to ask the target to give us the memory location of that argument if
12198 // available.
12199 ArgCopyElisionMapTy ArgCopyElisionCandidates;
12200 findArgumentCopyElisionCandidates(DL, FuncInfo: FuncInfo.get(),
12201 ArgCopyElisionCandidates);
12202
12203 // Set up the incoming argument description vector.
12204 for (const Argument &Arg : F.args()) {
12205 unsigned ArgNo = Arg.getArgNo();
12206 SmallVector<Type *, 4> Types;
12207 ComputeValueTypes(DL: DAG.getDataLayout(), Ty: Arg.getType(), Types);
12208 bool isArgValueUsed = !Arg.use_empty();
12209 Type *FinalType = Arg.getType();
12210 if (Arg.hasAttribute(Kind: Attribute::ByVal))
12211 FinalType = Arg.getParamByValType();
12212 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
12213 Ty: FinalType, CallConv: F.getCallingConv(), isVarArg: F.isVarArg(), DL);
12214 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
12215 ++Value) {
12216 Type *ArgTy = Types[Value];
12217 EVT VT = TLI->getValueType(DL, Ty: ArgTy);
12218 ISD::ArgFlagsTy Flags;
12219
12220 if (ArgTy->isPointerTy()) {
12221 Flags.setPointer();
12222 Flags.setPointerAddrSpace(cast<PointerType>(Val: ArgTy)->getAddressSpace());
12223 }
12224 if (Arg.hasAttribute(Kind: Attribute::ZExt))
12225 Flags.setZExt();
12226 if (Arg.hasAttribute(Kind: Attribute::SExt))
12227 Flags.setSExt();
12228 if (Arg.hasAttribute(Kind: Attribute::InReg)) {
12229 // If we are using vectorcall calling convention, a structure that is
12230 // passed InReg - is surely an HVA
12231 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
12232 isa<StructType>(Val: Arg.getType())) {
12233 // The first value of a structure is marked
12234 if (0 == Value)
12235 Flags.setHvaStart();
12236 Flags.setHva();
12237 }
12238 // Set InReg Flag
12239 Flags.setInReg();
12240 }
12241 if (Arg.hasAttribute(Kind: Attribute::StructRet))
12242 Flags.setSRet();
12243 if (Arg.hasAttribute(Kind: Attribute::SwiftSelf))
12244 Flags.setSwiftSelf();
12245 if (Arg.hasAttribute(Kind: Attribute::SwiftAsync))
12246 Flags.setSwiftAsync();
12247 if (Arg.hasAttribute(Kind: Attribute::SwiftError))
12248 Flags.setSwiftError();
12249 if (Arg.hasAttribute(Kind: Attribute::ByVal))
12250 Flags.setByVal();
12251 if (Arg.hasAttribute(Kind: Attribute::ByRef))
12252 Flags.setByRef();
12253 if (Arg.hasAttribute(Kind: Attribute::InAlloca)) {
12254 Flags.setInAlloca();
12255 // Set the byval flag for CCAssignFn callbacks that don't know about
12256 // inalloca. This way we can know how many bytes we should've allocated
12257 // and how many bytes a callee cleanup function will pop. If we port
12258 // inalloca to more targets, we'll have to add custom inalloca handling
12259 // in the various CC lowering callbacks.
12260 Flags.setByVal();
12261 }
12262 if (Arg.hasAttribute(Kind: Attribute::Preallocated)) {
12263 Flags.setPreallocated();
12264 // Set the byval flag for CCAssignFn callbacks that don't know about
12265 // preallocated. This way we can know how many bytes we should've
12266 // allocated and how many bytes a callee cleanup function will pop. If
12267 // we port preallocated to more targets, we'll have to add custom
12268 // preallocated handling in the various CC lowering callbacks.
12269 Flags.setByVal();
12270 }
12271
12272 // Certain targets (such as MIPS), may have a different ABI alignment
12273 // for a type depending on the context. Give the target a chance to
12274 // specify the alignment it wants.
12275 const Align OriginalAlignment(
12276 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
12277 Flags.setOrigAlign(OriginalAlignment);
12278
12279 Align MemAlign;
12280 Type *ArgMemTy = nullptr;
12281 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
12282 Flags.isByRef()) {
12283 if (!ArgMemTy)
12284 ArgMemTy = Arg.getPointeeInMemoryValueType();
12285
12286 uint64_t MemSize = DL.getTypeAllocSize(Ty: ArgMemTy);
12287
12288 // For in-memory arguments, size and alignment should be passed from FE.
12289 // BE will guess if this info is not there but there are cases it cannot
12290 // get right.
12291 if (auto ParamAlign = Arg.getParamStackAlign())
12292 MemAlign = *ParamAlign;
12293 else if ((ParamAlign = Arg.getParamAlign()))
12294 MemAlign = *ParamAlign;
12295 else
12296 MemAlign = TLI->getByValTypeAlignment(Ty: ArgMemTy, DL);
12297 if (Flags.isByRef())
12298 Flags.setByRefSize(MemSize);
12299 else
12300 Flags.setByValSize(MemSize);
12301 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
12302 MemAlign = *ParamAlign;
12303 } else {
12304 MemAlign = OriginalAlignment;
12305 }
12306 Flags.setMemAlign(MemAlign);
12307
12308 if (Arg.hasAttribute(Kind: Attribute::Nest))
12309 Flags.setNest();
12310 if (NeedsRegBlock)
12311 Flags.setInConsecutiveRegs();
12312 if (ArgCopyElisionCandidates.count(Val: &Arg))
12313 Flags.setCopyElisionCandidate();
12314 if (Arg.hasAttribute(Kind: Attribute::Returned))
12315 Flags.setReturned();
12316
12317 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
12318 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12319 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
12320 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12321 for (unsigned i = 0; i != NumRegs; ++i) {
12322 // For scalable vectors, use the minimum size; individual targets
12323 // are responsible for handling scalable vector arguments and
12324 // return values.
12325 ISD::InputArg MyFlags(
12326 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12327 i * RegisterVT.getStoreSize().getKnownMinValue());
12328 if (NumRegs > 1 && i == 0)
12329 MyFlags.Flags.setSplit();
12330 // if it isn't first piece, alignment must be 1
12331 else if (i > 0) {
12332 MyFlags.Flags.setOrigAlign(Align(1));
12333 if (i == NumRegs - 1)
12334 MyFlags.Flags.setSplitEnd();
12335 }
12336 Ins.push_back(Elt: MyFlags);
12337 }
12338 if (NeedsRegBlock && Value == NumValues - 1)
12339 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
12340 }
12341 }
12342
12343 // Call the target to set up the argument values.
12344 SmallVector<SDValue, 8> InVals;
12345 SDValue NewRoot = TLI->LowerFormalArguments(
12346 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
12347
12348 // Verify that the target's LowerFormalArguments behaved as expected.
12349 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
12350 "LowerFormalArguments didn't return a valid chain!");
12351 assert(InVals.size() == Ins.size() &&
12352 "LowerFormalArguments didn't emit the correct number of values!");
12353 assert(all_of(InVals, [](SDValue InVal) { return InVal.getNode(); }) &&
12354 "LowerFormalArguments emitted a null value!");
12355
12356 // Update the DAG with the new chain value resulting from argument lowering.
12357 DAG.setRoot(NewRoot);
12358
12359 // Set up the argument values.
12360 unsigned i = 0;
12361 if (!FuncInfo->CanLowerReturn) {
12362 // Create a virtual register for the sret pointer, and put in a copy
12363 // from the sret argument into it.
12364 MVT VT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
12365 MVT RegVT = TLI->getRegisterType(Context&: *CurDAG->getContext(), VT);
12366 std::optional<ISD::NodeType> AssertOp;
12367 SDValue ArgValue =
12368 getCopyFromParts(DAG, DL: dl, Parts: &InVals[0], NumParts: 1, PartVT: RegVT, ValueVT: VT, V: nullptr, InChain: NewRoot,
12369 CC: F.getCallingConv(), AssertOp);
12370
12371 MachineFunction& MF = SDB->DAG.getMachineFunction();
12372 MachineRegisterInfo& RegInfo = MF.getRegInfo();
12373 Register SRetReg =
12374 RegInfo.createVirtualRegister(RegClass: TLI->getRegClassFor(VT: RegVT));
12375 FuncInfo->DemoteRegister = SRetReg;
12376 NewRoot =
12377 SDB->DAG.getCopyToReg(Chain: NewRoot, dl: SDB->getCurSDLoc(), Reg: SRetReg, N: ArgValue);
12378 DAG.setRoot(NewRoot);
12379
12380 // i indexes lowered arguments. Bump it past the hidden sret argument.
12381 ++i;
12382 }
12383
12384 SmallVector<SDValue, 4> Chains;
12385 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12386 for (const Argument &Arg : F.args()) {
12387 SmallVector<SDValue, 4> ArgValues;
12388 SmallVector<EVT, 4> ValueVTs;
12389 ComputeValueVTs(TLI: *TLI, DL: DAG.getDataLayout(), Ty: Arg.getType(), ValueVTs);
12390 unsigned NumValues = ValueVTs.size();
12391 if (NumValues == 0)
12392 continue;
12393
12394 bool ArgHasUses = !Arg.use_empty();
12395
12396 // Elide the copying store if the target loaded this argument from a
12397 // suitable fixed stack object.
12398 if (Ins[i].Flags.isCopyElisionCandidate()) {
12399 unsigned NumParts = 0;
12400 for (EVT VT : ValueVTs)
12401 NumParts += TLI->getNumRegistersForCallingConv(Context&: *CurDAG->getContext(),
12402 CC: F.getCallingConv(), VT);
12403
12404 tryToElideArgumentCopy(FuncInfo&: *FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
12405 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
12406 ArgVals: ArrayRef(&InVals[i], NumParts), ArgHasUses);
12407 }
12408
12409 // If this argument is unused then remember its value. It is used to generate
12410 // debugging information.
12411 bool isSwiftErrorArg =
12412 TLI->supportSwiftError() &&
12413 Arg.hasAttribute(Kind: Attribute::SwiftError);
12414 if (!ArgHasUses && !isSwiftErrorArg) {
12415 SDB->setUnusedArgValue(V: &Arg, NewN: InVals[i]);
12416
12417 // Also remember any frame index for use in FastISel.
12418 if (FrameIndexSDNode *FI =
12419 dyn_cast<FrameIndexSDNode>(Val: InVals[i].getNode()))
12420 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12421 }
12422
12423 for (unsigned Val = 0; Val != NumValues; ++Val) {
12424 EVT VT = ValueVTs[Val];
12425 MVT PartVT = TLI->getRegisterTypeForCallingConv(Context&: *CurDAG->getContext(),
12426 CC: F.getCallingConv(), VT);
12427 unsigned NumParts = TLI->getNumRegistersForCallingConv(
12428 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12429
12430 // Even an apparent 'unused' swifterror argument needs to be returned. So
12431 // we do generate a copy for it that can be used on return from the
12432 // function.
12433 if (ArgHasUses || isSwiftErrorArg) {
12434 std::optional<ISD::NodeType> AssertOp;
12435 if (Arg.hasAttribute(Kind: Attribute::SExt))
12436 AssertOp = ISD::AssertSext;
12437 else if (Arg.hasAttribute(Kind: Attribute::ZExt))
12438 AssertOp = ISD::AssertZext;
12439
12440 SDValue OutVal =
12441 getCopyFromParts(DAG, DL: dl, Parts: &InVals[i], NumParts, PartVT, ValueVT: VT, V: nullptr,
12442 InChain: NewRoot, CC: F.getCallingConv(), AssertOp);
12443
12444 FPClassTest NoFPClass = Arg.getNoFPClass();
12445 if (NoFPClass != fcNone) {
12446 SDValue SDNoFPClass = DAG.getTargetConstant(
12447 Val: static_cast<uint64_t>(NoFPClass), DL: dl, VT: MVT::i32);
12448 OutVal = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: OutVal.getValueType(),
12449 N1: OutVal, N2: SDNoFPClass);
12450 }
12451 ArgValues.push_back(Elt: OutVal);
12452 }
12453
12454 i += NumParts;
12455 }
12456
12457 // We don't need to do anything else for unused arguments.
12458 if (ArgValues.empty())
12459 continue;
12460
12461 // Note down frame index.
12462 if (FrameIndexSDNode *FI =
12463 dyn_cast<FrameIndexSDNode>(Val: ArgValues[0].getNode()))
12464 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12465
12466 SDValue Res = DAG.getMergeValues(Ops: ArrayRef(ArgValues.data(), NumValues),
12467 dl: SDB->getCurSDLoc());
12468
12469 SDB->setValue(V: &Arg, NewN: Res);
12470 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
12471 // We want to associate the argument with the frame index, among
12472 // involved operands, that correspond to the lowest address. The
12473 // getCopyFromParts function, called earlier, is swapping the order of
12474 // the operands to BUILD_PAIR depending on endianness. The result of
12475 // that swapping is that the least significant bits of the argument will
12476 // be in the first operand of the BUILD_PAIR node, and the most
12477 // significant bits will be in the second operand.
12478 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
12479 if (LoadSDNode *LNode =
12480 dyn_cast<LoadSDNode>(Val: Res.getOperand(i: LowAddressOp).getNode()))
12481 if (FrameIndexSDNode *FI =
12482 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
12483 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12484 }
12485
12486 // Analyses past this point are naive and don't expect an assertion.
12487 if (Res.getOpcode() == ISD::AssertZext)
12488 Res = Res.getOperand(i: 0);
12489
12490 // Update the SwiftErrorVRegDefMap.
12491 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
12492 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
12493 if (Reg.isVirtual())
12494 SwiftError->setCurrentVReg(MBB: FuncInfo->MBB, SwiftError->getFunctionArg(),
12495 Reg);
12496 }
12497
12498 // If this argument is live outside of the entry block, insert a copy from
12499 // wherever we got it to the vreg that other BB's will reference it as.
12500 if (Res.getOpcode() == ISD::CopyFromReg) {
12501 // If we can, though, try to skip creating an unnecessary vreg.
12502 // FIXME: This isn't very clean... it would be nice to make this more
12503 // general.
12504 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
12505 if (Reg.isVirtual()) {
12506 FuncInfo->ValueMap[&Arg] = Reg;
12507 continue;
12508 }
12509 }
12510 if (!isOnlyUsedInEntryBlock(A: &Arg, FastISel: TM.Options.EnableFastISel)) {
12511 FuncInfo->InitializeRegForValue(V: &Arg);
12512 SDB->CopyToExportRegsIfNeeded(V: &Arg);
12513 }
12514 }
12515
12516 if (!Chains.empty()) {
12517 Chains.push_back(Elt: NewRoot);
12518 NewRoot = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
12519 }
12520
12521 DAG.setRoot(NewRoot);
12522
12523 assert(i == InVals.size() && "Argument register count mismatch!");
12524
12525 // If any argument copy elisions occurred and we have debug info, update the
12526 // stale frame indices used in the dbg.declare variable info table.
12527 if (!ArgCopyElisionFrameIndexMap.empty()) {
12528 for (MachineFunction::VariableDbgInfo &VI :
12529 MF->getInStackSlotVariableDbgInfo()) {
12530 auto I = ArgCopyElisionFrameIndexMap.find(Val: VI.getStackSlot());
12531 if (I != ArgCopyElisionFrameIndexMap.end())
12532 VI.updateStackSlot(NewSlot: I->second);
12533 }
12534 }
12535
12536 // Finally, if the target has anything special to do, allow it to do so.
12537 emitFunctionEntryCode();
12538}
12539
12540/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
12541/// ensure constants are generated when needed. Remember the virtual registers
12542/// that need to be added to the Machine PHI nodes as input. We cannot just
12543/// directly add them, because expansion might result in multiple MBB's for one
12544/// BB. As such, the start of the BB might correspond to a different MBB than
12545/// the end.
12546void
12547SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
12548 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12549
12550 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12551
12552 // Check PHI nodes in successors that expect a value to be available from this
12553 // block.
12554 for (const BasicBlock *SuccBB : successors(I: LLVMBB->getTerminator())) {
12555 if (!isa<PHINode>(Val: SuccBB->begin())) continue;
12556 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(BB: SuccBB);
12557
12558 // If this terminator has multiple identical successors (common for
12559 // switches), only handle each succ once.
12560 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
12561 continue;
12562
12563 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
12564
12565 // At this point we know that there is a 1-1 correspondence between LLVM PHI
12566 // nodes and Machine PHI nodes, but the incoming operands have not been
12567 // emitted yet.
12568 for (const PHINode &PN : SuccBB->phis()) {
12569 // Ignore dead phi's.
12570 if (PN.use_empty())
12571 continue;
12572
12573 // Skip empty types
12574 if (PN.getType()->isEmptyTy())
12575 continue;
12576
12577 Register Reg;
12578 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
12579
12580 if (const auto *C = dyn_cast<Constant>(Val: PHIOp)) {
12581 Register &RegOut = ConstantsOut[C];
12582 if (!RegOut) {
12583 RegOut = FuncInfo.CreateRegs(V: PHIOp);
12584 // We need to zero/sign extend ConstantInt phi operands to match
12585 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12586 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12587 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
12588 ExtendType = TLI.signExtendConstant(C: CI) ? ISD::SIGN_EXTEND
12589 : ISD::ZERO_EXTEND;
12590 CopyValueToVirtualRegister(V: C, Reg: RegOut, ExtendType);
12591 }
12592 Reg = RegOut;
12593 } else {
12594 auto I = FuncInfo.ValueMap.find(Val: PHIOp);
12595 if (I != FuncInfo.ValueMap.end())
12596 Reg = I->second;
12597 else {
12598 assert(isa<AllocaInst>(PHIOp) &&
12599 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12600 "Didn't codegen value into a register!??");
12601 Reg = FuncInfo.CreateRegs(V: PHIOp);
12602 CopyValueToVirtualRegister(V: PHIOp, Reg);
12603 }
12604 }
12605
12606 // Remember that this register needs to added to the machine PHI node as
12607 // the input for this MBB.
12608 SmallVector<EVT, 4> ValueVTs;
12609 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: PN.getType(), ValueVTs);
12610 for (EVT VT : ValueVTs) {
12611 const unsigned NumRegisters = TLI.getNumRegisters(Context&: *DAG.getContext(), VT);
12612 for (unsigned i = 0; i != NumRegisters; ++i)
12613 FuncInfo.PHINodesToUpdate.emplace_back(args: &*MBBI++, args: Reg + i);
12614 Reg += NumRegisters;
12615 }
12616 }
12617 }
12618
12619 ConstantsOut.clear();
12620}
12621
12622MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12623 MachineFunction::iterator I(MBB);
12624 if (++I == FuncInfo.MF->end())
12625 return nullptr;
12626 return &*I;
12627}
12628
12629/// During lowering new call nodes can be created (such as memset, etc.).
12630/// Those will become new roots of the current DAG, but complications arise
12631/// when they are tail calls. In such cases, the call lowering will update
12632/// the root, but the builder still needs to know that a tail call has been
12633/// lowered in order to avoid generating an additional return.
12634void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12635 // If the node is null, we do have a tail call.
12636 if (MaybeTC.getNode() != nullptr)
12637 DAG.setRoot(MaybeTC);
12638 else
12639 HasTailCall = true;
12640}
12641
12642void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12643 MachineBasicBlock *SwitchMBB,
12644 MachineBasicBlock *DefaultMBB) {
12645 MachineFunction *CurMF = FuncInfo.MF;
12646 MachineBasicBlock *NextMBB = nullptr;
12647 MachineFunction::iterator BBI(W.MBB);
12648 if (++BBI != FuncInfo.MF->end())
12649 NextMBB = &*BBI;
12650
12651 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12652
12653 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12654
12655 if (Size == 2 && W.MBB == SwitchMBB) {
12656 // If any two of the cases has the same destination, and if one value
12657 // is the same as the other, but has one bit unset that the other has set,
12658 // use bit manipulation to do two compares at once. For example:
12659 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12660 // TODO: This could be extended to merge any 2 cases in switches with 3
12661 // cases.
12662 // TODO: Handle cases where W.CaseBB != SwitchBB.
12663 CaseCluster &Small = *W.FirstCluster;
12664 CaseCluster &Big = *W.LastCluster;
12665
12666 if (Small.Low == Small.High && Big.Low == Big.High &&
12667 Small.MBB == Big.MBB) {
12668 const APInt &SmallValue = Small.Low->getValue();
12669 const APInt &BigValue = Big.Low->getValue();
12670
12671 // Check that there is only one bit different.
12672 APInt CommonBit = BigValue ^ SmallValue;
12673 if (CommonBit.isPowerOf2()) {
12674 SDValue CondLHS = getValue(V: Cond);
12675 EVT VT = CondLHS.getValueType();
12676 SDLoc DL = getCurSDLoc();
12677
12678 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: CondLHS,
12679 N2: DAG.getConstant(Val: CommonBit, DL, VT));
12680 SDValue Cond = DAG.getSetCC(
12681 DL, VT: MVT::i1, LHS: Or, RHS: DAG.getConstant(Val: BigValue | SmallValue, DL, VT),
12682 Cond: ISD::SETEQ);
12683
12684 // Update successor info.
12685 // Both Small and Big will jump to Small.BB, so we sum up the
12686 // probabilities.
12687 addSuccessorWithProb(Src: SwitchMBB, Dst: Small.MBB, Prob: Small.Prob + Big.Prob);
12688 if (BPI)
12689 addSuccessorWithProb(
12690 Src: SwitchMBB, Dst: DefaultMBB,
12691 // The default destination is the first successor in IR.
12692 Prob: BPI->getEdgeProbability(Src: SwitchMBB->getBasicBlock(), IndexInSuccessors: (unsigned)0));
12693 else
12694 addSuccessorWithProb(Src: SwitchMBB, Dst: DefaultMBB);
12695
12696 // Insert the true branch.
12697 SDValue BrCond =
12698 DAG.getNode(Opcode: ISD::BRCOND, DL, VT: MVT::Other, N1: getControlRoot(), N2: Cond,
12699 N3: DAG.getBasicBlock(MBB: Small.MBB));
12700 // Insert the false branch.
12701 BrCond = DAG.getNode(Opcode: ISD::BR, DL, VT: MVT::Other, N1: BrCond,
12702 N2: DAG.getBasicBlock(MBB: DefaultMBB));
12703
12704 DAG.setRoot(BrCond);
12705 return;
12706 }
12707 }
12708 }
12709
12710 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12711 // Here, we order cases by probability so the most likely case will be
12712 // checked first. However, two clusters can have the same probability in
12713 // which case their relative ordering is non-deterministic. So we use Low
12714 // as a tie-breaker as clusters are guaranteed to never overlap.
12715 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
12716 Comp: [](const CaseCluster &a, const CaseCluster &b) {
12717 return a.Prob != b.Prob ?
12718 a.Prob > b.Prob :
12719 a.Low->getValue().slt(RHS: b.Low->getValue());
12720 });
12721
12722 // Rearrange the case blocks so that the last one falls through if possible
12723 // without changing the order of probabilities.
12724 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12725 --I;
12726 if (I->Prob > W.LastCluster->Prob)
12727 break;
12728 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12729 std::swap(a&: *I, b&: *W.LastCluster);
12730 break;
12731 }
12732 }
12733 }
12734
12735 // Compute total probability.
12736 BranchProbability DefaultProb = W.DefaultProb;
12737 BranchProbability UnhandledProbs = DefaultProb;
12738 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12739 UnhandledProbs += I->Prob;
12740
12741 MachineBasicBlock *CurMBB = W.MBB;
12742 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12743 bool FallthroughUnreachable = false;
12744 MachineBasicBlock *Fallthrough;
12745 if (I == W.LastCluster) {
12746 // For the last cluster, fall through to the default destination.
12747 Fallthrough = DefaultMBB;
12748 FallthroughUnreachable = isa<UnreachableInst>(
12749 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
12750 } else {
12751 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
12752 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
12753 // Put Cond in a virtual register to make it available from the new blocks.
12754 ExportFromCurrentBlock(V: Cond);
12755 }
12756 UnhandledProbs -= I->Prob;
12757
12758 switch (I->Kind) {
12759 case CC_JumpTable: {
12760 // FIXME: Optimize away range check based on pivot comparisons.
12761 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12762 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12763
12764 // The jump block hasn't been inserted yet; insert it here.
12765 MachineBasicBlock *JumpMBB = JT->MBB;
12766 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
12767
12768 auto JumpProb = I->Prob;
12769 auto FallthroughProb = UnhandledProbs;
12770
12771 // If the default statement is a target of the jump table, we evenly
12772 // distribute the default probability to successors of CurMBB. Also
12773 // update the probability on the edge from JumpMBB to Fallthrough.
12774 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12775 SE = JumpMBB->succ_end();
12776 SI != SE; ++SI) {
12777 if (*SI == DefaultMBB) {
12778 JumpProb += DefaultProb / 2;
12779 FallthroughProb -= DefaultProb / 2;
12780 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
12781 JumpMBB->normalizeSuccProbs();
12782 break;
12783 }
12784 }
12785
12786 // If the default clause is unreachable, propagate that knowledge into
12787 // JTH->FallthroughUnreachable which will use it to suppress the range
12788 // check.
12789 //
12790 // However, don't do this if we're doing branch target enforcement,
12791 // because a table branch _without_ a range check can be a tempting JOP
12792 // gadget - out-of-bounds inputs that are impossible in correct
12793 // execution become possible again if an attacker can influence the
12794 // control flow. So if an attacker doesn't already have a BTI bypass
12795 // available, we don't want them to be able to get one out of this
12796 // table branch.
12797 if (FallthroughUnreachable) {
12798 Function &CurFunc = CurMF->getFunction();
12799 if (!CurFunc.hasFnAttribute(Kind: "branch-target-enforcement"))
12800 JTH->FallthroughUnreachable = true;
12801 }
12802
12803 if (!JTH->FallthroughUnreachable)
12804 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
12805 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
12806 CurMBB->normalizeSuccProbs();
12807
12808 // The jump table header will be inserted in our current block, do the
12809 // range check, and fall through to our fallthrough block.
12810 JTH->HeaderBB = CurMBB;
12811 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12812
12813 // If we're in the right place, emit the jump table header right now.
12814 if (CurMBB == SwitchMBB) {
12815 visitJumpTableHeader(JT&: *JT, JTH&: *JTH, SwitchBB: SwitchMBB);
12816 JTH->Emitted = true;
12817 }
12818 break;
12819 }
12820 case CC_BitTests: {
12821 // FIXME: Optimize away range check based on pivot comparisons.
12822 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12823
12824 // The bit test blocks haven't been inserted yet; insert them here.
12825 for (BitTestCase &BTC : BTB->Cases)
12826 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
12827
12828 // Fill in fields of the BitTestBlock.
12829 BTB->Parent = CurMBB;
12830 BTB->Default = Fallthrough;
12831
12832 BTB->DefaultProb = UnhandledProbs;
12833 // If the cases in bit test don't form a contiguous range, we evenly
12834 // distribute the probability on the edge to Fallthrough to two
12835 // successors of CurMBB.
12836 if (!BTB->ContiguousRange) {
12837 BTB->Prob += DefaultProb / 2;
12838 BTB->DefaultProb -= DefaultProb / 2;
12839 }
12840
12841 if (FallthroughUnreachable)
12842 BTB->FallthroughUnreachable = true;
12843
12844 // If we're in the right place, emit the bit test header right now.
12845 if (CurMBB == SwitchMBB) {
12846 visitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
12847 BTB->Emitted = true;
12848 }
12849 break;
12850 }
12851 case CC_Range: {
12852 const Value *RHS, *LHS, *MHS;
12853 ISD::CondCode CC;
12854 if (I->Low == I->High) {
12855 // Check Cond == I->Low.
12856 CC = ISD::SETEQ;
12857 LHS = Cond;
12858 RHS=I->Low;
12859 MHS = nullptr;
12860 } else {
12861 // Check I->Low <= Cond <= I->High.
12862 CC = ISD::SETLE;
12863 LHS = I->Low;
12864 MHS = Cond;
12865 RHS = I->High;
12866 }
12867
12868 // If Fallthrough is unreachable, fold away the comparison.
12869 if (FallthroughUnreachable)
12870 CC = ISD::SETTRUE;
12871
12872 // The false probability is the sum of all unhandled cases.
12873 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12874 getCurSDLoc(), I->Prob, UnhandledProbs);
12875
12876 if (CurMBB == SwitchMBB)
12877 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12878 else
12879 SL->SwitchCases.push_back(x: CB);
12880
12881 break;
12882 }
12883 }
12884 CurMBB = Fallthrough;
12885 }
12886}
12887
12888void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12889 const SwitchWorkListItem &W,
12890 Value *Cond,
12891 MachineBasicBlock *SwitchMBB) {
12892 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12893 "Clusters not sorted?");
12894 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12895
12896 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12897 SL->computeSplitWorkItemInfo(W);
12898
12899 // Use the first element on the right as pivot since we will make less-than
12900 // comparisons against it.
12901 CaseClusterIt PivotCluster = FirstRight;
12902 assert(PivotCluster > W.FirstCluster);
12903 assert(PivotCluster <= W.LastCluster);
12904
12905 CaseClusterIt FirstLeft = W.FirstCluster;
12906 CaseClusterIt LastRight = W.LastCluster;
12907
12908 const ConstantInt *Pivot = PivotCluster->Low;
12909
12910 // New blocks will be inserted immediately after the current one.
12911 MachineFunction::iterator BBI(W.MBB);
12912 ++BBI;
12913
12914 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12915 // we can branch to its destination directly if it's squeezed exactly in
12916 // between the known lower bound and Pivot - 1.
12917 MachineBasicBlock *LeftMBB;
12918 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12919 FirstLeft->Low == W.GE &&
12920 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12921 LeftMBB = FirstLeft->MBB;
12922 } else {
12923 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12924 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
12925 WorkList.push_back(
12926 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
12927 // Put Cond in a virtual register to make it available from the new blocks.
12928 ExportFromCurrentBlock(V: Cond);
12929 }
12930
12931 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12932 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12933 // directly if RHS.High equals the current upper bound.
12934 MachineBasicBlock *RightMBB;
12935 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12936 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12937 RightMBB = FirstRight->MBB;
12938 } else {
12939 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12940 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
12941 WorkList.push_back(
12942 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
12943 // Put Cond in a virtual register to make it available from the new blocks.
12944 ExportFromCurrentBlock(V: Cond);
12945 }
12946
12947 // Create the CaseBlock record that will be used to lower the branch.
12948 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12949 getCurSDLoc(), LeftProb, RightProb);
12950
12951 if (W.MBB == SwitchMBB)
12952 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12953 else
12954 SL->SwitchCases.push_back(x: CB);
12955}
12956
12957// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12958// from the swith statement.
12959static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
12960 BranchProbability PeeledCaseProb) {
12961 if (PeeledCaseProb == BranchProbability::getOne())
12962 return BranchProbability::getZero();
12963 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12964
12965 uint32_t Numerator = CaseProb.getNumerator();
12966 uint32_t Denominator = SwitchProb.scale(Num: CaseProb.getDenominator());
12967 return BranchProbability(Numerator, std::max(a: Numerator, b: Denominator));
12968}
12969
12970// Try to peel the top probability case if it exceeds the threshold.
12971// Return current MachineBasicBlock for the switch statement if the peeling
12972// does not occur.
12973// If the peeling is performed, return the newly created MachineBasicBlock
12974// for the peeled switch statement. Also update Clusters to remove the peeled
12975// case. PeeledCaseProb is the BranchProbability for the peeled case.
12976MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12977 const SwitchInst &SI, CaseClusterVector &Clusters,
12978 BranchProbability &PeeledCaseProb) {
12979 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12980 // Don't perform if there is only one cluster or optimizing for size.
12981 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12982 TM.getOptLevel() == CodeGenOptLevel::None ||
12983 SwitchMBB->getParent()->getFunction().hasMinSize())
12984 return SwitchMBB;
12985
12986 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
12987 unsigned PeeledCaseIndex = 0;
12988 bool SwitchPeeled = false;
12989 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12990 CaseCluster &CC = Clusters[Index];
12991 if (CC.Prob < TopCaseProb)
12992 continue;
12993 TopCaseProb = CC.Prob;
12994 PeeledCaseIndex = Index;
12995 SwitchPeeled = true;
12996 }
12997 if (!SwitchPeeled)
12998 return SwitchMBB;
12999
13000 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
13001 << TopCaseProb << "\n");
13002
13003 // Record the MBB for the peeled switch statement.
13004 MachineFunction::iterator BBI(SwitchMBB);
13005 ++BBI;
13006 MachineBasicBlock *PeeledSwitchMBB =
13007 FuncInfo.MF->CreateMachineBasicBlock(BB: SwitchMBB->getBasicBlock());
13008 FuncInfo.MF->insert(MBBI: BBI, MBB: PeeledSwitchMBB);
13009
13010 ExportFromCurrentBlock(V: SI.getCondition());
13011 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
13012 SwitchWorkListItem W = {.MBB: SwitchMBB, .FirstCluster: PeeledCaseIt, .LastCluster: PeeledCaseIt,
13013 .GE: nullptr, .LT: nullptr, .DefaultProb: TopCaseProb.getCompl()};
13014 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB: PeeledSwitchMBB);
13015
13016 Clusters.erase(position: PeeledCaseIt);
13017 for (CaseCluster &CC : Clusters) {
13018 LLVM_DEBUG(
13019 dbgs() << "Scale the probablity for one cluster, before scaling: "
13020 << CC.Prob << "\n");
13021 CC.Prob = scaleCaseProbality(CaseProb: CC.Prob, PeeledCaseProb: TopCaseProb);
13022 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
13023 }
13024 PeeledCaseProb = TopCaseProb;
13025 return PeeledSwitchMBB;
13026}
13027
13028void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
13029 // Extract cases from the switch.
13030 BranchProbabilityInfo *BPI = FuncInfo.BPI;
13031 CaseClusterVector Clusters;
13032 Clusters.reserve(n: SI.getNumCases());
13033 for (auto I : SI.cases()) {
13034 MachineBasicBlock *Succ = FuncInfo.getMBB(BB: I.getCaseSuccessor());
13035 const ConstantInt *CaseVal = I.getCaseValue();
13036 BranchProbability Prob =
13037 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
13038 : BranchProbability(1, SI.getNumCases() + 1);
13039 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
13040 }
13041
13042 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(BB: SI.getDefaultDest());
13043
13044 // Cluster adjacent cases with the same destination. We do this at all
13045 // optimization levels because it's cheap to do and will make codegen faster
13046 // if there are many clusters.
13047 sortAndRangeify(Clusters);
13048
13049 // The branch probablity of the peeled case.
13050 BranchProbability PeeledCaseProb = BranchProbability::getZero();
13051 MachineBasicBlock *PeeledSwitchMBB =
13052 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
13053
13054 // If there is only the default destination, jump there directly.
13055 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
13056 if (Clusters.empty()) {
13057 assert(PeeledSwitchMBB == SwitchMBB);
13058 SwitchMBB->addSuccessor(Succ: DefaultMBB);
13059 if (DefaultMBB != NextBlock(MBB: SwitchMBB)) {
13060 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other,
13061 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: DefaultMBB)));
13062 }
13063 return;
13064 }
13065
13066 SL->findJumpTables(Clusters, SI: &SI, SL: getCurSDLoc(), DefaultMBB, PSI: DAG.getPSI(),
13067 BFI: DAG.getBFI());
13068 SL->findBitTestClusters(Clusters, SI: &SI);
13069
13070 LLVM_DEBUG({
13071 dbgs() << "Case clusters: ";
13072 for (const CaseCluster &C : Clusters) {
13073 if (C.Kind == CC_JumpTable)
13074 dbgs() << "JT:";
13075 if (C.Kind == CC_BitTests)
13076 dbgs() << "BT:";
13077
13078 C.Low->getValue().print(dbgs(), true);
13079 if (C.Low != C.High) {
13080 dbgs() << '-';
13081 C.High->getValue().print(dbgs(), true);
13082 }
13083 dbgs() << ' ';
13084 }
13085 dbgs() << '\n';
13086 });
13087
13088 assert(!Clusters.empty());
13089 SwitchWorkList WorkList;
13090 CaseClusterIt First = Clusters.begin();
13091 CaseClusterIt Last = Clusters.end() - 1;
13092 auto DefaultProb = getEdgeProbability(Src: PeeledSwitchMBB, Dst: DefaultMBB);
13093 // Scale the branchprobability for DefaultMBB if the peel occurs and
13094 // DefaultMBB is not replaced.
13095 if (PeeledCaseProb != BranchProbability::getZero() &&
13096 DefaultMBB == FuncInfo.getMBB(BB: SI.getDefaultDest()))
13097 DefaultProb = scaleCaseProbality(CaseProb: DefaultProb, PeeledCaseProb);
13098 WorkList.push_back(
13099 Elt: {.MBB: PeeledSwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
13100
13101 while (!WorkList.empty()) {
13102 SwitchWorkListItem W = WorkList.pop_back_val();
13103 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
13104
13105 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
13106 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
13107 // For optimized builds, lower large range as a balanced binary tree.
13108 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB);
13109 continue;
13110 }
13111
13112 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB);
13113 }
13114}
13115
13116void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
13117 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13118 auto DL = getCurSDLoc();
13119 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13120 setValue(V: &I, NewN: DAG.getStepVector(DL, ResVT: ResultVT));
13121}
13122
13123void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
13124 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13125 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13126
13127 SDLoc DL = getCurSDLoc();
13128 SDValue V = getValue(V: I.getOperand(i_nocapture: 0));
13129 assert(VT == V.getValueType() && "Malformed vector.reverse!");
13130
13131 if (VT.isScalableVector()) {
13132 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V));
13133 return;
13134 }
13135
13136 // Use VECTOR_SHUFFLE for the fixed-length vector
13137 // to maintain existing behavior.
13138 SmallVector<int, 8> Mask;
13139 unsigned NumElts = VT.getVectorMinNumElements();
13140 for (unsigned i = 0; i != NumElts; ++i)
13141 Mask.push_back(Elt: NumElts - 1 - i);
13142
13143 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V, N2: DAG.getUNDEF(VT), Mask));
13144}
13145
13146void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
13147 unsigned Factor) {
13148 auto DL = getCurSDLoc();
13149 SDValue InVec = getValue(V: I.getOperand(i_nocapture: 0));
13150
13151 SmallVector<EVT, 4> ValueVTs;
13152 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
13153 ValueVTs);
13154
13155 EVT OutVT = ValueVTs[0];
13156 unsigned OutNumElts = OutVT.getVectorMinNumElements();
13157
13158 SmallVector<SDValue, 4> SubVecs(Factor);
13159 for (unsigned i = 0; i != Factor; ++i) {
13160 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
13161 SubVecs[i] = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OutVT, N1: InVec,
13162 N2: DAG.getVectorIdxConstant(Val: OutNumElts * i, DL));
13163 }
13164
13165 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
13166 // from existing legalisation and combines.
13167 if (OutVT.isFixedLengthVector() && Factor == 2) {
13168 SDValue Even = DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: SubVecs[0], N2: SubVecs[1],
13169 Mask: createStrideMask(Start: 0, Stride: 2, VF: OutNumElts));
13170 SDValue Odd = DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: SubVecs[0], N2: SubVecs[1],
13171 Mask: createStrideMask(Start: 1, Stride: 2, VF: OutNumElts));
13172 SDValue Res = DAG.getMergeValues(Ops: {Even, Odd}, dl: getCurSDLoc());
13173 setValue(V: &I, NewN: Res);
13174 return;
13175 }
13176
13177 SDValue Res = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
13178 VTList: DAG.getVTList(VTs: ValueVTs), Ops: SubVecs);
13179 setValue(V: &I, NewN: Res);
13180}
13181
13182void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
13183 unsigned Factor) {
13184 auto DL = getCurSDLoc();
13185 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13186 EVT InVT = getValue(V: I.getOperand(i_nocapture: 0)).getValueType();
13187 EVT OutVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13188
13189 SmallVector<SDValue, 8> InVecs(Factor);
13190 for (unsigned i = 0; i < Factor; ++i) {
13191 InVecs[i] = getValue(V: I.getOperand(i_nocapture: i));
13192 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
13193 "Expected VTs to be the same");
13194 }
13195
13196 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
13197 // from existing legalisation and combines.
13198 if (OutVT.isFixedLengthVector() && Factor == 2) {
13199 unsigned NumElts = InVT.getVectorMinNumElements();
13200 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: InVecs);
13201 setValue(V: &I, NewN: DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: V, N2: DAG.getUNDEF(VT: OutVT),
13202 Mask: createInterleaveMask(VF: NumElts, NumVecs: 2)));
13203 return;
13204 }
13205
13206 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
13207 SDValue Res =
13208 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, VTList: DAG.getVTList(VTs: ValueVTs), Ops: InVecs);
13209
13210 SmallVector<SDValue, 8> Results(Factor);
13211 for (unsigned i = 0; i < Factor; ++i)
13212 Results[i] = Res.getValue(R: i);
13213
13214 Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: Results);
13215 setValue(V: &I, NewN: Res);
13216}
13217
13218void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
13219 SmallVector<EVT, 4> ValueVTs;
13220 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
13221 ValueVTs);
13222 unsigned NumValues = ValueVTs.size();
13223 if (NumValues == 0) return;
13224
13225 SmallVector<SDValue, 4> Values(NumValues);
13226 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
13227
13228 for (unsigned i = 0; i != NumValues; ++i)
13229 Values[i] = DAG.getNode(Opcode: ISD::FREEZE, DL: getCurSDLoc(), VT: ValueVTs[i],
13230 Operand: SDValue(Op.getNode(), Op.getResNo() + i));
13231
13232 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
13233 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
13234}
13235
13236void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
13237 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13238 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13239
13240 SDLoc DL = getCurSDLoc();
13241 SDValue V1 = getValue(V: I.getOperand(i_nocapture: 0));
13242 SDValue V2 = getValue(V: I.getOperand(i_nocapture: 1));
13243 const bool IsLeft = I.getIntrinsicID() == Intrinsic::vector_splice_left;
13244
13245 // VECTOR_SHUFFLE doesn't support a scalable or non-constant mask.
13246 if (VT.isScalableVector() || !isa<ConstantInt>(Val: I.getOperand(i_nocapture: 2))) {
13247 SDValue Offset = DAG.getZExtOrTrunc(
13248 Op: getValue(V: I.getOperand(i_nocapture: 2)), DL, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
13249 setValue(V: &I, NewN: DAG.getNode(Opcode: IsLeft ? ISD::VECTOR_SPLICE_LEFT
13250 : ISD::VECTOR_SPLICE_RIGHT,
13251 DL, VT, N1: V1, N2: V2, N3: Offset));
13252 return;
13253 }
13254 uint64_t Imm = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->getZExtValue();
13255
13256 unsigned NumElts = VT.getVectorNumElements();
13257
13258 uint64_t Idx = IsLeft ? Imm : NumElts - Imm;
13259
13260 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
13261 SmallVector<int, 8> Mask;
13262 for (unsigned i = 0; i < NumElts; ++i)
13263 Mask.push_back(Elt: Idx + i);
13264 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask));
13265}
13266
13267// Consider the following MIR after SelectionDAG, which produces output in
13268// phyregs in the first case or virtregs in the second case.
13269//
13270// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
13271// %5:gr32 = COPY $ebx
13272// %6:gr32 = COPY $edx
13273// %1:gr32 = COPY %6:gr32
13274// %0:gr32 = COPY %5:gr32
13275//
13276// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
13277// %1:gr32 = COPY %6:gr32
13278// %0:gr32 = COPY %5:gr32
13279//
13280// Given %0, we'd like to return $ebx in the first case and %5 in the second.
13281// Given %1, we'd like to return $edx in the first case and %6 in the second.
13282//
13283// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
13284// to a single virtreg (such as %0). The remaining outputs monotonically
13285// increase in virtreg number from there. If a callbr has no outputs, then it
13286// should not have a corresponding callbr landingpad; in fact, the callbr
13287// landingpad would not even be able to refer to such a callbr.
13288static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
13289 MachineInstr *MI = MRI.def_begin(RegNo: Reg)->getParent();
13290 // There is definitely at least one copy.
13291 assert(MI->getOpcode() == TargetOpcode::COPY &&
13292 "start of copy chain MUST be COPY");
13293 Reg = MI->getOperand(i: 1).getReg();
13294
13295 // If the copied register in the first copy must be virtual.
13296 assert(Reg.isVirtual() && "expected COPY of virtual register");
13297 MI = MRI.def_begin(RegNo: Reg)->getParent();
13298
13299 // There may be an optional second copy.
13300 if (MI->getOpcode() == TargetOpcode::COPY) {
13301 assert(Reg.isVirtual() && "expected COPY of virtual register");
13302 Reg = MI->getOperand(i: 1).getReg();
13303 assert(Reg.isPhysical() && "expected COPY of physical register");
13304 } else {
13305 // The start of the chain must be an INLINEASM_BR.
13306 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13307 "end of copy chain MUST be INLINEASM_BR");
13308 }
13309
13310 return Reg;
13311}
13312
13313// We must do this walk rather than the simpler
13314// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
13315// otherwise we will end up with copies of virtregs only valid along direct
13316// edges.
13317void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
13318 SmallVector<EVT, 8> ResultVTs;
13319 SmallVector<SDValue, 8> ResultValues;
13320 const auto *CBR =
13321 cast<CallBrInst>(Val: I.getParent()->getUniquePredecessor()->getTerminator());
13322
13323 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13324 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
13325 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
13326
13327 Register InitialDef = FuncInfo.ValueMap[CBR];
13328 SDValue Chain = DAG.getRoot();
13329
13330 // Re-parse the asm constraints string.
13331 TargetLowering::AsmOperandInfoVector TargetConstraints =
13332 TLI.ParseConstraints(DL: DAG.getDataLayout(), TRI, Call: *CBR);
13333 for (auto &T : TargetConstraints) {
13334 SDISelAsmOperandInfo OpInfo(T);
13335 if (OpInfo.Type != InlineAsm::isOutput)
13336 continue;
13337
13338 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
13339 // individual constraint.
13340 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
13341
13342 switch (OpInfo.ConstraintType) {
13343 case TargetLowering::C_Register:
13344 case TargetLowering::C_RegisterClass: {
13345 // Fill in OpInfo.AssignedRegs.Regs.
13346 getRegistersForValue(DAG, DL: getCurSDLoc(), OpInfo, RefOpInfo&: OpInfo);
13347
13348 // getRegistersForValue may produce 1 to many registers based on whether
13349 // the OpInfo.ConstraintVT is legal on the target or not.
13350 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
13351 Register OriginalDef = FollowCopyChain(MRI, Reg: InitialDef++);
13352 if (OriginalDef.isPhysical())
13353 FuncInfo.MBB->addLiveIn(PhysReg: OriginalDef);
13354 // Update the assigned registers to use the original defs.
13355 Reg = OriginalDef;
13356 }
13357
13358 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
13359 DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V: CBR);
13360 ResultValues.push_back(Elt: V);
13361 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
13362 break;
13363 }
13364 case TargetLowering::C_Other: {
13365 SDValue Flag;
13366 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Glue&: Flag, DL: getCurSDLoc(),
13367 OpInfo, DAG);
13368 ++InitialDef;
13369 ResultValues.push_back(Elt: V);
13370 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
13371 break;
13372 }
13373 default:
13374 break;
13375 }
13376 }
13377 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
13378 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
13379 setValue(V: &I, NewN: V);
13380}
13381