1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/Analysis/AliasAnalysis.h"
24#include "llvm/Analysis/BranchProbabilityInfo.h"
25#include "llvm/Analysis/ConstantFolding.h"
26#include "llvm/Analysis/Loads.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/TargetTransformInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/Analysis/VectorUtils.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
34#include "llvm/CodeGen/CodeGenCommonISel.h"
35#include "llvm/CodeGen/FunctionLoweringInfo.h"
36#include "llvm/CodeGen/GCMetadata.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/MachineBasicBlock.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineInstrBuilder.h"
42#include "llvm/CodeGen/MachineInstrBundleIterator.h"
43#include "llvm/CodeGen/MachineMemOperand.h"
44#include "llvm/CodeGen/MachineModuleInfo.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/MachineRegisterInfo.h"
47#include "llvm/CodeGen/SelectionDAG.h"
48#include "llvm/CodeGen/SelectionDAGNodes.h"
49#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
50#include "llvm/CodeGen/StackMaps.h"
51#include "llvm/CodeGen/SwiftErrorValueTracking.h"
52#include "llvm/CodeGen/TargetFrameLowering.h"
53#include "llvm/CodeGen/TargetInstrInfo.h"
54#include "llvm/CodeGen/TargetOpcodes.h"
55#include "llvm/CodeGen/TargetRegisterInfo.h"
56#include "llvm/CodeGen/TargetSubtargetInfo.h"
57#include "llvm/CodeGen/WinEHFuncInfo.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
64#include "llvm/IR/ConstantRange.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
68#include "llvm/IR/DebugInfoMetadata.h"
69#include "llvm/IR/DerivedTypes.h"
70#include "llvm/IR/DiagnosticInfo.h"
71#include "llvm/IR/EHPersonalities.h"
72#include "llvm/IR/Function.h"
73#include "llvm/IR/GetElementPtrTypeIterator.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
76#include "llvm/IR/Instructions.h"
77#include "llvm/IR/IntrinsicInst.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsBPF.h"
82#include "llvm/IR/IntrinsicsWebAssembly.h"
83#include "llvm/IR/LLVMContext.h"
84#include "llvm/IR/MemCacheHint.h"
85#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
86#include "llvm/IR/Metadata.h"
87#include "llvm/IR/Module.h"
88#include "llvm/IR/Operator.h"
89#include "llvm/IR/PatternMatch.h"
90#include "llvm/IR/ProfDataUtils.h"
91#include "llvm/IR/Statepoint.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCAsmInfo.h"
96#include "llvm/MC/MCContext.h"
97#include "llvm/Support/AtomicOrdering.h"
98#include "llvm/Support/Casting.h"
99#include "llvm/Support/CommandLine.h"
100#include "llvm/Support/Compiler.h"
101#include "llvm/Support/Debug.h"
102#include "llvm/Support/InstructionCost.h"
103#include "llvm/Support/MathExtras.h"
104#include "llvm/Support/raw_ostream.h"
105#include "llvm/Target/TargetMachine.h"
106#include "llvm/Target/TargetOptions.h"
107#include "llvm/TargetParser/Triple.h"
108#include "llvm/Transforms/Utils/Local.h"
109#include <cstddef>
110#include <limits>
111#include <optional>
112#include <tuple>
113
114using namespace llvm;
115using namespace PatternMatch;
116using namespace SwitchCG;
117
118#define DEBUG_TYPE "isel"
119
120/// LimitFloatPrecision - Generate low-precision inline sequences for
121/// some float libcalls (6, 8 or 12 bits).
122static unsigned LimitFloatPrecision;
123
124static cl::opt<bool>
125 InsertAssertAlign("insert-assert-align", cl::init(Val: true),
126 cl::desc("Insert the experimental `assertalign` node."),
127 cl::ReallyHidden);
128
129static cl::opt<unsigned, true>
130 LimitFPPrecision("limit-float-precision",
131 cl::desc("Generate low-precision inline sequences "
132 "for some float libcalls"),
133 cl::location(L&: LimitFloatPrecision), cl::Hidden,
134 cl::init(Val: 0));
135
136static cl::opt<unsigned> SwitchPeelThreshold(
137 "switch-peel-threshold", cl::Hidden, cl::init(Val: 66),
138 cl::desc("Set the case probability threshold for peeling the case from a "
139 "switch statement. A value greater than 100 will void this "
140 "optimization"));
141
142// Limit the width of DAG chains. This is important in general to prevent
143// DAG-based analysis from blowing up. For example, alias analysis and
144// load clustering may not complete in reasonable time. It is difficult to
145// recognize and avoid this situation within each individual analysis, and
146// future analyses are likely to have the same behavior. Limiting DAG width is
147// the safe approach and will be especially important with global DAGs.
148//
149// MaxParallelChains default is arbitrarily high to avoid affecting
150// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
151// sequence over this should have been converted to llvm.memcpy by the
152// frontend. It is easy to induce this behavior with .ll code such as:
153// %buffer = alloca [4096 x i8]
154// %data = load [4096 x i8]* %argPtr
155// store [4096 x i8] %data, [4096 x i8]* %buffer
156static const unsigned MaxParallelChains = 64;
157
158static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
159 const SDValue *Parts, unsigned NumParts,
160 MVT PartVT, EVT ValueVT, const Value *V,
161 SDValue InChain,
162 std::optional<CallingConv::ID> CC);
163
164/// getCopyFromParts - Create a value that contains the specified legal parts
165/// combined into the value they represent. If the parts combine to a type
166/// larger than ValueVT then AssertOp can be used to specify whether the extra
167/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
168/// (ISD::AssertSext).
169static SDValue
170getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
171 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
172 SDValue InChain,
173 std::optional<CallingConv::ID> CC = std::nullopt,
174 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
175 // Let the target assemble the parts if it wants to
176 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
177 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
178 PartVT, ValueVT, CC))
179 return Val;
180
181 if (ValueVT.isVector())
182 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
183 InChain, CC);
184
185 assert(NumParts > 0 && "No parts to assemble!");
186 SDValue Val = Parts[0];
187
188 if (NumParts > 1) {
189 // Assemble the value from multiple parts.
190 if (ValueVT.isInteger()) {
191 unsigned PartBits = PartVT.getSizeInBits();
192 unsigned ValueBits = ValueVT.getSizeInBits();
193
194 // Assemble the power of 2 part.
195 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
196 unsigned RoundBits = PartBits * RoundParts;
197 EVT RoundVT = RoundBits == ValueBits ?
198 ValueVT : EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits);
199 SDValue Lo, Hi;
200
201 EVT HalfVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits/2);
202
203 if (RoundParts > 2) {
204 Lo = getCopyFromParts(DAG, DL, Parts, NumParts: RoundParts / 2, PartVT, ValueVT: HalfVT, V,
205 InChain);
206 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts / 2, NumParts: RoundParts / 2,
207 PartVT, ValueVT: HalfVT, V, InChain);
208 } else {
209 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[0]);
210 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[1]);
211 }
212
213 if (DAG.getDataLayout().isBigEndian())
214 std::swap(a&: Lo, b&: Hi);
215
216 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: RoundVT, N1: Lo, N2: Hi);
217
218 if (RoundParts < NumParts) {
219 // Assemble the trailing non-power-of-2 part.
220 unsigned OddParts = NumParts - RoundParts;
221 EVT OddVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OddParts * PartBits);
222 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts, NumParts: OddParts, PartVT,
223 ValueVT: OddVT, V, InChain, CC);
224
225 // Combine the round and odd parts.
226 Lo = Val;
227 if (DAG.getDataLayout().isBigEndian())
228 std::swap(a&: Lo, b&: Hi);
229 EVT TotalVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
230 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: TotalVT, Operand: Hi);
231 Hi = DAG.getNode(
232 Opcode: ISD::SHL, DL, VT: TotalVT, N1: Hi,
233 N2: DAG.getShiftAmountConstant(Val: Lo.getValueSizeInBits(), VT: TotalVT, DL));
234 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: TotalVT, Operand: Lo);
235 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: TotalVT, N1: Lo, N2: Hi);
236 }
237 } else if (PartVT.isFloatingPoint()) {
238 // FP split into multiple FP parts (for ppcf128)
239 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
240 "Unexpected split");
241 SDValue Lo, Hi;
242 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EVT(MVT::f64), Operand: Parts[0]);
243 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EVT(MVT::f64), Operand: Parts[1]);
244 if (TLI.hasBigEndianPartOrdering(VT: ValueVT, DL: DAG.getDataLayout()))
245 std::swap(a&: Lo, b&: Hi);
246 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: ValueVT, N1: Lo, N2: Hi);
247 } else {
248 // FP split into integer parts (soft fp)
249 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
250 !PartVT.isVector() && "Unexpected split");
251 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
252 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, ValueVT: IntVT, V,
253 InChain, CC);
254 }
255 }
256
257 // There is now one part, held in Val. Correct it to match ValueVT.
258 // PartEVT is the type of the register class that holds the value.
259 // ValueVT is the type of the inline asm operation.
260 EVT PartEVT = Val.getValueType();
261
262 if (PartEVT == ValueVT)
263 return Val;
264
265 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
266 ValueVT.bitsLT(VT: PartEVT)) {
267 // For an FP value in an integer part, we need to truncate to the right
268 // width first.
269 PartEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
270 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: PartEVT, Operand: Val);
271 }
272
273 // Handle types that have the same size.
274 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
275 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
276
277 // Handle types with different sizes.
278 if (PartEVT.isInteger() && ValueVT.isInteger()) {
279 if (ValueVT.bitsLT(VT: PartEVT)) {
280 // For a truncate, see if we have any information to
281 // indicate whether the truncated bits will always be
282 // zero or sign-extension.
283 if (AssertOp)
284 Val = DAG.getNode(Opcode: *AssertOp, DL, VT: PartEVT, N1: Val,
285 N2: DAG.getValueType(ValueVT));
286 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
287 }
288 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
289 }
290
291 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
292 // FP_ROUND's are always exact here.
293 if (ValueVT.bitsLT(VT: Val.getValueType())) {
294
295 SDValue NoChange =
296 DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
297
298 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
299 Kind: llvm::Attribute::StrictFP)) {
300 return DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL,
301 VTList: DAG.getVTList(VT1: ValueVT, VT2: MVT::Other), N1: InChain, N2: Val,
302 N3: NoChange);
303 }
304
305 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ValueVT, N1: Val, N2: NoChange);
306 }
307
308 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: ValueVT, Operand: Val);
309 }
310
311 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
312 // then truncating.
313 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
314 ValueVT.bitsLT(VT: PartEVT)) {
315 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Val);
316 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
317 }
318
319 report_fatal_error(reason: "Unknown mismatch in getCopyFromParts!");
320}
321
322static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
323 const Twine &ErrMsg) {
324 const Instruction *I = dyn_cast_or_null<Instruction>(Val: V);
325 if (!I)
326 return Ctx.emitError(ErrorStr: ErrMsg);
327
328 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
329 if (CI->isInlineAsm()) {
330 return Ctx.diagnose(DI: DiagnosticInfoInlineAsm(
331 *CI, ErrMsg + ", possible invalid constraint for vector type"));
332 }
333
334 return Ctx.emitError(I, ErrorStr: ErrMsg);
335}
336
337/// Emit a fatal error if the broken down registers don't match part type
338/// or count.
339static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts,
340 MVT RegisterVT, MVT PartVT) {
341 if (NumRegs == NumParts && RegisterVT == PartVT)
342 return;
343
344 llvm::reportFatalInternalError(
345 reason: Twine("Part count doesn't match vector breakdown! ") + Twine(NumRegs) +
346 " registers, " + Twine(NumParts) + " parts, " + RegisterVT.getString() +
347 " RegisterVT, " + PartVT.getString() + " PartVT");
348}
349
350/// getCopyFromPartsVector - Create a value that contains the specified legal
351/// parts combined into the value they represent. If the parts combine to a
352/// type larger than ValueVT then AssertOp can be used to specify whether the
353/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
354/// ValueVT (ISD::AssertSext).
355static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
356 const SDValue *Parts, unsigned NumParts,
357 MVT PartVT, EVT ValueVT, const Value *V,
358 SDValue InChain,
359 std::optional<CallingConv::ID> CallConv) {
360 assert(ValueVT.isVector() && "Not a vector value");
361 assert(NumParts > 0 && "No parts to assemble!");
362 const bool IsABIRegCopy = CallConv.has_value();
363
364 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
365 SDValue Val = Parts[0];
366
367 // Handle a multi-element vector.
368 if (NumParts > 1) {
369 EVT IntermediateVT;
370 MVT RegisterVT;
371 unsigned NumIntermediates;
372 unsigned NumRegs;
373
374 if (IsABIRegCopy) {
375 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
376 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT,
377 NumIntermediates, RegisterVT);
378 } else {
379 NumRegs =
380 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
381 NumIntermediates, RegisterVT);
382 }
383
384 ensureMatchedVecRegParts(NumRegs, NumParts, RegisterVT, PartVT);
385 assert(RegisterVT.getSizeInBits() ==
386 Parts[0].getSimpleValueType().getSizeInBits() &&
387 "Part type sizes don't match!");
388
389 // Assemble the parts into intermediate operands.
390 SmallVector<SDValue, 8> Ops(NumIntermediates);
391 if (NumIntermediates == NumParts) {
392 // If the register was not expanded, truncate or copy the value,
393 // as appropriate.
394 for (unsigned i = 0; i != NumParts; ++i)
395 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i], NumParts: 1, PartVT, ValueVT: IntermediateVT,
396 V, InChain, CC: CallConv);
397 } else if (NumParts > 0) {
398 // If the intermediate type was expanded, build the intermediate
399 // operands from the parts.
400 assert(NumParts % NumIntermediates == 0 &&
401 "Must expand into a divisible number of parts!");
402 unsigned Factor = NumParts / NumIntermediates;
403 for (unsigned i = 0; i != NumIntermediates; ++i)
404 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i * Factor], NumParts: Factor, PartVT,
405 ValueVT: IntermediateVT, V, InChain, CC: CallConv);
406 }
407
408 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
409 // intermediate operands.
410 EVT BuiltVectorTy =
411 IntermediateVT.isVector()
412 ? EVT::getVectorVT(
413 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(),
414 EC: IntermediateVT.getVectorElementCount() * NumParts)
415 : EVT::getVectorVT(Context&: *DAG.getContext(),
416 VT: IntermediateVT.getScalarType(),
417 NumElements: NumIntermediates);
418 Val = DAG.getNode(Opcode: IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
419 : ISD::BUILD_VECTOR,
420 DL, VT: BuiltVectorTy, Ops);
421 }
422
423 // There is now one part, held in Val. Correct it to match ValueVT.
424 EVT PartEVT = Val.getValueType();
425
426 if (PartEVT == ValueVT)
427 return Val;
428
429 if (PartEVT.isVector()) {
430 // Vector/Vector bitcast.
431 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
432 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
433
434 // If the parts vector has more elements than the value vector, then we
435 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
436 // Extract the elements we want.
437 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
438 assert((PartEVT.getVectorElementCount().getKnownMinValue() >
439 ValueVT.getVectorElementCount().getKnownMinValue()) &&
440 (PartEVT.getVectorElementCount().isScalable() ==
441 ValueVT.getVectorElementCount().isScalable()) &&
442 "Cannot narrow, it would be a lossy transformation");
443 PartEVT =
444 EVT::getVectorVT(Context&: *DAG.getContext(), VT: PartEVT.getVectorElementType(),
445 EC: ValueVT.getVectorElementCount());
446 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: PartEVT, N1: Val,
447 N2: DAG.getVectorIdxConstant(Val: 0, DL));
448 if (PartEVT == ValueVT)
449 return Val;
450 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
451 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
452
453 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
454 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
455 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
456 }
457
458 // Promoted vector extract
459 return DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueVT);
460 }
461
462 // Trivial bitcast if the types are the same size and the destination
463 // vector type is legal.
464 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
465 TLI.isTypeLegal(VT: ValueVT))
466 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
467
468 if (ValueVT.getVectorNumElements() != 1) {
469 // Certain ABIs require that vectors are passed as integers. For vectors
470 // are the same size, this is an obvious bitcast.
471 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
472 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
473 } else if (ValueVT.bitsLT(VT: PartEVT)) {
474 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
475 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
476 // Drop the extra bits.
477 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
478 return DAG.getBitcast(VT: ValueVT, V: Val);
479 }
480
481 diagnosePossiblyInvalidConstraint(
482 Ctx&: *DAG.getContext(), V, ErrMsg: "non-trivial scalar-to-vector conversion");
483 return DAG.getUNDEF(VT: ValueVT);
484 }
485
486 // Handle cases such as i8 -> <1 x i1>
487 EVT ValueSVT = ValueVT.getVectorElementType();
488 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
489 unsigned ValueSize = ValueSVT.getSizeInBits();
490 if (ValueSize == PartEVT.getSizeInBits()) {
491 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueSVT, Operand: Val);
492 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
493 // It's possible a scalar floating point type gets softened to integer and
494 // then promoted to a larger integer. If PartEVT is the larger integer
495 // we need to truncate it and then bitcast to the FP type.
496 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
497 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
498 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
499 Val = DAG.getBitcast(VT: ValueSVT, V: Val);
500 } else {
501 Val = ValueVT.isFloatingPoint()
502 ? DAG.getFPExtendOrRound(Op: Val, DL, VT: ValueSVT)
503 : DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueSVT);
504 }
505 }
506
507 return DAG.getBuildVector(VT: ValueVT, DL, Ops: Val);
508}
509
510static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
511 SDValue Val, SDValue *Parts, unsigned NumParts,
512 MVT PartVT, const Value *V,
513 std::optional<CallingConv::ID> CallConv);
514
515/// getCopyToParts - Create a series of nodes that contain the specified value
516/// split into legal parts. If the parts contain more bits than Val, then, for
517/// integers, ExtendKind can be used to specify how to generate the extra bits.
518static void
519getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
520 unsigned NumParts, MVT PartVT, const Value *V,
521 std::optional<CallingConv::ID> CallConv = std::nullopt,
522 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
523 // Let the target split the parts if it wants to
524 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
525 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
526 CC: CallConv))
527 return;
528 EVT ValueVT = Val.getValueType();
529
530 // Handle the vector case separately.
531 if (ValueVT.isVector())
532 return getCopyToPartsVector(DAG, dl: DL, Val, Parts, NumParts, PartVT, V,
533 CallConv);
534
535 unsigned OrigNumParts = NumParts;
536 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
537 "Copying to an illegal type!");
538
539 if (NumParts == 0)
540 return;
541
542 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
543 EVT PartEVT = PartVT;
544 if (PartEVT == ValueVT) {
545 assert(NumParts == 1 && "No-op copy with multiple parts!");
546 Parts[0] = Val;
547 return;
548 }
549
550 unsigned PartBits = PartVT.getSizeInBits();
551 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
552 // If the parts cover more bits than the value has, promote the value.
553 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
554 assert(NumParts == 1 && "Do not know what to promote to!");
555 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
556 } else {
557 if (ValueVT.isFloatingPoint()) {
558 // FP values need to be bitcast, then extended if they are being put
559 // into a larger container.
560 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
561 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
562 }
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
567 Val = DAG.getNode(Opcode: ExtendKind, DL, VT: ValueVT, Operand: Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
570 }
571 } else if (PartBits == ValueVT.getSizeInBits()) {
572 // Different types of the same size.
573 assert(NumParts == 1 && PartEVT != ValueVT);
574 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
575 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
576 // If the parts cover less bits than value has, truncate the value.
577 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
578 ValueVT.isInteger() &&
579 "Unknown mismatch!");
580 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
581 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
582 if (PartVT == MVT::x86mmx)
583 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
584 }
585
586 // The value may have changed - recompute ValueVT.
587 ValueVT = Val.getValueType();
588 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
589 "Failed to tile the value with PartVT!");
590
591 if (NumParts == 1) {
592 if (PartEVT != ValueVT) {
593 diagnosePossiblyInvalidConstraint(Ctx&: *DAG.getContext(), V,
594 ErrMsg: "scalar-to-vector conversion failed");
595 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
596 }
597
598 Parts[0] = Val;
599 return;
600 }
601
602 // Expand the value into multiple parts.
603 if (NumParts & (NumParts - 1)) {
604 // The number of parts is not a power of 2. Split off and copy the tail.
605 assert(PartVT.isInteger() && ValueVT.isInteger() &&
606 "Do not know what to expand to!");
607 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
608 unsigned RoundBits = RoundParts * PartBits;
609 unsigned OddParts = NumParts - RoundParts;
610 SDValue OddVal = DAG.getNode(Opcode: ISD::SRL, DL, VT: ValueVT, N1: Val,
611 N2: DAG.getShiftAmountConstant(Val: RoundBits, VT: ValueVT, DL));
612
613 getCopyToParts(DAG, DL, Val: OddVal, Parts: Parts + RoundParts, NumParts: OddParts, PartVT, V,
614 CallConv);
615
616 if (DAG.getDataLayout().isBigEndian())
617 // The odd parts were reversed by getCopyToParts - unreverse them.
618 std::reverse(first: Parts + RoundParts, last: Parts + NumParts);
619
620 NumParts = RoundParts;
621 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
622 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
623 }
624
625 // The number of parts is a power of 2. Repeatedly bisect the value using
626 // EXTRACT_ELEMENT.
627 Parts[0] = DAG.getNode(Opcode: ISD::BITCAST, DL,
628 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
629 BitWidth: ValueVT.getSizeInBits()),
630 Operand: Val);
631
632 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
633 for (unsigned i = 0; i < NumParts; i += StepSize) {
634 unsigned ThisBits = StepSize * PartBits / 2;
635 EVT ThisVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ThisBits);
636 SDValue &Part0 = Parts[i];
637 SDValue &Part1 = Parts[i+StepSize/2];
638
639 Part1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
640 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 1, DL));
641 Part0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
642 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 0, DL));
643
644 if (ThisBits == PartBits && ThisVT != PartVT) {
645 Part0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part0);
646 Part1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part1);
647 }
648 }
649 }
650
651 if (DAG.getDataLayout().isBigEndian())
652 std::reverse(first: Parts, last: Parts + OrigNumParts);
653}
654
655static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
656 const SDLoc &DL, EVT PartVT) {
657 if (!PartVT.isVector())
658 return SDValue();
659
660 EVT ValueVT = Val.getValueType();
661 EVT PartEVT = PartVT.getVectorElementType();
662 EVT ValueEVT = ValueVT.getVectorElementType();
663 ElementCount PartNumElts = PartVT.getVectorElementCount();
664 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
665
666 // We only support widening vectors with equivalent element types and
667 // fixed/scalable properties. If a target needs to widen a fixed-length type
668 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
669 if (ElementCount::isKnownLE(LHS: PartNumElts, RHS: ValueNumElts) ||
670 PartNumElts.isScalable() != ValueNumElts.isScalable())
671 return SDValue();
672
673 // Have a try for bf16 because some targets share its ABI with fp16.
674 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
675 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
676 "Cannot widen to illegal type");
677 Val = DAG.getNode(
678 Opcode: ISD::BITCAST, DL,
679 VT: ValueVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::f16), Operand: Val);
680 } else if (PartEVT != ValueEVT) {
681 return SDValue();
682 }
683
684 // Widening a scalable vector to another scalable vector is done by inserting
685 // the vector into a larger undef one.
686 if (PartNumElts.isScalable())
687 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: PartVT, N1: DAG.getUNDEF(VT: PartVT),
688 N2: Val, N3: DAG.getVectorIdxConstant(Val: 0, DL));
689
690 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
691 // undef elements.
692 SmallVector<SDValue, 16> Ops;
693 DAG.ExtractVectorElements(Op: Val, Args&: Ops);
694 SDValue EltUndef = DAG.getUNDEF(VT: PartEVT);
695 Ops.append(NumInputs: (PartNumElts - ValueNumElts).getFixedValue(), Elt: EltUndef);
696
697 // FIXME: Use CONCAT for 2x -> 4x.
698 return DAG.getBuildVector(VT: PartVT, DL, Ops);
699}
700
701/// getCopyToPartsVector - Create a series of nodes that contain the specified
702/// value split into legal parts.
703static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
704 SDValue Val, SDValue *Parts, unsigned NumParts,
705 MVT PartVT, const Value *V,
706 std::optional<CallingConv::ID> CallConv) {
707 EVT ValueVT = Val.getValueType();
708 assert(ValueVT.isVector() && "Not a vector");
709 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
710 const bool IsABIRegCopy = CallConv.has_value();
711
712 if (NumParts == 1) {
713 EVT PartEVT = PartVT;
714 if (PartEVT == ValueVT) {
715 // Nothing to do.
716 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
717 // Bitconvert vector->vector case.
718 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
719 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
720 Val = Widened;
721 } else if (PartVT.isVector() &&
722 PartEVT.getVectorElementType().bitsGE(
723 VT: ValueVT.getVectorElementType()) &&
724 PartEVT.getVectorElementCount() ==
725 ValueVT.getVectorElementCount()) {
726
727 // Promoted vector extract
728 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
729 } else if (PartEVT.isVector() &&
730 PartEVT.getVectorElementType() !=
731 ValueVT.getVectorElementType() &&
732 TLI.getTypeAction(Context&: *DAG.getContext(), VT: ValueVT) ==
733 TargetLowering::TypeWidenVector) {
734 // Combination of widening and promotion.
735 EVT WidenVT =
736 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
737 EC: PartVT.getVectorElementCount());
738 SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: WidenVT);
739 Val = DAG.getAnyExtOrTrunc(Op: Widened, DL, VT: PartVT);
740 } else {
741 // Don't extract an integer from a float vector. This can happen if the
742 // FP type gets softened to integer and then promoted. The promotion
743 // prevents it from being picked up by the earlier bitcast case.
744 if (ValueVT.getVectorElementCount().isScalar() &&
745 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
746 // If we reach this condition and PartVT is FP, this means that
747 // ValueVT is also FP and both have a different size, otherwise we
748 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
749 // would be invalid since that would mean the smaller FP type has to
750 // be extended to the larger one.
751 if (PartVT.isFloatingPoint()) {
752 Val = DAG.getBitcast(VT: ValueVT.getScalarType(), V: Val);
753 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
754 } else
755 Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: PartVT, N1: Val,
756 N2: DAG.getVectorIdxConstant(Val: 0, DL));
757 } else {
758 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
759 assert(PartVT.getFixedSizeInBits() > ValueSize &&
760 "lossy conversion of vector to scalar type");
761 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
762 Val = DAG.getBitcast(VT: IntermediateType, V: Val);
763 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
764 }
765 }
766
767 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
768 Parts[0] = Val;
769 return;
770 }
771
772 // Handle a multi-element vector.
773 EVT IntermediateVT;
774 MVT RegisterVT;
775 unsigned NumIntermediates;
776 unsigned NumRegs;
777 if (IsABIRegCopy) {
778 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
779 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT, NumIntermediates,
780 RegisterVT);
781 } else {
782 NumRegs =
783 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
784 NumIntermediates, RegisterVT);
785 }
786
787 ensureMatchedVecRegParts(NumRegs, NumParts, RegisterVT, PartVT);
788 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
789 "Mixing scalable and fixed vectors when copying in parts");
790
791 std::optional<ElementCount> DestEltCnt;
792
793 if (IntermediateVT.isVector())
794 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
795 else
796 DestEltCnt = ElementCount::getFixed(MinVal: NumIntermediates);
797
798 EVT BuiltVectorTy = EVT::getVectorVT(
799 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(), EC: *DestEltCnt);
800
801 if (ValueVT == BuiltVectorTy) {
802 // Nothing to do.
803 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
804 // Bitconvert vector->vector case.
805 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BuiltVectorTy, Operand: Val);
806 } else {
807 if (BuiltVectorTy.getVectorElementType().bitsGT(
808 VT: ValueVT.getVectorElementType())) {
809 // Integer promotion.
810 ValueVT = EVT::getVectorVT(Context&: *DAG.getContext(),
811 VT: BuiltVectorTy.getVectorElementType(),
812 EC: ValueVT.getVectorElementCount());
813 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
814 }
815
816 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: BuiltVectorTy)) {
817 Val = Widened;
818 }
819 }
820
821 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
822
823 // Split the vector into intermediate operands.
824 SmallVector<SDValue, 8> Ops(NumIntermediates);
825 for (unsigned i = 0; i != NumIntermediates; ++i) {
826 if (IntermediateVT.isVector()) {
827 // This does something sensible for scalable vectors - see the
828 // definition of EXTRACT_SUBVECTOR for further details.
829 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
830 Ops[i] =
831 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: IntermediateVT, N1: Val,
832 N2: DAG.getVectorIdxConstant(Val: i * IntermediateNumElts, DL));
833 } else {
834 Ops[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IntermediateVT, N1: Val,
835 N2: DAG.getVectorIdxConstant(Val: i, DL));
836 }
837 }
838
839 // Split the intermediate operands into legal parts.
840 if (NumParts == NumIntermediates) {
841 // If the register was not expanded, promote or copy the value,
842 // as appropriate.
843 for (unsigned i = 0; i != NumParts; ++i)
844 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i], NumParts: 1, PartVT, V, CallConv);
845 } else if (NumParts > 0) {
846 // If the intermediate type was expanded, split each the value into
847 // legal parts.
848 assert(NumIntermediates != 0 && "division by zero");
849 assert(NumParts % NumIntermediates == 0 &&
850 "Must expand into a divisible number of parts!");
851 unsigned Factor = NumParts / NumIntermediates;
852 for (unsigned i = 0; i != NumIntermediates; ++i)
853 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i * Factor], NumParts: Factor, PartVT, V,
854 CallConv);
855 }
856}
857
858static void failForInvalidBundles(const CallBase &I, StringRef Name,
859 ArrayRef<uint32_t> AllowedBundles) {
860 if (I.hasOperandBundlesOtherThan(IDs: AllowedBundles)) {
861 ListSeparator LS;
862 std::string Error;
863 raw_string_ostream OS(Error);
864 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
865 OperandBundleUse U = I.getOperandBundleAt(Index: i);
866 if (!is_contained(Range&: AllowedBundles, Element: U.getTagID()))
867 OS << LS << U.getTagName();
868 }
869 reportFatalUsageError(
870 reason: Twine("cannot lower ", Name)
871 .concat(Suffix: Twine(" with arbitrary operand bundles: ", Error)));
872 }
873}
874
875RegsForValue::RegsForValue(const SmallVector<Register, 4> &regs, MVT regvt,
876 EVT valuevt, std::optional<CallingConv::ID> CC)
877 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
878 RegCount(1, regs.size()), CallConv(CC) {}
879
880RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
881 const DataLayout &DL, Register Reg, Type *Ty,
882 std::optional<CallingConv::ID> CC) {
883 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
884
885 CallConv = CC;
886
887 for (EVT ValueVT : ValueVTs) {
888 unsigned NumRegs =
889 isABIMangled()
890 ? TLI.getNumRegistersForCallingConv(Context, CC: *CC, VT: ValueVT)
891 : TLI.getNumRegisters(Context, VT: ValueVT);
892 MVT RegisterVT =
893 isABIMangled()
894 ? TLI.getRegisterTypeForCallingConv(Context, CC: *CC, VT: ValueVT)
895 : TLI.getRegisterType(Context, VT: ValueVT);
896 for (unsigned i = 0; i != NumRegs; ++i)
897 Regs.push_back(Elt: Reg + i);
898 RegVTs.push_back(Elt: RegisterVT);
899 RegCount.push_back(Elt: NumRegs);
900 Reg = Reg.id() + NumRegs;
901 }
902}
903
904SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
905 FunctionLoweringInfo &FuncInfo,
906 const SDLoc &dl, SDValue &Chain,
907 SDValue *Glue, const Value *V) const {
908 // A Value with type {} or [0 x %t] needs no registers.
909 if (ValueVTs.empty())
910 return SDValue();
911
912 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
913
914 // Assemble the legal parts into the final values.
915 SmallVector<SDValue, 4> Values(ValueVTs.size());
916 SmallVector<SDValue, 8> Parts;
917 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
918 // Copy the legal parts from the registers.
919 EVT ValueVT = ValueVTs[Value];
920 unsigned NumRegs = RegCount[Value];
921 MVT RegisterVT = isABIMangled()
922 ? TLI.getRegisterTypeForCallingConv(
923 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
924 : RegVTs[Value];
925
926 Parts.resize(N: NumRegs);
927 for (unsigned i = 0; i != NumRegs; ++i) {
928 SDValue P;
929 if (!Glue) {
930 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT);
931 } else {
932 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT, Glue: *Glue);
933 *Glue = P.getValue(R: 2);
934 }
935
936 Chain = P.getValue(R: 1);
937 Parts[i] = P;
938
939 // If the source register was virtual and if we know something about it,
940 // add an assert node.
941 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
942 continue;
943
944 const FunctionLoweringInfo::LiveOutInfo *LOI =
945 FuncInfo.GetLiveOutRegInfo(Reg: Regs[Part+i]);
946 if (!LOI)
947 continue;
948
949 unsigned RegSize = RegisterVT.getScalarSizeInBits();
950 unsigned NumSignBits = LOI->NumSignBits;
951 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
952
953 if (NumZeroBits == RegSize) {
954 // The current value is a zero.
955 // Explicitly express that as it would be easier for
956 // optimizations to kick in.
957 Parts[i] = DAG.getConstant(Val: 0, DL: dl, VT: RegisterVT);
958 continue;
959 }
960
961 // FIXME: We capture more information than the dag can represent. For
962 // now, just use the tightest assertzext/assertsext possible.
963 bool isSExt;
964 EVT FromVT(MVT::Other);
965 if (NumZeroBits) {
966 FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
967 isSExt = false;
968 } else if (NumSignBits > 1) {
969 FromVT =
970 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumSignBits + 1);
971 isSExt = true;
972 } else {
973 continue;
974 }
975 // Add an assertion node.
976 assert(FromVT != MVT::Other);
977 Parts[i] = DAG.getNode(Opcode: isSExt ? ISD::AssertSext : ISD::AssertZext, DL: dl,
978 VT: RegisterVT, N1: P, N2: DAG.getValueType(FromVT));
979 }
980
981 Values[Value] = getCopyFromParts(DAG, DL: dl, Parts: Parts.begin(), NumParts: NumRegs,
982 PartVT: RegisterVT, ValueVT, V, InChain: Chain, CC: CallConv);
983 Part += NumRegs;
984 Parts.clear();
985 }
986
987 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values);
988}
989
990void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
991 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
992 const Value *V,
993 ISD::NodeType PreferredExtendType) const {
994 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
995 ISD::NodeType ExtendKind = PreferredExtendType;
996
997 // Get the list of the values's legal parts.
998 unsigned NumRegs = Regs.size();
999 SmallVector<SDValue, 8> Parts(NumRegs);
1000 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
1001 unsigned NumParts = RegCount[Value];
1002
1003 MVT RegisterVT = isABIMangled()
1004 ? TLI.getRegisterTypeForCallingConv(
1005 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
1006 : RegVTs[Value];
1007
1008 if (ExtendKind == ISD::ANY_EXTEND)
1009 if (TLI.isZExtFree(Val: peekThroughFreeze(V: Val), VT2: RegisterVT))
1010 ExtendKind = ISD::ZERO_EXTEND;
1011
1012 getCopyToParts(DAG, DL: dl, Val: Val.getValue(R: Val.getResNo() + Value), Parts: &Parts[Part],
1013 NumParts, PartVT: RegisterVT, V, CallConv, ExtendKind);
1014 Part += NumParts;
1015 }
1016
1017 // Copy the parts into the registers.
1018 SmallVector<SDValue, 8> Chains(NumRegs);
1019 for (unsigned i = 0; i != NumRegs; ++i) {
1020 SDValue Part;
1021 if (!Glue) {
1022 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i]);
1023 } else {
1024 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i], Glue: *Glue);
1025 *Glue = Part.getValue(R: 1);
1026 }
1027
1028 Chains[i] = Part.getValue(R: 0);
1029 }
1030
1031 if (NumRegs == 1 || Glue)
1032 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1033 // flagged to it. That is the CopyToReg nodes and the user are considered
1034 // a single scheduling unit. If we create a TokenFactor and return it as
1035 // chain, then the TokenFactor is both a predecessor (operand) of the
1036 // user as well as a successor (the TF operands are flagged to the user).
1037 // c1, f1 = CopyToReg
1038 // c2, f2 = CopyToReg
1039 // c3 = TokenFactor c1, c2
1040 // ...
1041 // = op c3, ..., f2
1042 Chain = Chains[NumRegs-1];
1043 else
1044 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
1045}
1046
1047void RegsForValue::AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching,
1048 unsigned MatchingIdx, const SDLoc &dl,
1049 SelectionDAG &DAG,
1050 std::vector<SDValue> &Ops) const {
1051 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1052
1053 InlineAsm::Flag Flag(Code, Regs.size());
1054 if (HasMatching)
1055 Flag.setMatchingOp(MatchingIdx);
1056 else if (!Regs.empty() && Regs.front().isVirtual()) {
1057 // Put the register class of the virtual registers in the flag word. That
1058 // way, later passes can recompute register class constraints for inline
1059 // assembly as well as normal instructions.
1060 // Don't do this for tied operands that can use the regclass information
1061 // from the def.
1062 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
1063 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Regs.front());
1064 Flag.setRegClass(RC->getID());
1065 }
1066
1067 SDValue Res = DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32);
1068 Ops.push_back(x: Res);
1069
1070 if (Code == InlineAsm::Kind::Clobber) {
1071 // Clobbers should always have a 1:1 mapping with registers, and may
1072 // reference registers that have illegal (e.g. vector) types. Hence, we
1073 // shouldn't try to apply any sort of splitting logic to them.
1074 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1075 "No 1:1 mapping from clobbers to regs?");
1076 Register SP = TLI.getStackPointerRegisterToSaveRestore();
1077 (void)SP;
1078 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1079 Ops.push_back(x: DAG.getRegister(Reg: Regs[I], VT: RegVTs[I]));
1080 assert(
1081 (Regs[I] != SP ||
1082 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
1083 "If we clobbered the stack pointer, MFI should know about it.");
1084 }
1085 return;
1086 }
1087
1088 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1089 MVT RegisterVT = RegVTs[Value];
1090 unsigned NumRegs = TLI.getNumRegisters(Context&: *DAG.getContext(), VT: ValueVTs[Value],
1091 RegisterVT);
1092 for (unsigned i = 0; i != NumRegs; ++i) {
1093 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1094 Register TheReg = Regs[Reg++];
1095 Ops.push_back(x: DAG.getRegister(Reg: TheReg, VT: RegisterVT));
1096 }
1097 }
1098}
1099
1100SmallVector<std::pair<Register, TypeSize>, 4>
1101RegsForValue::getRegsAndSizes() const {
1102 SmallVector<std::pair<Register, TypeSize>, 4> OutVec;
1103 unsigned I = 0;
1104 for (auto CountAndVT : zip_first(t: RegCount, u: RegVTs)) {
1105 unsigned RegCount = std::get<0>(t&: CountAndVT);
1106 MVT RegisterVT = std::get<1>(t&: CountAndVT);
1107 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1108 for (unsigned E = I + RegCount; I != E; ++I)
1109 OutVec.push_back(Elt: std::make_pair(x: Regs[I], y&: RegisterSize));
1110 }
1111 return OutVec;
1112}
1113
1114void SelectionDAGBuilder::init(GCFunctionInfo *gfi, BatchAAResults *aa,
1115 AssumptionCache *ac, const TargetLibraryInfo *li,
1116 const TargetTransformInfo &TTI) {
1117 BatchAA = aa;
1118 AC = ac;
1119 GFI = gfi;
1120 LibInfo = li;
1121 Context = DAG.getContext();
1122 LPadToCallSiteMap.clear();
1123 this->TTI = &TTI;
1124 SL->init(tli: DAG.getTargetLoweringInfo(), tm: TM, dl: DAG.getDataLayout());
1125 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1126 M: *DAG.getMachineFunction().getFunction().getParent());
1127 CanDescribeGlobalAddressInLocationList =
1128 canDescribeGlobalAddressInLocationList(MF: DAG.getMachineFunction());
1129}
1130
1131void SelectionDAGBuilder::clear() {
1132 NodeMap.clear();
1133 UnusedArgNodeMap.clear();
1134 PendingLoads.clear();
1135 PendingExports.clear();
1136 PendingConstrainedFP.clear();
1137 PendingConstrainedFPStrict.clear();
1138 CurInst = nullptr;
1139 HasTailCall = false;
1140 SDNodeOrder = LowestSDNodeOrder;
1141 StatepointLowering.clear();
1142}
1143
1144void SelectionDAGBuilder::clearDanglingDebugInfo() {
1145 DanglingDebugInfoMap.clear();
1146}
1147
1148// Update DAG root to include dependencies on Pending chains.
1149SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1150 SDValue Root = DAG.getRoot();
1151
1152 if (Pending.empty())
1153 return Root;
1154
1155 // Add current root to PendingChains, unless we already indirectly
1156 // depend on it.
1157 if (Root.getOpcode() != ISD::EntryToken) {
1158 unsigned i = 0, e = Pending.size();
1159 for (; i != e; ++i) {
1160 assert(Pending[i].getNode()->getNumOperands() > 1);
1161 if (Pending[i].getNode()->getOperand(Num: 0) == Root)
1162 break; // Don't add the root if we already indirectly depend on it.
1163 }
1164
1165 if (i == e)
1166 Pending.push_back(Elt: Root);
1167 }
1168
1169 if (Pending.size() == 1)
1170 Root = Pending[0];
1171 else
1172 Root = DAG.getTokenFactor(DL: getCurSDLoc(), Vals&: Pending);
1173
1174 DAG.setRoot(Root);
1175 Pending.clear();
1176 return Root;
1177}
1178
1179SDValue SelectionDAGBuilder::getMemoryRoot() {
1180 return updateRoot(Pending&: PendingLoads);
1181}
1182
1183SDValue SelectionDAGBuilder::getFPOperationRoot(fp::ExceptionBehavior EB) {
1184 // If the new exception behavior differs from that of the pending
1185 // ones, chain up them and update the root.
1186 switch (EB) {
1187 case fp::ExceptionBehavior::ebMayTrap:
1188 case fp::ExceptionBehavior::ebIgnore:
1189 // Floating-point exceptions produced by such operations are not intended
1190 // to be observed, so the sequence of these operations does not need to be
1191 // preserved.
1192 //
1193 // They however must not be mixed with the instructions that have strict
1194 // exception behavior. Placing an operation with 'ebIgnore' behavior between
1195 // 'ebStrict' operations could distort the observed exception behavior.
1196 if (!PendingConstrainedFPStrict.empty()) {
1197 assert(PendingConstrainedFP.empty());
1198 updateRoot(Pending&: PendingConstrainedFPStrict);
1199 }
1200 break;
1201 case fp::ExceptionBehavior::ebStrict:
1202 // Floating-point exception produced by these operations may be observed, so
1203 // they must be correctly chained. If trapping on FP exceptions is
1204 // disabled, the exceptions can be observed only by functions that read
1205 // exception flags, like 'llvm.get_fpenv' or 'fetestexcept'. It means that
1206 // the order of operations is not significant between barriers.
1207 //
1208 // If trapping is enabled, each operation becomes an implicit observation
1209 // point, so the operations must be sequenced according their original
1210 // source order.
1211 if (!PendingConstrainedFP.empty()) {
1212 assert(PendingConstrainedFPStrict.empty());
1213 updateRoot(Pending&: PendingConstrainedFP);
1214 }
1215 // TODO: Add support for trapping-enabled scenarios.
1216 }
1217 return DAG.getRoot();
1218}
1219
1220SDValue SelectionDAGBuilder::getRoot() {
1221 // Chain up all pending constrained intrinsics together with all
1222 // pending loads, by simply appending them to PendingLoads and
1223 // then calling getMemoryRoot().
1224 PendingLoads.reserve(N: PendingLoads.size() +
1225 PendingConstrainedFP.size() +
1226 PendingConstrainedFPStrict.size());
1227 PendingLoads.append(in_start: PendingConstrainedFP.begin(),
1228 in_end: PendingConstrainedFP.end());
1229 PendingLoads.append(in_start: PendingConstrainedFPStrict.begin(),
1230 in_end: PendingConstrainedFPStrict.end());
1231 PendingConstrainedFP.clear();
1232 PendingConstrainedFPStrict.clear();
1233 return getMemoryRoot();
1234}
1235
1236SDValue SelectionDAGBuilder::getControlRoot() {
1237 // We need to emit pending fpexcept.strict constrained intrinsics,
1238 // so append them to the PendingExports list.
1239 PendingExports.append(in_start: PendingConstrainedFPStrict.begin(),
1240 in_end: PendingConstrainedFPStrict.end());
1241 PendingConstrainedFPStrict.clear();
1242 return updateRoot(Pending&: PendingExports);
1243}
1244
1245void SelectionDAGBuilder::handleDebugDeclare(Value *Address,
1246 DILocalVariable *Variable,
1247 DIExpression *Expression,
1248 DebugLoc DL) {
1249 assert(Variable && "Missing variable");
1250
1251 // Check if address has undef value.
1252 if (!Address || isa<UndefValue>(Val: Address) ||
1253 (Address->use_empty() && !isa<Argument>(Val: Address))) {
1254 LLVM_DEBUG(
1255 dbgs()
1256 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1257 return;
1258 }
1259
1260 bool IsParameter = Variable->isParameter() || isa<Argument>(Val: Address);
1261
1262 SDValue &N = NodeMap[Address];
1263 if (!N.getNode() && isa<Argument>(Val: Address))
1264 // Check unused arguments map.
1265 N = UnusedArgNodeMap[Address];
1266 SDDbgValue *SDV;
1267 if (N.getNode()) {
1268 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Val: Address))
1269 Address = BCI->getOperand(i_nocapture: 0);
1270 // Parameters are handled specially.
1271 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: N.getNode());
1272 if (IsParameter && FINode) {
1273 // Byval parameter. We have a frame index at this point.
1274 SDV = DAG.getFrameIndexDbgValue(Var: Variable, Expr: Expression, FI: FINode->getIndex(),
1275 /*IsIndirect*/ true, DL, O: SDNodeOrder);
1276 } else if (isa<Argument>(Val: Address)) {
1277 // Address is an argument, so try to emit its dbg value using
1278 // virtual register info from the FuncInfo.ValueMap.
1279 EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1280 Kind: FuncArgumentDbgValueKind::Declare, N);
1281 return;
1282 } else {
1283 SDV = DAG.getDbgValue(Var: Variable, Expr: Expression, N: N.getNode(), R: N.getResNo(),
1284 IsIndirect: true, DL, O: SDNodeOrder);
1285 }
1286 DAG.AddDbgValue(DB: SDV, isParameter: IsParameter);
1287 } else {
1288 // If Address is an argument then try to emit its dbg value using
1289 // virtual register info from the FuncInfo.ValueMap.
1290 if (!EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1291 Kind: FuncArgumentDbgValueKind::Declare, N)) {
1292 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1293 << " (could not emit func-arg dbg_value)\n");
1294 }
1295 }
1296}
1297
1298void SelectionDAGBuilder::visitDbgInfo(const Instruction &I) {
1299 // Add SDDbgValue nodes for any var locs here. Do so before updating
1300 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1301 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1302 // Add SDDbgValue nodes for any var locs here. Do so before updating
1303 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1304 for (auto It = FnVarLocs->locs_begin(Before: &I), End = FnVarLocs->locs_end(Before: &I);
1305 It != End; ++It) {
1306 auto *Var = FnVarLocs->getDILocalVariable(ID: It->VariableID);
1307 dropDanglingDebugInfo(Variable: Var, Expr: It->Expr);
1308 if (It->Values.isKillLocation(Expression: It->Expr)) {
1309 handleKillDebugValue(Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder);
1310 continue;
1311 }
1312 SmallVector<Value *> Values(It->Values.location_ops());
1313 if (!handleDebugValue(Values, Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder,
1314 IsVariadic: It->Values.hasArgList())) {
1315 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1316 addDanglingDebugInfo(Values&: Vals,
1317 Var: FnVarLocs->getDILocalVariable(ID: It->VariableID),
1318 Expr: It->Expr, IsVariadic: Vals.size() > 1, DL: It->DL, Order: SDNodeOrder);
1319 }
1320 }
1321 }
1322
1323 // We must skip DbgVariableRecords if they've already been processed above as
1324 // we have just emitted the debug values resulting from assignment tracking
1325 // analysis, making any existing DbgVariableRecords redundant (and probably
1326 // less correct). We still need to process DbgLabelRecords. This does sink
1327 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1328 // be important as it does so deterministcally and ordering between
1329 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1330 // printing).
1331 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1332 // Is there is any debug-info attached to this instruction, in the form of
1333 // DbgRecord non-instruction debug-info records.
1334 for (DbgRecord &DR : I.getDbgRecordRange()) {
1335 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1336 assert(DLR->getLabel() && "Missing label");
1337 SDDbgLabel *SDV =
1338 DAG.getDbgLabel(Label: DLR->getLabel(), DL: DLR->getDebugLoc(), O: SDNodeOrder);
1339 DAG.AddDbgLabel(DB: SDV);
1340 continue;
1341 }
1342
1343 if (SkipDbgVariableRecords)
1344 continue;
1345 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1346 DILocalVariable *Variable = DVR.getVariable();
1347 DIExpression *Expression = DVR.getExpression();
1348 dropDanglingDebugInfo(Variable, Expr: Expression);
1349
1350 if (DVR.getType() == DbgVariableRecord::LocationType::Declare) {
1351 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1352 continue;
1353 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1354 << "\n");
1355 handleDebugDeclare(Address: DVR.getVariableLocationOp(OpIdx: 0), Variable, Expression,
1356 DL: DVR.getDebugLoc());
1357 continue;
1358 }
1359
1360 // A DbgVariableRecord with no locations is a kill location.
1361 SmallVector<Value *, 4> Values(DVR.location_ops());
1362 if (Values.empty()) {
1363 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1364 Order: SDNodeOrder);
1365 continue;
1366 }
1367
1368 // A DbgVariableRecord with an undef or absent location is also a kill
1369 // location.
1370 if (llvm::any_of(Range&: Values,
1371 P: [](Value *V) { return !V || isa<UndefValue>(Val: V); })) {
1372 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1373 Order: SDNodeOrder);
1374 continue;
1375 }
1376
1377 bool IsVariadic = DVR.hasArgList();
1378 if (!handleDebugValue(Values, Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1379 Order: SDNodeOrder, IsVariadic)) {
1380 addDanglingDebugInfo(Values, Var: Variable, Expr: Expression, IsVariadic,
1381 DL: DVR.getDebugLoc(), Order: SDNodeOrder);
1382 }
1383 }
1384}
1385
1386void SelectionDAGBuilder::visit(const Instruction &I) {
1387 visitDbgInfo(I);
1388
1389 // Set up outgoing PHI node register values before emitting the terminator.
1390 if (I.isTerminator()) {
1391 HandlePHINodesInSuccessorBlocks(LLVMBB: I.getParent());
1392 }
1393
1394 ++SDNodeOrder;
1395 CurInst = &I;
1396
1397 // Set inserted listener only if required.
1398 bool NodeInserted = false;
1399 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1400 MDNode *PCSectionsMD = I.getMetadata(KindID: LLVMContext::MD_pcsections);
1401 MDNode *MMRA = I.getMetadata(KindID: LLVMContext::MD_mmra);
1402 if (PCSectionsMD || MMRA) {
1403 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1404 args&: DAG, args: [&](SDNode *) { NodeInserted = true; });
1405 }
1406
1407 visit(Opcode: I.getOpcode(), I);
1408
1409 if (!I.isTerminator() && !HasTailCall &&
1410 !isa<GCStatepointInst>(Val: I)) // statepoints handle their exports internally
1411 CopyToExportRegsIfNeeded(V: &I);
1412
1413 // Handle metadata.
1414 if (PCSectionsMD || MMRA) {
1415 auto It = NodeMap.find(Val: &I);
1416 if (It != NodeMap.end()) {
1417 if (PCSectionsMD)
1418 DAG.addPCSections(Node: It->second.getNode(), MD: PCSectionsMD);
1419 if (MMRA)
1420 DAG.addMMRAMetadata(Node: It->second.getNode(), MMRA);
1421 } else if (NodeInserted) {
1422 // This should not happen; if it does, don't let it go unnoticed so we can
1423 // fix it. Relevant visit*() function is probably missing a setValue().
1424 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1425 << I.getModule()->getName() << "]\n";
1426 LLVM_DEBUG(I.dump());
1427 assert(false);
1428 }
1429 }
1430
1431 CurInst = nullptr;
1432}
1433
1434void SelectionDAGBuilder::visitPHI(const PHINode &) {
1435 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1436}
1437
1438void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1439 // Note: this doesn't use InstVisitor, because it has to work with
1440 // ConstantExpr's in addition to instructions.
1441 switch (Opcode) {
1442 default: llvm_unreachable("Unknown instruction type encountered!");
1443 // Build the switch statement using the Instruction.def file.
1444#define HANDLE_INST(NUM, OPCODE, CLASS) \
1445 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1446#include "llvm/IR/Instruction.def"
1447 }
1448}
1449
1450static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,
1451 DILocalVariable *Variable,
1452 DebugLoc DL, unsigned Order,
1453 SmallVectorImpl<Value *> &Values,
1454 DIExpression *Expression) {
1455 // For variadic dbg_values we will now insert poison.
1456 // FIXME: We can potentially recover these!
1457 SmallVector<SDDbgOperand, 2> Locs;
1458 for (const Value *V : Values) {
1459 auto *Poison = PoisonValue::get(T: V->getType());
1460 Locs.push_back(Elt: SDDbgOperand::fromConst(Const: Poison));
1461 }
1462 SDDbgValue *SDV = DAG.getDbgValueList(Var: Variable, Expr: Expression, Locs, Dependencies: {},
1463 /*IsIndirect=*/false, DL, O: Order,
1464 /*IsVariadic=*/true);
1465 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1466 return true;
1467}
1468
1469void SelectionDAGBuilder::addDanglingDebugInfo(SmallVectorImpl<Value *> &Values,
1470 DILocalVariable *Var,
1471 DIExpression *Expr,
1472 bool IsVariadic, DebugLoc DL,
1473 unsigned Order) {
1474 if (IsVariadic) {
1475 handleDanglingVariadicDebugInfo(DAG, Variable: Var, DL, Order, Values, Expression: Expr);
1476 return;
1477 }
1478 // TODO: Dangling debug info will eventually either be resolved or produce
1479 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1480 // between the original dbg.value location and its resolved DBG_VALUE,
1481 // which we should ideally fill with an extra poison DBG_VALUE.
1482 assert(Values.size() == 1);
1483 DanglingDebugInfoMap[Values[0]].emplace_back(args&: Var, args&: Expr, args&: DL, args&: Order);
1484}
1485
1486void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1487 const DIExpression *Expr) {
1488 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1489 DIVariable *DanglingVariable = DDI.getVariable();
1490 DIExpression *DanglingExpr = DDI.getExpression();
1491 if (DanglingVariable == Variable && Expr->fragmentsOverlap(Other: DanglingExpr)) {
1492 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1493 << printDDI(nullptr, DDI) << "\n");
1494 return true;
1495 }
1496 return false;
1497 };
1498
1499 for (auto &DDIMI : DanglingDebugInfoMap) {
1500 DanglingDebugInfoVector &DDIV = DDIMI.second;
1501
1502 // If debug info is to be dropped, run it through final checks to see
1503 // whether it can be salvaged.
1504 for (auto &DDI : DDIV)
1505 if (isMatchingDbgValue(DDI))
1506 salvageUnresolvedDbgValue(V: DDIMI.first, DDI);
1507
1508 erase_if(C&: DDIV, P: isMatchingDbgValue);
1509 }
1510}
1511
1512// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1513// generate the debug data structures now that we've seen its definition.
1514void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1515 SDValue Val) {
1516 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(Key: V);
1517 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1518 return;
1519
1520 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1521 for (auto &DDI : DDIV) {
1522 DebugLoc DL = DDI.getDebugLoc();
1523 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1524 DILocalVariable *Variable = DDI.getVariable();
1525 DIExpression *Expr = DDI.getExpression();
1526 assert(Variable->isValidLocationForIntrinsic(DL) &&
1527 "Expected inlined-at fields to agree");
1528 SDDbgValue *SDV;
1529 if (Val.getNode()) {
1530 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1531 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1532 // we couldn't resolve it directly when examining the DbgValue intrinsic
1533 // in the first place we should not be more successful here). Unless we
1534 // have some test case that prove this to be correct we should avoid
1535 // calling EmitFuncArgumentDbgValue here.
1536 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1537 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1538 Kind: FuncArgumentDbgValueKind::Value, N: Val)) {
1539 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1540 << printDDI(V, DDI) << "\n");
1541 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1542 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1543 // inserted after the definition of Val when emitting the instructions
1544 // after ISel. An alternative could be to teach
1545 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1546 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1547 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1548 << ValSDNodeOrder << "\n");
1549 SDV = getDbgValue(N: Val, Variable, Expr, dl: DL,
1550 DbgSDNodeOrder: std::max(a: DbgSDNodeOrder, b: ValSDNodeOrder));
1551 DAG.AddDbgValue(DB: SDV, isParameter: false);
1552 } else
1553 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1554 << printDDI(V, DDI)
1555 << " in EmitFuncArgumentDbgValue\n");
1556 } else {
1557 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1558 << "\n");
1559 auto Poison = PoisonValue::get(T: V->getType());
1560 auto SDV =
1561 DAG.getConstantDbgValue(Var: Variable, Expr, C: Poison, DL, O: DbgSDNodeOrder);
1562 DAG.AddDbgValue(DB: SDV, isParameter: false);
1563 }
1564 }
1565 DDIV.clear();
1566}
1567
1568/// If \p V is the address of a describable global, possibly displaced by a
1569/// constant, return the global and fold the displacement into location operand
1570/// \p OpIdx of \p Expr. The displacement rides along in the expression rather
1571/// than in the operand, so that it survives into a DBG_INSTR_REF.
1572static const GlobalValue *
1573getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx,
1574 const MachineFunction &MF) {
1575 const auto *C = dyn_cast<Constant>(Val: V);
1576 if (!C)
1577 return nullptr;
1578 int64_t Offset;
1579 const GlobalValue *GV = getDescribableGlobalAddress(C, Offset, MF);
1580 if (GV && Offset) {
1581 SmallVector<uint64_t, 3> Ops;
1582 DIExpression::appendOffset(Ops, Offset);
1583 Expr = DIExpression::appendOpsToArg(Expr, Ops, ArgNo: OpIdx, /*StackValue=*/false);
1584 }
1585 return GV;
1586}
1587
1588void SelectionDAGBuilder::salvageUnresolvedDbgValue(const Value *V,
1589 DanglingDebugInfo &DDI) {
1590 // TODO: For the variadic implementation, instead of only checking the fail
1591 // state of `handleDebugValue`, we need know specifically which values were
1592 // invalid, so that we attempt to salvage only those values when processing
1593 // a DIArgList.
1594 const Value *OrigV = V;
1595 DILocalVariable *Var = DDI.getVariable();
1596 DIExpression *Expr = DDI.getExpression();
1597 DebugLoc DL = DDI.getDebugLoc();
1598 unsigned SDOrder = DDI.getSDNodeOrder();
1599
1600 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1601 // that DW_OP_stack_value is desired.
1602 bool StackValue = true;
1603
1604 // handleDebugValue holds out for a register with the address of a global
1605 // that a location list could not name. With no such register forthcoming,
1606 // naming the global still beats dropping the location.
1607 auto HandleGlobalAddress = [&] {
1608 DIExpression *GVExpr = Expr;
1609 const GlobalValue *GV =
1610 getGlobalAddressDbgOperand(V, Expr&: GVExpr, OpIdx: 0, MF: DAG.getMachineFunction());
1611 if (!GV)
1612 return false;
1613 SDDbgValue *SDV = DAG.getDbgValueList(
1614 Var, Expr: GVExpr, Locs: SDDbgOperand::fromGlobalAddr(GV), /*Dependencies=*/{},
1615 /*IsIndirect=*/false, DL, O: SDOrder, /*IsVariadic=*/false);
1616 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1617 return true;
1618 };
1619
1620 // Can this Value can be encoded without any further work?
1621 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false) ||
1622 HandleGlobalAddress())
1623 return;
1624
1625 // Attempt to salvage back through as many instructions as possible. Bail if
1626 // a non-instruction is seen, such as a constant expression or global
1627 // variable. FIXME: Further work could recover those too.
1628 while (isa<Instruction>(Val: V)) {
1629 const Instruction &VAsInst = *cast<const Instruction>(Val: V);
1630 // Temporary "0", awaiting real implementation.
1631 SmallVector<uint64_t, 16> Ops;
1632 SmallVector<Value *, 4> AdditionalValues;
1633 V = salvageDebugInfoImpl(I&: const_cast<Instruction &>(VAsInst),
1634 CurrentLocOps: Expr->getNumLocationOperands(), Ops,
1635 AdditionalValues);
1636 // If we cannot salvage any further, and haven't yet found a suitable debug
1637 // expression, bail out.
1638 if (!V)
1639 break;
1640
1641 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1642 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1643 // here for variadic dbg_values, remove that condition.
1644 if (!AdditionalValues.empty())
1645 break;
1646
1647 // New value and expr now represent this debuginfo.
1648 Expr = DIExpression::appendOpsToArg(Expr, Ops, ArgNo: 0, StackValue);
1649
1650 // Some kind of simplification occurred: check whether the operand of the
1651 // salvaged debug expression can be encoded in this DAG.
1652 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false) ||
1653 HandleGlobalAddress()) {
1654 LLVM_DEBUG(
1655 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1656 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1657 return;
1658 }
1659 }
1660
1661 // This was the final opportunity to salvage this debug information, and it
1662 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1663 // any earlier variable location.
1664 assert(OrigV && "V shouldn't be null");
1665 auto *Poison = PoisonValue::get(T: OrigV->getType());
1666 auto *SDV = DAG.getConstantDbgValue(Var, Expr, C: Poison, DL, O: SDOrder);
1667 DAG.AddDbgValue(DB: SDV, isParameter: false);
1668 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1669 << printDDI(OrigV, DDI) << "\n");
1670}
1671
1672void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var,
1673 DIExpression *Expr,
1674 DebugLoc DbgLoc,
1675 unsigned Order) {
1676 Value *Poison = PoisonValue::get(T: Type::getInt1Ty(C&: *Context));
1677 DIExpression *NewExpr =
1678 const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr));
1679 handleDebugValue(Values: Poison, Var, Expr: NewExpr, DbgLoc, Order,
1680 /*IsVariadic*/ false);
1681}
1682
1683bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1684 DILocalVariable *Var,
1685 DIExpression *Expr, DebugLoc DbgLoc,
1686 unsigned Order, bool IsVariadic) {
1687 if (Values.empty())
1688 return true;
1689
1690 // Filter EntryValue locations out early.
1691 if (visitEntryValueDbgValue(Values, Variable: Var, Expr, DbgLoc))
1692 return true;
1693
1694 SmallVector<SDDbgOperand> LocationOps;
1695 SmallVector<SDNode *> Dependencies;
1696 for (const auto &[OpIdx, V] : enumerate(First&: Values)) {
1697 // Constant value.
1698 if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V) || isa<UndefValue>(Val: V) ||
1699 isa<ConstantPointerNull>(Val: V)) {
1700 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: V));
1701 continue;
1702 }
1703
1704 // Look through IntToPtr constants.
1705 if (auto *CE = dyn_cast<ConstantExpr>(Val: V))
1706 if (CE->getOpcode() == Instruction::IntToPtr) {
1707 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: CE->getOperand(i_nocapture: 0)));
1708 continue;
1709 }
1710
1711 // The address of a global is a link-time constant, and so is a constant
1712 // displacement from one. A global whose address cannot be described this
1713 // way falls through to be described by whatever materializes it instead.
1714 // So does one that a location list could not name, should the variable
1715 // need one; salvageUnresolvedDbgValue names it if nothing materializes it.
1716 if (CanDescribeGlobalAddressInLocationList)
1717 if (const GlobalValue *GV = getGlobalAddressDbgOperand(
1718 V, Expr, OpIdx, MF: DAG.getMachineFunction())) {
1719 LocationOps.emplace_back(Args: SDDbgOperand::fromGlobalAddr(GV));
1720 continue;
1721 }
1722
1723 // If the Value is a frame index, we can create a FrameIndex debug value
1724 // without relying on the DAG at all.
1725 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
1726 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
1727 if (SI != FuncInfo.StaticAllocaMap.end()) {
1728 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: SI->second));
1729 continue;
1730 }
1731 }
1732
1733 // Do not use getValue() in here; we don't want to generate code at
1734 // this point if it hasn't been done yet.
1735 SDValue N = NodeMap[V];
1736 if (!N.getNode() && isa<Argument>(Val: V)) // Check unused arguments map.
1737 N = UnusedArgNodeMap[V];
1738
1739 if (N.getNode()) {
1740 // Only emit func arg dbg value for non-variadic dbg.values for now.
1741 if (!IsVariadic &&
1742 EmitFuncArgumentDbgValue(V, Variable: Var, Expr, DL: DbgLoc,
1743 Kind: FuncArgumentDbgValueKind::Value, N))
1744 return true;
1745 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
1746 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1747 // describe stack slot locations.
1748 //
1749 // Consider "int x = 0; int *px = &x;". There are two kinds of
1750 // interesting debug values here after optimization:
1751 //
1752 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1753 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1754 //
1755 // Both describe the direct values of their associated variables.
1756 Dependencies.push_back(Elt: N.getNode());
1757 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex()));
1758 continue;
1759 }
1760 LocationOps.emplace_back(
1761 Args: SDDbgOperand::fromNode(Node: N.getNode(), ResNo: N.getResNo()));
1762 continue;
1763 }
1764
1765 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1766 // Special rules apply for the first dbg.values of parameter variables in a
1767 // function. Identify them by the fact they reference Argument Values, that
1768 // they're parameters, and they are parameters of the current function. We
1769 // need to let them dangle until they get an SDNode.
1770 bool IsParamOfFunc =
1771 isa<Argument>(Val: V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1772 if (IsParamOfFunc)
1773 return false;
1774
1775 // The value is not used in this block yet (or it would have an SDNode).
1776 // We still want the value to appear for the user if possible -- if it has
1777 // an associated VReg, we can refer to that instead.
1778 auto VMI = FuncInfo.ValueMap.find(Val: V);
1779 if (VMI != FuncInfo.ValueMap.end()) {
1780 Register Reg = VMI->second;
1781 // If this is a PHI node, it may be split up into several MI PHI nodes
1782 // (in FunctionLoweringInfo::set).
1783 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1784 V->getType(), std::nullopt);
1785 if (RFV.occupiesMultipleRegs()) {
1786 // FIXME: We could potentially support variadic dbg_values here.
1787 if (IsVariadic)
1788 return false;
1789 unsigned Offset = 0;
1790 unsigned BitsToDescribe = 0;
1791 if (auto VarSize = Var->getSizeInBits())
1792 BitsToDescribe = *VarSize;
1793 if (auto Fragment = Expr->getFragmentInfo())
1794 BitsToDescribe = Fragment->SizeInBits;
1795 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1796 // Bail out if all bits are described already.
1797 if (Offset >= BitsToDescribe)
1798 break;
1799 // TODO: handle scalable vectors.
1800 unsigned RegisterSize = RegAndSize.second;
1801 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1802 ? BitsToDescribe - Offset
1803 : RegisterSize;
1804 auto FragmentExpr = DIExpression::createFragmentExpression(
1805 Expr, OffsetInBits: Offset, SizeInBits: FragmentSize);
1806 if (!FragmentExpr)
1807 continue;
1808 SDDbgValue *SDV = DAG.getVRegDbgValue(
1809 Var, Expr: *FragmentExpr, VReg: RegAndSize.first, IsIndirect: false, DL: DbgLoc, O: Order);
1810 DAG.AddDbgValue(DB: SDV, isParameter: false);
1811 Offset += RegisterSize;
1812 }
1813 return true;
1814 }
1815 // We can use simple vreg locations for variadic dbg_values as well.
1816 LocationOps.emplace_back(Args: SDDbgOperand::fromVReg(VReg: Reg));
1817 continue;
1818 }
1819 // We failed to create a SDDbgOperand for V.
1820 return false;
1821 }
1822
1823 // We have created a SDDbgOperand for each Value in Values.
1824 assert(!LocationOps.empty());
1825 SDDbgValue *SDV =
1826 DAG.getDbgValueList(Var, Expr, Locs: LocationOps, Dependencies,
1827 /*IsIndirect=*/false, DL: DbgLoc, O: Order, IsVariadic);
1828 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1829 return true;
1830}
1831
1832void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1833 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1834 for (auto &Pair : DanglingDebugInfoMap)
1835 for (auto &DDI : Pair.second)
1836 salvageUnresolvedDbgValue(V: const_cast<Value *>(Pair.first), DDI);
1837 clearDanglingDebugInfo();
1838}
1839
1840/// getCopyFromRegs - If there was virtual register allocated for the value V
1841/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1842SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1843 auto It = FuncInfo.ValueMap.find(Val: V);
1844 SDValue Result;
1845
1846 if (It != FuncInfo.ValueMap.end()) {
1847 Register InReg = It->second;
1848
1849 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1850 DAG.getDataLayout(), InReg, Ty,
1851 std::nullopt); // This is not an ABI copy.
1852 SDValue Chain = DAG.getEntryNode();
1853 Result = RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr,
1854 V);
1855 resolveDanglingDebugInfo(V, Val: Result);
1856 }
1857
1858 return Result;
1859}
1860
1861/// getValue - Return an SDValue for the given Value.
1862SDValue SelectionDAGBuilder::getValue(const Value *V) {
1863 // If we already have an SDValue for this value, use it. It's important
1864 // to do this first, so that we don't create a CopyFromReg if we already
1865 // have a regular SDValue.
1866 SDValue &N = NodeMap[V];
1867 if (N.getNode()) return N;
1868
1869 // If there's a virtual register allocated and initialized for this
1870 // value, use it.
1871 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
1872 return copyFromReg;
1873
1874 // Otherwise create a new SDValue and remember it.
1875 SDValue Val = getValueImpl(V);
1876 NodeMap[V] = Val;
1877 resolveDanglingDebugInfo(V, Val);
1878 return Val;
1879}
1880
1881void SelectionDAGBuilder::setValueToPoison(const Value *V, const SDLoc &dl) {
1882 if (V->getType()->isVoidTy())
1883 return;
1884
1885 SmallVector<EVT, 4> ValueVTs;
1886 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
1887 Ty: V->getType(), ValueVTs);
1888 if (ValueVTs.empty())
1889 return;
1890 setValue(V, NewN: DAG.getErrorMergeValues(ResultTypes: ValueVTs, Chain: SDValue(), dl));
1891}
1892
1893/// getNonRegisterValue - Return an SDValue for the given Value, but
1894/// don't look in FuncInfo.ValueMap for a virtual register.
1895SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1896 // If we already have an SDValue for this value, use it.
1897 SDValue &N = NodeMap[V];
1898 if (N.getNode()) {
1899 if (isIntOrFPConstant(V: N)) {
1900 // Remove the debug location from the node as the node is about to be used
1901 // in a location which may differ from the original debug location. This
1902 // is relevant to Constant and ConstantFP nodes because they can appear
1903 // as constant expressions inside PHI nodes.
1904 N->setDebugLoc(DebugLoc());
1905 }
1906 return N;
1907 }
1908
1909 // Otherwise create a new SDValue and remember it.
1910 SDValue Val = getValueImpl(V);
1911 NodeMap[V] = Val;
1912 resolveDanglingDebugInfo(V, Val);
1913 return Val;
1914}
1915
1916/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1917/// Create an SDValue for the given value.
1918SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1919 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1920
1921 if (const Constant *C = dyn_cast<Constant>(Val: V)) {
1922 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: V->getType(), AllowUnknown: true);
1923
1924 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
1925 SDLoc DL = getCurSDLoc();
1926
1927 // DAG.getConstant() may attempt to legalise the vector constant which can
1928 // significantly change the combines applied to the DAG. To reduce the
1929 // divergence when enabling ConstantInt based vectors we try to construct
1930 // the DAG in the same way as shufflevector based splats. TODO: The
1931 // divergence sometimes leads to better optimisations. Ideally we should
1932 // prevent DAG.getConstant() from legalising too early but there are some
1933 // degradations preventing this.
1934 if (VT.isScalableVector())
1935 return DAG.getNode(
1936 Opcode: ISD::SPLAT_VECTOR, DL, VT,
1937 Operand: DAG.getConstant(Val: CI->getValue(), DL, VT: VT.getVectorElementType()));
1938 if (VT.isFixedLengthVector())
1939 return DAG.getSplatBuildVector(
1940 VT, DL,
1941 Op: DAG.getConstant(Val: CI->getValue(), DL, VT: VT.getVectorElementType()));
1942 return DAG.getConstant(Val: *CI, DL, VT);
1943 }
1944
1945 if (const ConstantByte *CB = dyn_cast<ConstantByte>(Val: C))
1946 return DAG.getConstant(Val: CB->getValue(), DL: getCurSDLoc(), VT);
1947
1948 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: C))
1949 return DAG.getGlobalAddress(GV, DL: getCurSDLoc(), VT);
1950
1951 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
1952 return DAG.getNode(Opcode: ISD::PtrAuthGlobalAddress, DL: getCurSDLoc(), VT,
1953 N1: getValue(V: CPA->getPointer()), N2: getValue(V: CPA->getKey()),
1954 N3: getValue(V: CPA->getAddrDiscriminator()),
1955 N4: getValue(V: CPA->getDiscriminator()));
1956 }
1957
1958 if (isa<ConstantPointerNull>(Val: C))
1959 return DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT);
1960
1961 if (match(V: C, P: m_VScale()))
1962 return DAG.getVScale(DL: getCurSDLoc(), VT, MulImm: APInt(VT.getSizeInBits(), 1));
1963
1964 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C))
1965 return DAG.getConstantFP(V: *CFP, DL: getCurSDLoc(), VT);
1966
1967 if (isa<UndefValue>(Val: C) && !V->getType()->isAggregateType())
1968 return isa<PoisonValue>(Val: C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1969
1970 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
1971 visit(Opcode: CE->getOpcode(), I: *CE);
1972 SDValue N1 = NodeMap[V];
1973 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1974 return N1;
1975 }
1976
1977 if (isa<ConstantStruct>(Val: C) || isa<ConstantArray>(Val: C)) {
1978 SmallVector<SDValue, 4> Constants;
1979 for (const Use &U : C->operands()) {
1980 SDNode *Val = getValue(V: U).getNode();
1981 // If the operand is an empty aggregate, there are no values.
1982 if (!Val) continue;
1983 // Add each leaf value from the operand to the Constants list
1984 // to form a flattened list of all the values.
1985 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1986 Constants.push_back(Elt: SDValue(Val, i));
1987 }
1988
1989 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
1990 }
1991
1992 if (const ConstantDataSequential *CDS =
1993 dyn_cast<ConstantDataSequential>(Val: C)) {
1994 SmallVector<SDValue, 4> Ops;
1995 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1996 SDNode *Val = getValue(V: CDS->getElementAsConstant(i)).getNode();
1997 // Add each leaf value from the operand to the Constants list
1998 // to form a flattened list of all the values.
1999 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
2000 Ops.push_back(Elt: SDValue(Val, i));
2001 }
2002
2003 if (isa<ArrayType>(Val: CDS->getType()))
2004 return DAG.getMergeValues(Ops, dl: getCurSDLoc());
2005 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
2006 }
2007
2008 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
2009 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
2010 "Unknown struct or array constant!");
2011
2012 SmallVector<EVT, 4> ValueVTs;
2013 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: C->getType(), ValueVTs);
2014 unsigned NumElts = ValueVTs.size();
2015 if (NumElts == 0)
2016 return SDValue(); // empty struct
2017 SmallVector<SDValue, 4> Constants(NumElts);
2018 for (unsigned i = 0; i != NumElts; ++i) {
2019 EVT EltVT = ValueVTs[i];
2020 if (isa<UndefValue>(Val: C))
2021 Constants[i] = DAG.getUNDEF(VT: EltVT);
2022 else if (EltVT.isFloatingPoint())
2023 Constants[i] = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2024 else
2025 Constants[i] = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2026 }
2027
2028 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
2029 }
2030
2031 if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C))
2032 return DAG.getBlockAddress(BA, VT);
2033
2034 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C))
2035 return getValue(V: Equiv->getGlobalValue());
2036
2037 if (const auto *NC = dyn_cast<NoCFIValue>(Val: C))
2038 return getValue(V: NC->getGlobalValue());
2039
2040 if (VT == MVT::aarch64svcount) {
2041 assert(C->isNullValue() && "Can only zero this target type!");
2042 return DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT,
2043 Operand: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::nxv16i1));
2044 }
2045
2046 if (VT.isRISCVVectorTuple()) {
2047 assert(C->isNullValue() && "Can only zero this target type!");
2048 return DAG.getNode(
2049 Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT,
2050 Operand: DAG.getNode(
2051 Opcode: ISD::SPLAT_VECTOR, DL: getCurSDLoc(),
2052 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8,
2053 NumElements: VT.getSizeInBits().getKnownMinValue() / 8, IsScalable: true),
2054 Operand: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::getIntegerVT(BitWidth: 8))));
2055 }
2056
2057 if (VT == MVT::externref || VT == MVT::funcref) {
2058 assert(C->isNullValue() && "Can only zero this target type!");
2059 // The zero value of a WebAssembly reference type is the null reference,
2060 // materialized with ref.null.
2061 Intrinsic::ID IID = VT == MVT::externref ? Intrinsic::wasm_ref_null_extern
2062 : Intrinsic::wasm_ref_null_func;
2063 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: getCurSDLoc(), VT,
2064 Operand: DAG.getTargetConstant(Val: IID, DL: getCurSDLoc(), VT: MVT::i32));
2065 }
2066
2067 VectorType *VecTy = cast<VectorType>(Val: V->getType());
2068
2069 // Now that we know the number and type of the elements, get that number of
2070 // elements into the Ops array based on what kind of constant it is.
2071 if (const ConstantVector *CV = dyn_cast<ConstantVector>(Val: C)) {
2072 SmallVector<SDValue, 16> Ops;
2073 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2074 for (unsigned i = 0; i != NumElements; ++i)
2075 Ops.push_back(Elt: getValue(V: CV->getOperand(i_nocapture: i)));
2076
2077 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
2078 }
2079
2080 if (isa<ConstantAggregateZero>(Val: C)) {
2081 EVT EltVT =
2082 TLI.getValueType(DL: DAG.getDataLayout(), Ty: VecTy->getElementType());
2083
2084 SDValue Op;
2085 if (EltVT.isFloatingPoint())
2086 Op = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2087 else
2088 Op = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2089
2090 return DAG.getSplat(VT, DL: getCurSDLoc(), Op);
2091 }
2092
2093 llvm_unreachable("Unknown vector constant");
2094 }
2095
2096 // If this is a static alloca, generate it as the frameindex instead of
2097 // computation.
2098 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
2099 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
2100 if (SI != FuncInfo.StaticAllocaMap.end())
2101 return DAG.getFrameIndex(
2102 FI: SI->second, VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: AI->getType()));
2103 }
2104
2105 // If this is an instruction which fast-isel has deferred, select it now.
2106 if (const Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
2107 Register InReg = FuncInfo.InitializeRegForValue(V: Inst);
2108 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
2109 Inst->getType(), std::nullopt);
2110 SDValue Chain = DAG.getEntryNode();
2111 return RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V);
2112 }
2113
2114 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(Val: V))
2115 return DAG.getMDNode(MD: cast<MDNode>(Val: MD->getMetadata()));
2116
2117 if (const auto *BB = dyn_cast<BasicBlock>(Val: V))
2118 return DAG.getBasicBlock(MBB: FuncInfo.getMBB(BB));
2119
2120 llvm_unreachable("Can't get register for value!");
2121}
2122
2123void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
2124 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2125 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
2126 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
2127 bool IsSEH = isAsynchronousEHPersonality(Pers);
2128 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
2129 if (IsSEH) {
2130 // For SEH, EHCont Guard needs to know that this catchpad is a target.
2131 CatchPadMBB->setIsEHContTarget(true);
2132 DAG.getMachineFunction().setHasEHContTarget(true);
2133 } else
2134 CatchPadMBB->setIsEHScopeEntry();
2135 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
2136 if (IsMSVCCXX || IsCoreCLR)
2137 CatchPadMBB->setIsEHFuncletEntry();
2138}
2139
2140void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
2141 // Update machine-CFG edge.
2142 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(BB: I.getSuccessor());
2143 FuncInfo.MBB->addSuccessor(Succ: TargetMBB);
2144
2145 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2146 bool IsSEH = isAsynchronousEHPersonality(Pers);
2147 if (IsSEH) {
2148 // If this is not a fall-through branch or optimizations are switched off,
2149 // emit the branch.
2150 if (TargetMBB != NextBlock(MBB: FuncInfo.MBB) ||
2151 TM.getOptLevel() == CodeGenOptLevel::None)
2152 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other,
2153 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: TargetMBB)));
2154 return;
2155 }
2156
2157 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2158 TargetMBB->setIsEHContTarget(true);
2159 DAG.getMachineFunction().setHasEHContTarget(true);
2160
2161 // Figure out the funclet membership for the catchret's successor.
2162 // This will be used by the FuncletLayout pass to determine how to order the
2163 // BB's.
2164 // A 'catchret' returns to the outer scope's color.
2165 Value *ParentPad = I.getCatchSwitchParentPad();
2166 const BasicBlock *SuccessorColor;
2167 if (isa<ConstantTokenNone>(Val: ParentPad))
2168 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2169 else
2170 SuccessorColor = cast<Instruction>(Val: ParentPad)->getParent();
2171 assert(SuccessorColor && "No parent funclet for catchret!");
2172 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(BB: SuccessorColor);
2173 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2174
2175 // Create the terminator node.
2176 SDValue Ret = DAG.getNode(Opcode: ISD::CATCHRET, DL: getCurSDLoc(), VT: MVT::Other,
2177 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: TargetMBB),
2178 N3: DAG.getBasicBlock(MBB: SuccessorColorMBB));
2179 DAG.setRoot(Ret);
2180}
2181
2182void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2183 // Don't emit any special code for the cleanuppad instruction. It just marks
2184 // the start of an EH scope/funclet.
2185 FuncInfo.MBB->setIsEHScopeEntry();
2186 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2187 if (Pers != EHPersonality::Wasm_CXX && Pers != EHPersonality::Wasm_D) {
2188 FuncInfo.MBB->setIsEHFuncletEntry();
2189 FuncInfo.MBB->setIsCleanupFuncletEntry();
2190 }
2191}
2192
2193/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2194/// many places it could ultimately go. In the IR, we have a single unwind
2195/// destination, but in the machine CFG, we enumerate all the possible blocks.
2196/// This function skips over imaginary basic blocks that hold catchswitch
2197/// instructions, and finds all the "real" machine
2198/// basic block destinations. As those destinations may not be successors of
2199/// EHPadBB, here we also calculate the edge probability to those destinations.
2200/// The passed-in Prob is the edge probability to EHPadBB.
2201static void findUnwindDestinations(
2202 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2203 BranchProbability Prob,
2204 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2205 &UnwindDests) {
2206 EHPersonality Personality =
2207 classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2208 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2209 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2210 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2211 bool IsWasmD = Personality == EHPersonality::Wasm_D;
2212 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2213
2214 while (EHPadBB) {
2215 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
2216 BasicBlock *NewEHPadBB = nullptr;
2217 if (isa<LandingPadInst>(Val: Pad)) {
2218 // Stop on landingpads. They are not funclets.
2219 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2220 break;
2221 } else if (isa<CleanupPadInst>(Val: Pad)) {
2222 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2223 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2224 // which always catches an exception.
2225 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2226 UnwindDests.back().first->setIsEHScopeEntry();
2227 // In Wasm, EH scopes are not funclets
2228 if (!IsWasmCXX && !IsWasmD)
2229 UnwindDests.back().first->setIsEHFuncletEntry();
2230 break;
2231 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
2232 // Add the catchpad handlers to the possible destinations.
2233 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2234 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: CatchPadBB), Args&: Prob);
2235 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2236 if (IsMSVCCXX || IsCoreCLR)
2237 UnwindDests.back().first->setIsEHFuncletEntry();
2238 if (!IsSEH)
2239 UnwindDests.back().first->setIsEHScopeEntry();
2240 }
2241 NewEHPadBB = CatchSwitch->getUnwindDest();
2242 } else {
2243 continue;
2244 }
2245
2246 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2247 if (BPI && NewEHPadBB)
2248 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
2249 EHPadBB = NewEHPadBB;
2250 }
2251}
2252
2253void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2254 // Update successor info.
2255 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2256 auto UnwindDest = I.getUnwindDest();
2257 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2258 BranchProbability UnwindDestProb =
2259 (BPI && UnwindDest)
2260 ? BPI->getEdgeProbability(Src: FuncInfo.MBB->getBasicBlock(), Dst: UnwindDest)
2261 : BranchProbability::getZero();
2262 findUnwindDestinations(FuncInfo, EHPadBB: UnwindDest, Prob: UnwindDestProb, UnwindDests);
2263 for (auto &UnwindDest : UnwindDests) {
2264 UnwindDest.first->setIsEHPad();
2265 addSuccessorWithProb(Src: FuncInfo.MBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
2266 }
2267 FuncInfo.MBB->normalizeSuccProbs();
2268
2269 // Create the terminator node.
2270 MachineBasicBlock *CleanupPadMBB =
2271 FuncInfo.getMBB(BB: I.getCleanupPad()->getParent());
2272 SDValue Ret = DAG.getNode(Opcode: ISD::CLEANUPRET, DL: getCurSDLoc(), VT: MVT::Other,
2273 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: CleanupPadMBB));
2274 DAG.setRoot(Ret);
2275}
2276
2277void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2278 report_fatal_error(reason: "visitCatchSwitch not yet implemented!");
2279}
2280
2281void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2282 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2283 auto &DL = DAG.getDataLayout();
2284 SDValue Chain = getControlRoot();
2285 SmallVector<ISD::OutputArg, 8> Outs;
2286 SmallVector<SDValue, 8> OutVals;
2287
2288 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2289 // lower
2290 //
2291 // %val = call <ty> @llvm.experimental.deoptimize()
2292 // ret <ty> %val
2293 //
2294 // differently.
2295 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2296 LowerDeoptimizingReturn();
2297 return;
2298 }
2299
2300 if (!FuncInfo.CanLowerReturn) {
2301 Register DemoteReg = FuncInfo.DemoteRegister;
2302
2303 // Emit a store of the return value through the virtual register.
2304 // Leave Outs empty so that LowerReturn won't try to load return
2305 // registers the usual way.
2306 MVT PtrValueVT = TLI.getPointerTy(DL, AS: DL.getAllocaAddrSpace());
2307 SDValue RetPtr =
2308 DAG.getCopyFromReg(Chain, dl: getCurSDLoc(), Reg: DemoteReg, VT: PtrValueVT);
2309 Type *RetTy = I.getOperand(i_nocapture: 0)->getType();
2310 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
2311 RetPtr =
2312 TLI.annotateStackObjectPointer(Ptr: RetPtr, DAG, DL: getCurSDLoc(), Alignment: BaseAlign);
2313 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2314
2315 SmallVector<EVT, 4> ValueVTs, MemVTs;
2316 SmallVector<uint64_t, 4> Offsets;
2317 ComputeValueVTs(TLI, DL, Ty: RetTy, ValueVTs, MemVTs: &MemVTs, FixedOffsets: &Offsets, StartingOffset: 0);
2318 unsigned NumValues = ValueVTs.size();
2319
2320 SmallVector<SDValue, 4> Chains(NumValues);
2321 for (unsigned i = 0; i != NumValues; ++i) {
2322 // An aggregate return value cannot wrap around the address space, so
2323 // offsets to its parts don't wrap either.
2324 SDValue Ptr = DAG.getObjectPtrOffset(SL: getCurSDLoc(), Ptr: RetPtr,
2325 Offset: TypeSize::getFixed(ExactSize: Offsets[i]));
2326
2327 SDValue Val = RetOp.getValue(R: RetOp.getResNo() + i);
2328 if (MemVTs[i] != ValueVTs[i])
2329 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: getCurSDLoc(), VT: MemVTs[i]);
2330 Chains[i] = DAG.getStore(
2331 Chain, dl: getCurSDLoc(), Val,
2332 // FIXME: better loc info would be nice.
2333 Ptr, PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
2334 Alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]));
2335 }
2336
2337 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: getCurSDLoc(),
2338 VT: MVT::Other, Ops: Chains);
2339 } else if (I.getNumOperands() != 0) {
2340 SmallVector<Type *, 4> Types;
2341 ComputeValueTypes(DL, Ty: I.getOperand(i_nocapture: 0)->getType(), Types);
2342 unsigned NumValues = Types.size();
2343 if (NumValues) {
2344 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2345
2346 const Function *F = I.getParent()->getParent();
2347
2348 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2349 Ty: I.getOperand(i_nocapture: 0)->getType(), CallConv: F->getCallingConv(),
2350 /*IsVarArg*/ isVarArg: false, DL);
2351
2352 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2353 if (F->getAttributes().hasRetAttr(Kind: Attribute::SExt))
2354 ExtendKind = ISD::SIGN_EXTEND;
2355 else if (F->getAttributes().hasRetAttr(Kind: Attribute::ZExt))
2356 ExtendKind = ISD::ZERO_EXTEND;
2357
2358 LLVMContext &Context = F->getContext();
2359 bool RetInReg = F->getAttributes().hasRetAttr(Kind: Attribute::InReg);
2360
2361 for (unsigned j = 0; j != NumValues; ++j) {
2362 EVT VT = TLI.getValueType(DL, Ty: Types[j]);
2363
2364 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2365 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2366
2367 CallingConv::ID CC = F->getCallingConv();
2368
2369 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2370 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2371 SmallVector<SDValue, 4> Parts(NumParts);
2372 getCopyToParts(DAG, DL: getCurSDLoc(),
2373 Val: SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2374 Parts: &Parts[0], NumParts, PartVT, V: &I, CallConv: CC, ExtendKind);
2375
2376 // 'inreg' on function refers to return value
2377 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2378 if (RetInReg)
2379 Flags.setInReg();
2380
2381 if (I.getOperand(i_nocapture: 0)->getType()->isPointerTy()) {
2382 Flags.setPointer();
2383 Flags.setPointerAddrSpace(
2384 cast<PointerType>(Val: I.getOperand(i_nocapture: 0)->getType())->getAddressSpace());
2385 }
2386
2387 if (NeedsRegBlock) {
2388 Flags.setInConsecutiveRegs();
2389 if (j == NumValues - 1)
2390 Flags.setInConsecutiveRegsLast();
2391 }
2392
2393 // Propagate extension type if any
2394 if (ExtendKind == ISD::SIGN_EXTEND)
2395 Flags.setSExt();
2396 else if (ExtendKind == ISD::ZERO_EXTEND)
2397 Flags.setZExt();
2398 else if (F->getAttributes().hasRetAttr(Kind: Attribute::NoExt))
2399 Flags.setNoExt();
2400
2401 for (unsigned i = 0; i < NumParts; ++i) {
2402 Outs.push_back(Elt: ISD::OutputArg(Flags,
2403 Parts[i].getValueType().getSimpleVT(),
2404 VT, Types[j], 0, 0));
2405 OutVals.push_back(Elt: Parts[i]);
2406 }
2407 }
2408 }
2409 }
2410
2411 // Push in swifterror virtual register as the last element of Outs. This makes
2412 // sure swifterror virtual register will be returned in the swifterror
2413 // physical register.
2414 const Function *F = I.getParent()->getParent();
2415 if (TLI.supportSwiftError() &&
2416 F->getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError)) {
2417 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2418 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2419 Flags.setSwiftError();
2420 Outs.push_back(Elt: ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
2421 /*argvt=*/EVT(TLI.getPointerTy(DL)),
2422 PointerType::getUnqual(C&: *DAG.getContext()),
2423 /*origidx=*/1, /*partOffs=*/0));
2424 // Create SDNode for the swifterror virtual register.
2425 OutVals.push_back(
2426 Elt: DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(
2427 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2428 VT: EVT(TLI.getPointerTy(DL))));
2429 }
2430
2431 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2432 CallingConv::ID CallConv =
2433 DAG.getMachineFunction().getFunction().getCallingConv();
2434 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2435 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2436
2437 // Verify that the target's LowerReturn behaved as expected.
2438 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2439 "LowerReturn didn't return a valid chain!");
2440
2441 // Update the DAG with the new chain value resulting from return lowering.
2442 DAG.setRoot(Chain);
2443}
2444
2445/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2446/// created for it, emit nodes to copy the value into the virtual
2447/// registers.
2448void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2449 // Skip empty types
2450 if (V->getType()->isEmptyTy())
2451 return;
2452
2453 auto VMI = FuncInfo.ValueMap.find(Val: V);
2454 if (VMI != FuncInfo.ValueMap.end()) {
2455 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2456 "Unused value assigned virtual registers!");
2457 CopyValueToVirtualRegister(V, Reg: VMI->second);
2458 }
2459}
2460
2461/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2462/// the current basic block, add it to ValueMap now so that we'll get a
2463/// CopyTo/FromReg.
2464void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2465 // No need to export constants.
2466 if (!isa<Instruction>(Val: V) && !isa<Argument>(Val: V)) return;
2467
2468 // Already exported?
2469 if (FuncInfo.isExportedInst(V)) return;
2470
2471 Register Reg = FuncInfo.InitializeRegForValue(V);
2472 CopyValueToVirtualRegister(V, Reg);
2473}
2474
2475bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2476 const BasicBlock *FromBB) {
2477 // The operands of the setcc have to be in this block. We don't know
2478 // how to export them from some other block.
2479 if (const Instruction *VI = dyn_cast<Instruction>(Val: V)) {
2480 // Can export from current BB.
2481 if (VI->getParent() == FromBB)
2482 return true;
2483
2484 // Is already exported, noop.
2485 return FuncInfo.isExportedInst(V);
2486 }
2487
2488 // If this is an argument, we can export it if the BB is the entry block or
2489 // if it is already exported.
2490 if (isa<Argument>(Val: V)) {
2491 if (FromBB->isEntryBlock())
2492 return true;
2493
2494 // Otherwise, can only export this if it is already exported.
2495 return FuncInfo.isExportedInst(V);
2496 }
2497
2498 // Otherwise, constants can always be exported.
2499 return true;
2500}
2501
2502/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2503BranchProbability
2504SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2505 const MachineBasicBlock *Dst) const {
2506 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2507 const BasicBlock *SrcBB = Src->getBasicBlock();
2508 const BasicBlock *DstBB = Dst->getBasicBlock();
2509 if (!BPI) {
2510 // If BPI is not available, set the default probability as 1 / N, where N is
2511 // the number of successors.
2512 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
2513 return BranchProbability(1, SuccSize);
2514 }
2515 return BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
2516}
2517
2518void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2519 MachineBasicBlock *Dst,
2520 BranchProbability Prob) {
2521 if (!FuncInfo.BPI)
2522 Src->addSuccessorWithoutProb(Succ: Dst);
2523 else {
2524 if (Prob.isUnknown())
2525 Prob = getEdgeProbability(Src, Dst);
2526 Src->addSuccessor(Succ: Dst, Prob);
2527 }
2528}
2529
2530static bool InBlock(const Value *V, const BasicBlock *BB) {
2531 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
2532 return I->getParent() == BB;
2533 return true;
2534}
2535
2536/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2537/// This function emits a branch and is used at the leaves of an OR or an
2538/// AND operator tree.
2539void
2540SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2541 MachineBasicBlock *TBB,
2542 MachineBasicBlock *FBB,
2543 MachineBasicBlock *CurBB,
2544 MachineBasicBlock *SwitchBB,
2545 BranchProbability TProb,
2546 BranchProbability FProb,
2547 bool InvertCond) {
2548 const BasicBlock *BB = CurBB->getBasicBlock();
2549
2550 // If the leaf of the tree is a comparison, merge the condition into
2551 // the caseblock.
2552 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
2553 // The operands of the cmp have to be in this block. We don't know
2554 // how to export them from some other block. If this is the first block
2555 // of the sequence, no exporting is needed.
2556 if (CurBB == SwitchBB ||
2557 (isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 0), FromBB: BB) &&
2558 isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 1), FromBB: BB))) {
2559 ISD::CondCode Condition;
2560 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
2561 ICmpInst::Predicate Pred =
2562 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2563 Condition = getICmpCondCode(Pred);
2564 } else {
2565 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
2566 FCmpInst::Predicate Pred =
2567 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2568 Condition = getFCmpCondCode(Pred);
2569 if (FC->hasNoNaNs() ||
2570 (isKnownNeverNaN(V: FC->getOperand(i_nocapture: 0),
2571 SQ: SimplifyQuery(DAG.getDataLayout(), FC)) &&
2572 isKnownNeverNaN(V: FC->getOperand(i_nocapture: 1),
2573 SQ: SimplifyQuery(DAG.getDataLayout(), FC))))
2574 Condition = getFCmpCodeWithoutNaN(CC: Condition);
2575 }
2576
2577 CaseBlock CB(Condition, BOp->getOperand(i_nocapture: 0), BOp->getOperand(i_nocapture: 1), nullptr,
2578 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2579 SL->SwitchCases.push_back(x: CB);
2580 return;
2581 }
2582 }
2583
2584 // Create a CaseBlock record representing this branch.
2585 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2586 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(Context&: *DAG.getContext()),
2587 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2588 SL->SwitchCases.push_back(x: CB);
2589}
2590
2591// Collect dependencies on V recursively. This is used for the cost analysis in
2592// `shouldKeepJumpConditionsTogether`.
2593static bool collectInstructionDeps(
2594 SmallMapVector<const Instruction *, bool, 8> *Deps, const Value *V,
2595 SmallMapVector<const Instruction *, bool, 8> *Necessary = nullptr,
2596 unsigned Depth = 0) {
2597 // Return false if we have an incomplete count.
2598 if (Depth >= SelectionDAG::MaxRecursionDepth)
2599 return false;
2600
2601 auto *I = dyn_cast<Instruction>(Val: V);
2602 if (I == nullptr)
2603 return true;
2604
2605 if (Necessary != nullptr) {
2606 // This instruction is necessary for the other side of the condition so
2607 // don't count it.
2608 if (Necessary->contains(Key: I))
2609 return true;
2610 }
2611
2612 // Already added this dep.
2613 if (!Deps->try_emplace(Key: I, Args: false).second)
2614 return true;
2615
2616 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2617 if (!collectInstructionDeps(Deps, V: I->getOperand(i: OpIdx), Necessary,
2618 Depth: Depth + 1))
2619 return false;
2620 return true;
2621}
2622
2623bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether(
2624 const FunctionLoweringInfo &FuncInfo, const CondBrInst &I,
2625 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2626 TargetLoweringBase::CondMergingParams Params) const {
2627 if (Params.BaseCost < 0)
2628 return false;
2629
2630 // Baseline cost.
2631 InstructionCost CostThresh = Params.BaseCost;
2632
2633 BranchProbabilityInfo *BPI = nullptr;
2634 if (Params.LikelyBias || Params.UnlikelyBias)
2635 BPI = FuncInfo.BPI;
2636 if (BPI != nullptr) {
2637 // See if we are either likely to get an early out or compute both lhs/rhs
2638 // of the condition.
2639 BasicBlock *IfTrue = I.getSuccessor(i: 0);
2640 BasicBlock *IfFalse = I.getSuccessor(i: 1);
2641 if (ProfcheckDisableMetadataFixes)
2642 std::swap(a&: IfTrue, b&: IfFalse);
2643
2644 std::optional<bool> Likely;
2645 if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfTrue))
2646 Likely = true;
2647 else if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfFalse))
2648 Likely = false;
2649
2650 if (Likely) {
2651 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2652 // Its likely we will have to compute both lhs and rhs of condition
2653 CostThresh += Params.LikelyBias;
2654 else {
2655 if (Params.UnlikelyBias < 0)
2656 return false;
2657 // Its likely we will get an early out.
2658 CostThresh -= Params.UnlikelyBias;
2659 }
2660 }
2661 }
2662
2663 if (CostThresh <= 0)
2664 return false;
2665
2666 // Collect "all" instructions that lhs condition is dependent on.
2667 // Use map for stable iteration (to avoid non-determanism of iteration of
2668 // SmallPtrSet). The `bool` value is just a dummy.
2669 SmallMapVector<const Instruction *, bool, 8> LhsDeps, RhsDeps;
2670 collectInstructionDeps(Deps: &LhsDeps, V: Lhs);
2671 // Collect "all" instructions that rhs condition is dependent on AND are
2672 // dependencies of lhs. This gives us an estimate on which instructions we
2673 // stand to save by splitting the condition.
2674 if (!collectInstructionDeps(Deps: &RhsDeps, V: Rhs, Necessary: &LhsDeps))
2675 return false;
2676 // Add the compare instruction itself unless its a dependency on the LHS.
2677 if (const auto *RhsI = dyn_cast<Instruction>(Val: Rhs))
2678 if (!LhsDeps.contains(Key: RhsI))
2679 RhsDeps.try_emplace(Key: RhsI, Args: false);
2680
2681 InstructionCost CostOfIncluding = 0;
2682 // See if this instruction will need to computed independently of whether RHS
2683 // is.
2684 Value *BrCond = I.getCondition();
2685 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2686 for (const auto *U : Ins->users()) {
2687 // If user is independent of RHS calculation we don't need to count it.
2688 if (auto *UIns = dyn_cast<Instruction>(Val: U))
2689 if (UIns != BrCond && !RhsDeps.contains(Key: UIns))
2690 return false;
2691 }
2692 return true;
2693 };
2694
2695 // Prune instructions from RHS Deps that are dependencies of unrelated
2696 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2697 // arbitrary and just meant to cap the how much time we spend in the pruning
2698 // loop. Its highly unlikely to come into affect.
2699 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2700 // Stop after a certain point. No incorrectness from including too many
2701 // instructions.
2702 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2703 const Instruction *ToDrop = nullptr;
2704 for (const auto &InsPair : RhsDeps) {
2705 if (!ShouldCountInsn(InsPair.first)) {
2706 ToDrop = InsPair.first;
2707 break;
2708 }
2709 }
2710 if (ToDrop == nullptr)
2711 break;
2712 RhsDeps.erase(Key: ToDrop);
2713 }
2714
2715 for (const auto &InsPair : RhsDeps) {
2716 // Finally accumulate latency that we can only attribute to computing the
2717 // RHS condition. Use latency because we are essentially trying to calculate
2718 // the cost of the dependency chain.
2719 // Possible TODO: We could try to estimate ILP and make this more precise.
2720 CostOfIncluding += TTI->getInstructionCost(
2721 U: InsPair.first, CostKind: TargetTransformInfo::TCK_Latency);
2722
2723 if (CostOfIncluding > CostThresh)
2724 return false;
2725 }
2726 return true;
2727}
2728
2729void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2730 MachineBasicBlock *TBB,
2731 MachineBasicBlock *FBB,
2732 MachineBasicBlock *CurBB,
2733 MachineBasicBlock *SwitchBB,
2734 Instruction::BinaryOps Opc,
2735 BranchProbability TProb,
2736 BranchProbability FProb,
2737 bool InvertCond) {
2738 // Skip over not part of the tree and remember to invert op and operands at
2739 // next level.
2740 Value *NotCond;
2741 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
2742 InBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
2743 FindMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2744 InvertCond: !InvertCond);
2745 return;
2746 }
2747
2748 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
2749 const Value *BOpOp0, *BOpOp1;
2750 // Compute the effective opcode for Cond, taking into account whether it needs
2751 // to be inverted, e.g.
2752 // and (not (or A, B)), C
2753 // gets lowered as
2754 // and (and (not A, not B), C)
2755 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2756 if (BOp) {
2757 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2758 ? Instruction::And
2759 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2760 ? Instruction::Or
2761 : (Instruction::BinaryOps)0);
2762 if (InvertCond) {
2763 if (BOpc == Instruction::And)
2764 BOpc = Instruction::Or;
2765 else if (BOpc == Instruction::Or)
2766 BOpc = Instruction::And;
2767 }
2768 }
2769
2770 // If this node is not part of the or/and tree, emit it as a branch.
2771 // Note that all nodes in the tree should have same opcode.
2772 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2773 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2774 !InBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
2775 !InBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
2776 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2777 TProb, FProb, InvertCond);
2778 return;
2779 }
2780
2781 // Create TmpBB after CurBB.
2782 MachineFunction::iterator BBI(CurBB);
2783 MachineFunction &MF = DAG.getMachineFunction();
2784 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
2785 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
2786
2787 if (Opc == Instruction::Or) {
2788 // Codegen X | Y as:
2789 // BB1:
2790 // jmp_if_X TBB
2791 // jmp TmpBB
2792 // TmpBB:
2793 // jmp_if_Y TBB
2794 // jmp FBB
2795 //
2796
2797 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2798 // The requirement is that
2799 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2800 // = TrueProb for original BB.
2801 // Assuming the original probabilities are A and B, one choice is to set
2802 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2803 // A/(1+B) and 2B/(1+B). This choice assumes that
2804 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2805 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2806 // TmpBB, but the math is more complicated.
2807
2808 auto NewTrueProb = TProb / 2;
2809 auto NewFalseProb = TProb / 2 + FProb;
2810 // Emit the LHS condition.
2811 FindMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2812 FProb: NewFalseProb, InvertCond);
2813
2814 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2815 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2816 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2817 // Emit the RHS condition into TmpBB.
2818 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2819 FProb: Probs[1], InvertCond);
2820 } else {
2821 assert(Opc == Instruction::And && "Unknown merge op!");
2822 // Codegen X & Y as:
2823 // BB1:
2824 // jmp_if_X TmpBB
2825 // jmp FBB
2826 // TmpBB:
2827 // jmp_if_Y TBB
2828 // jmp FBB
2829 //
2830 // This requires creation of TmpBB after CurBB.
2831
2832 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2833 // The requirement is that
2834 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2835 // = FalseProb for original BB.
2836 // Assuming the original probabilities are A and B, one choice is to set
2837 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2838 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2839 // TrueProb for BB1 * FalseProb for TmpBB.
2840
2841 auto NewTrueProb = TProb + FProb / 2;
2842 auto NewFalseProb = FProb / 2;
2843 // Emit the LHS condition.
2844 FindMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2845 FProb: NewFalseProb, InvertCond);
2846
2847 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2848 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2849 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2850 // Emit the RHS condition into TmpBB.
2851 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2852 FProb: Probs[1], InvertCond);
2853 }
2854}
2855
2856/// If the set of cases should be emitted as a series of branches, return true.
2857/// If we should emit this as a bunch of and/or'd together conditions, return
2858/// false.
2859bool
2860SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2861 if (Cases.size() != 2) return true;
2862
2863 // If this is two comparisons of the same values or'd or and'd together, they
2864 // will get folded into a single comparison, so don't emit two blocks.
2865 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2866 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2867 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2868 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2869 return false;
2870 }
2871
2872 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2873 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2874 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2875 Cases[0].CC == Cases[1].CC &&
2876 isa<Constant>(Val: Cases[0].CmpRHS) &&
2877 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
2878 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2879 return false;
2880 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2881 return false;
2882 }
2883
2884 return true;
2885}
2886
2887void SelectionDAGBuilder::visitUncondBr(const UncondBrInst &I) {
2888 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2889
2890 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2891
2892 // Update machine-CFG edges.
2893 BrMBB->addSuccessor(Succ: Succ0MBB);
2894
2895 // If this is not a fall-through branch or optimizations are switched off,
2896 // emit the branch.
2897 if (Succ0MBB != NextBlock(MBB: BrMBB) ||
2898 TM.getOptLevel() == CodeGenOptLevel::None) {
2899 auto Br = DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other, N1: getControlRoot(),
2900 N2: DAG.getBasicBlock(MBB: Succ0MBB));
2901 setValue(V: &I, NewN: Br);
2902 DAG.setRoot(Br);
2903 }
2904}
2905
2906void SelectionDAGBuilder::visitCondBr(const CondBrInst &I) {
2907 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2908
2909 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2910
2911 // If this condition is one of the special cases we handle, do special stuff
2912 // now.
2913 const Value *CondVal = I.getCondition();
2914 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 1));
2915
2916 // If this is a series of conditions that are or'd or and'd together, emit
2917 // this as a sequence of branches instead of setcc's with and/or operations.
2918 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2919 // unpredictable branches, and vector extracts because those jumps are likely
2920 // expensive for any target), this should improve performance.
2921 // For example, instead of something like:
2922 // cmp A, B
2923 // C = seteq
2924 // cmp D, E
2925 // F = setle
2926 // or C, F
2927 // jnz foo
2928 // Emit:
2929 // cmp A, B
2930 // je foo
2931 // cmp D, E
2932 // jle foo
2933 bool IsUnpredictable = I.hasMetadata(KindID: LLVMContext::MD_unpredictable);
2934 const Instruction *BOp = dyn_cast<Instruction>(Val: CondVal);
2935 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2936 BOp->hasOneUse() && !IsUnpredictable) {
2937 Value *Vec;
2938 const Value *BOp0, *BOp1;
2939 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2940 if (match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2941 Opcode = Instruction::And;
2942 else if (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2943 Opcode = Instruction::Or;
2944
2945 if (Opcode &&
2946 !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
2947 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value()))) &&
2948 !shouldKeepJumpConditionsTogether(
2949 FuncInfo, I, Opc: Opcode, Lhs: BOp0, Rhs: BOp1,
2950 Params: DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2951 Opcode, BOp0, BOp1, FuncInfo.Fn))) {
2952 FindMergedConditions(Cond: BOp, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: BrMBB, SwitchBB: BrMBB, Opc: Opcode,
2953 TProb: getEdgeProbability(Src: BrMBB, Dst: Succ0MBB),
2954 FProb: getEdgeProbability(Src: BrMBB, Dst: Succ1MBB),
2955 /*InvertCond=*/false);
2956 // If the compares in later blocks need to use values not currently
2957 // exported from this block, export them now. This block should always
2958 // be the first entry.
2959 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2960
2961 // Allow some cases to be rejected.
2962 if (ShouldEmitAsBranches(Cases: SL->SwitchCases)) {
2963 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2964 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpLHS);
2965 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpRHS);
2966 }
2967
2968 // Emit the branch for this block.
2969 visitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: BrMBB);
2970 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
2971 return;
2972 }
2973
2974 // Okay, we decided not to do this, remove any inserted MBB's and clear
2975 // SwitchCases.
2976 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2977 FuncInfo.MF->erase(MBBI: SL->SwitchCases[i].ThisBB);
2978
2979 SL->SwitchCases.clear();
2980 }
2981 }
2982
2983 // Create a CaseBlock record representing this branch.
2984 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(Context&: *DAG.getContext()),
2985 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2986 BranchProbability::getUnknown(), BranchProbability::getUnknown(),
2987 IsUnpredictable);
2988
2989 // Use visitSwitchCase to actually insert the fast branch sequence for this
2990 // cond branch.
2991 visitSwitchCase(CB, SwitchBB: BrMBB);
2992}
2993
2994/// visitSwitchCase - Emits the necessary code to represent a single node in
2995/// the binary search tree resulting from lowering a switch instruction.
2996void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2997 MachineBasicBlock *SwitchBB) {
2998 SDValue Cond;
2999 SDValue CondLHS = getValue(V: CB.CmpLHS);
3000 SDLoc dl = CB.DL;
3001
3002 if (CB.CC == ISD::SETTRUE) {
3003 // Branch or fall through to TrueBB.
3004 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
3005 SwitchBB->normalizeSuccProbs();
3006 if (CB.TrueBB != NextBlock(MBB: SwitchBB)) {
3007 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: getControlRoot(),
3008 N2: DAG.getBasicBlock(MBB: CB.TrueBB)));
3009 }
3010 return;
3011 }
3012
3013 auto &TLI = DAG.getTargetLoweringInfo();
3014 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: CB.CmpLHS->getType());
3015
3016 // Build the setcc now.
3017 if (!CB.CmpMHS) {
3018 // Fold "(X == true)" to X and "(X == false)" to !X to
3019 // handle common cases produced by branch lowering.
3020 if (CB.CmpRHS == ConstantInt::getTrue(Context&: *DAG.getContext()) &&
3021 CB.CC == ISD::SETEQ)
3022 Cond = CondLHS;
3023 else if (CB.CmpRHS == ConstantInt::getFalse(Context&: *DAG.getContext()) &&
3024 CB.CC == ISD::SETEQ) {
3025 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: CondLHS.getValueType());
3026 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CondLHS.getValueType(), N1: CondLHS, N2: True);
3027 } else {
3028 SDValue CondRHS = getValue(V: CB.CmpRHS);
3029
3030 // If a pointer's DAG type is larger than its memory type then the DAG
3031 // values are zero-extended. This breaks signed comparisons so truncate
3032 // back to the underlying type before doing the compare.
3033 if (CondLHS.getValueType() != MemVT) {
3034 CondLHS = DAG.getPtrExtOrTrunc(Op: CondLHS, DL: getCurSDLoc(), VT: MemVT);
3035 CondRHS = DAG.getPtrExtOrTrunc(Op: CondRHS, DL: getCurSDLoc(), VT: MemVT);
3036 }
3037 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: CondLHS, RHS: CondRHS, Cond: CB.CC);
3038 }
3039 } else {
3040 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
3041
3042 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
3043 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
3044
3045 SDValue CmpOp = getValue(V: CB.CmpMHS);
3046 EVT VT = CmpOp.getValueType();
3047
3048 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
3049 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: CmpOp, RHS: DAG.getConstant(Val: High, DL: dl, VT),
3050 Cond: ISD::SETLE);
3051 } else {
3052 SDValue SUB = DAG.getNode(Opcode: ISD::SUB, DL: dl,
3053 VT, N1: CmpOp, N2: DAG.getConstant(Val: Low, DL: dl, VT));
3054 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: SUB,
3055 RHS: DAG.getConstant(Val: High-Low, DL: dl, VT), Cond: ISD::SETULE);
3056 }
3057 }
3058
3059 // Update successor info
3060 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
3061 // TrueBB and FalseBB are always different unless the incoming IR is
3062 // degenerate. This only happens when running llc on weird IR.
3063 if (CB.TrueBB != CB.FalseBB)
3064 addSuccessorWithProb(Src: SwitchBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
3065 SwitchBB->normalizeSuccProbs();
3066
3067 // If the lhs block is the next block, invert the condition so that we can
3068 // fall through to the lhs instead of the rhs block.
3069 if (CB.TrueBB == NextBlock(MBB: SwitchBB)) {
3070 std::swap(a&: CB.TrueBB, b&: CB.FalseBB);
3071 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: Cond.getValueType());
3072 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: Cond.getValueType(), N1: Cond, N2: True);
3073 }
3074
3075 SDNodeFlags Flags;
3076 Flags.setUnpredictable(CB.IsUnpredictable);
3077 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: getControlRoot(),
3078 N2: Cond, N3: DAG.getBasicBlock(MBB: CB.TrueBB), Flags);
3079
3080 setValue(V: CurInst, NewN: BrCond);
3081
3082 // Insert the false branch. Do this even if it's a fall through branch,
3083 // this makes it easier to do DAG optimizations which require inverting
3084 // the branch condition.
3085 BrCond = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrCond,
3086 N2: DAG.getBasicBlock(MBB: CB.FalseBB));
3087
3088 DAG.setRoot(BrCond);
3089}
3090
3091/// visitJumpTable - Emit JumpTable node in the current MBB
3092void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
3093 // Emit the code for the jump table
3094 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3095 assert(JT.Reg && "Should lower JT Header first!");
3096 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DL: DAG.getDataLayout());
3097 SDValue Index = DAG.getCopyFromReg(Chain: getControlRoot(), dl: *JT.SL, Reg: JT.Reg, VT: PTy);
3098 SDValue Table = DAG.getJumpTable(JTI: JT.JTI, VT: PTy);
3099 SDValue BrJumpTable = DAG.getNode(Opcode: ISD::BR_JT, DL: *JT.SL, VT: MVT::Other,
3100 N1: Index.getValue(R: 1), N2: Table, N3: Index);
3101 DAG.setRoot(BrJumpTable);
3102}
3103
3104/// visitJumpTableHeader - This function emits necessary code to produce index
3105/// in the JumpTable from switch case.
3106void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
3107 JumpTableHeader &JTH,
3108 MachineBasicBlock *SwitchBB) {
3109 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3110 const SDLoc &dl = *JT.SL;
3111
3112 // Subtract the lowest switch case value from the value being switched on.
3113 SDValue SwitchOp = getValue(V: JTH.SValue);
3114 EVT VT = SwitchOp.getValueType();
3115 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp,
3116 N2: DAG.getConstant(Val: JTH.First, DL: dl, VT));
3117
3118 // The SDNode we just created, which holds the value being switched on minus
3119 // the smallest case value, needs to be copied to a virtual register so it
3120 // can be used as an index into the jump table in a subsequent basic block.
3121 // This value may be smaller or larger than the target's pointer type, and
3122 // therefore require extension or truncating.
3123 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3124 SwitchOp =
3125 DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
3126
3127 Register JumpTableReg =
3128 FuncInfo.CreateReg(VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
3129 SDValue CopyTo =
3130 DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: JumpTableReg, N: SwitchOp);
3131 JT.Reg = JumpTableReg;
3132
3133 if (!JTH.FallthroughUnreachable) {
3134 // Emit the range check for the jump table, and branch to the default block
3135 // for the switch statement if the value being switched on exceeds the
3136 // largest case in the switch.
3137 SDValue CMP = DAG.getSetCC(
3138 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3139 VT: Sub.getValueType()),
3140 LHS: Sub, RHS: DAG.getConstant(Val: JTH.Last - JTH.First, DL: dl, VT), Cond: ISD::SETUGT);
3141
3142 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl,
3143 VT: MVT::Other, N1: CopyTo, N2: CMP,
3144 N3: DAG.getBasicBlock(MBB: JT.Default));
3145
3146 // Avoid emitting unnecessary branches to the next block.
3147 if (JT.MBB != NextBlock(MBB: SwitchBB))
3148 BrCond = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrCond,
3149 N2: DAG.getBasicBlock(MBB: JT.MBB));
3150
3151 DAG.setRoot(BrCond);
3152 } else {
3153 // Avoid emitting unnecessary branches to the next block.
3154 if (JT.MBB != NextBlock(MBB: SwitchBB))
3155 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: CopyTo,
3156 N2: DAG.getBasicBlock(MBB: JT.MBB)));
3157 else
3158 DAG.setRoot(CopyTo);
3159 }
3160}
3161
3162/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3163/// variable if there exists one.
3164static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
3165 SDValue &Chain) {
3166 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3167 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
3168 EVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout());
3169 MachineFunction &MF = DAG.getMachineFunction();
3170 Value *Global =
3171 TLI.getSDagStackGuard(M: *MF.getFunction().getParent(), Libcalls: DAG.getLibcalls());
3172 MachineSDNode *Node =
3173 DAG.getMachineNode(Opcode: TargetOpcode::LOAD_STACK_GUARD, dl: DL, VT: PtrTy, Op1: Chain);
3174 if (Global) {
3175 MachinePointerInfo MPInfo(Global);
3176 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
3177 MachineMemOperand::MODereferenceable;
3178 MachineMemOperand *MemRef = MF.getMachineMemOperand(
3179 PtrInfo: MPInfo, F: Flags, Size: PtrTy.getSizeInBits() / 8, BaseAlignment: DAG.getEVTAlign(MemoryVT: PtrTy));
3180 DAG.setNodeMemRefs(N: Node, NewMemRefs: {MemRef});
3181 }
3182 if (PtrTy != PtrMemTy)
3183 return DAG.getPtrExtOrTrunc(Op: SDValue(Node, 0), DL, VT: PtrMemTy);
3184 return SDValue(Node, 0);
3185}
3186
3187/// Codegen a new tail for a stack protector check ParentMBB which has had its
3188/// tail spliced into a stack protector check success bb.
3189///
3190/// For a high level explanation of how this fits into the stack protector
3191/// generation see the comment on the declaration of class
3192/// StackProtectorDescriptor.
3193void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
3194 MachineBasicBlock *ParentBB) {
3195
3196 // First create the loads to the guard/stack slot for the comparison.
3197 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3198 auto &DL = DAG.getDataLayout();
3199 EVT PtrTy = TLI.getFrameIndexTy(DL);
3200 EVT PtrMemTy = TLI.getPointerMemTy(DL, AS: DL.getAllocaAddrSpace());
3201
3202 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3203 int FI = MFI.getStackProtectorIndex();
3204
3205 SDValue Guard;
3206 SDLoc dl = getCurSDLoc();
3207 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3208 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3209 Align Align = DL.getPrefTypeAlign(
3210 Ty: PointerType::get(C&: M.getContext(), AddressSpace: DL.getAllocaAddrSpace()));
3211
3212 // Generate code to load the content of the guard slot.
3213 SDValue GuardVal = DAG.getLoad(
3214 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3215 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3216 MMOFlags: MachineMemOperand::MOVolatile);
3217
3218 // If cookie mixing is enabled, unmix the stored GuardVal to get back the
3219 // original cookie for comparison. The prologue stored (FP - Cookie) or
3220 // (FP XOR Cookie), so we apply the same operation again to unmix:
3221 // FP - (FP - Cookie) = Cookie, or (FP XOR Cookie) XOR FP = Cookie.
3222 if (TLI.useStackGuardMixFP())
3223 GuardVal = TLI.emitStackGuardMixFP(DAG, Val: GuardVal, DL: dl);
3224
3225 // If we're using function-based instrumentation, call the guard check
3226 // function
3227 if (SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3228 // Get the guard check function from the target and verify it exists since
3229 // we're using function-based instrumentation
3230 const Function *GuardCheckFn =
3231 TLI.getSSPStackGuardCheck(M, Libcalls: DAG.getLibcalls());
3232 assert(GuardCheckFn && "Guard check function is null");
3233
3234 // The target provides a guard check function to validate the guard value.
3235 // Generate a call to that function with the content of the guard slot as
3236 // argument.
3237 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3238 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3239
3240 TargetLowering::ArgListTy Args;
3241 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(i: 0));
3242 if (GuardCheckFn->hasParamAttribute(ArgNo: 0, Kind: Attribute::AttrKind::InReg))
3243 Entry.IsInReg = true;
3244 Args.push_back(x: Entry);
3245
3246 TargetLowering::CallLoweringInfo CLI(DAG);
3247 CLI.setDebugLoc(getCurSDLoc())
3248 .setChain(DAG.getEntryNode())
3249 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3250 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3251
3252 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3253 DAG.setRoot(Result.second);
3254 return;
3255 }
3256
3257 // Load the fresh guard value for comparison.
3258 // For targets that mix the cookie in LOAD_STACK_GUARD expansion, we need to
3259 // load directly without using LOAD_STACK_GUARD to avoid unwanted mixing.
3260 SDValue Chain = DAG.getEntryNode();
3261 if (TLI.useStackGuardMixFP()) {
3262 // Mixing targets: load cookie directly to avoid mixing in LOAD_STACK_GUARD
3263 if (const Value *IRGuard = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls())) {
3264 SDValue GuardPtr = getValue(V: IRGuard);
3265 Guard = DAG.getLoad(VT: PtrMemTy, dl, Chain, Ptr: GuardPtr,
3266 PtrInfo: MachinePointerInfo(IRGuard, 0), Alignment: Align,
3267 MMOFlags: MachineMemOperand::MOVolatile);
3268 } else {
3269 LLVMContext &Ctx = *DAG.getContext();
3270 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
3271 Guard = DAG.getPOISON(VT: PtrMemTy);
3272 }
3273 } else {
3274 // Non-mixing targets: use LOAD_STACK_GUARD or direct load as usual
3275 if (TLI.useLoadStackGuardNode(M)) {
3276 Guard = getLoadStackGuard(DAG, DL: dl, Chain);
3277 } else {
3278 if (const Value *IRGuard = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls())) {
3279 SDValue GuardPtr = getValue(V: IRGuard);
3280 Guard = DAG.getLoad(VT: PtrMemTy, dl, Chain, Ptr: GuardPtr,
3281 PtrInfo: MachinePointerInfo(IRGuard, 0), Alignment: Align,
3282 MMOFlags: MachineMemOperand::MOVolatile);
3283 } else {
3284 LLVMContext &Ctx = *DAG.getContext();
3285 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
3286 Guard = DAG.getPOISON(VT: PtrMemTy);
3287 }
3288 }
3289 }
3290
3291 // Now both Guard (fresh cookie) and GuardVal (unmixed from stored value)
3292 // contain unmixed cookie values that can be compared directly.
3293
3294 // Perform the comparison via a getsetcc.
3295 SDValue Cmp = DAG.getSetCC(
3296 DL: dl, VT: TLI.getSetCCResultType(DL, Context&: *DAG.getContext(), VT: Guard.getValueType()),
3297 LHS: Guard, RHS: GuardVal, Cond: ISD::SETNE);
3298
3299 // If the guard/stackslot do not equal, branch to failure MBB.
3300 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: getControlRoot(),
3301 N2: Cmp, N3: DAG.getBasicBlock(MBB: SPD.getFailureMBB()));
3302 // Otherwise branch to success MBB.
3303 SDValue Br = DAG.getNode(Opcode: ISD::BR, DL: dl,
3304 VT: MVT::Other, N1: BrCond,
3305 N2: DAG.getBasicBlock(MBB: SPD.getSuccessMBB()));
3306
3307 DAG.setRoot(Br);
3308}
3309
3310/// Codegen the failure basic block for a stack protector check.
3311///
3312/// A failure stack protector machine basic block consists simply of a call to
3313/// __stack_chk_fail().
3314///
3315/// For a high level explanation of how this fits into the stack protector
3316/// generation see the comment on the declaration of class
3317/// StackProtectorDescriptor.
3318void SelectionDAGBuilder::visitSPDescriptorFailure(
3319 StackProtectorDescriptor &SPD) {
3320
3321 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3322 MachineBasicBlock *ParentBB = SPD.getParentMBB();
3323 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3324 SDValue Chain;
3325
3326 // For -Oz builds with a guard check function, we use function-based
3327 // instrumentation. Otherwise, if we have a guard check function, we call it
3328 // in the failure block.
3329 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M, Libcalls: DAG.getLibcalls());
3330 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3331 // First create the loads to the guard/stack slot for the comparison.
3332 auto &DL = DAG.getDataLayout();
3333 EVT PtrTy = TLI.getFrameIndexTy(DL);
3334 EVT PtrMemTy = TLI.getPointerMemTy(DL, AS: DL.getAllocaAddrSpace());
3335
3336 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3337 int FI = MFI.getStackProtectorIndex();
3338
3339 SDLoc dl = getCurSDLoc();
3340 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3341 Align Align = DL.getPrefTypeAlign(
3342 Ty: PointerType::get(C&: M.getContext(), AddressSpace: DL.getAllocaAddrSpace()));
3343
3344 // Generate code to load the content of the guard slot.
3345 SDValue GuardVal = DAG.getLoad(
3346 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3347 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3348 MMOFlags: MachineMemOperand::MOVolatile);
3349
3350 if (TLI.useStackGuardMixFP())
3351 GuardVal = TLI.emitStackGuardMixFP(DAG, Val: GuardVal, DL: dl);
3352
3353 // The target provides a guard check function to validate the guard value.
3354 // Generate a call to that function with the content of the guard slot as
3355 // argument.
3356 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3357 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3358
3359 TargetLowering::ArgListTy Args;
3360 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(i: 0));
3361 if (GuardCheckFn->hasParamAttribute(ArgNo: 0, Kind: Attribute::AttrKind::InReg))
3362 Entry.IsInReg = true;
3363 Args.push_back(x: Entry);
3364
3365 TargetLowering::CallLoweringInfo CLI(DAG);
3366 CLI.setDebugLoc(getCurSDLoc())
3367 .setChain(DAG.getEntryNode())
3368 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3369 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3370
3371 Chain = TLI.LowerCallTo(CLI).second;
3372 } else {
3373 TargetLowering::MakeLibCallOptions CallOptions;
3374 CallOptions.setDiscardResult(true);
3375 Chain = TLI.makeLibCall(DAG, LC: RTLIB::STACKPROTECTOR_CHECK_FAIL, RetVT: MVT::isVoid,
3376 Ops: {}, CallOptions, dl: getCurSDLoc())
3377 .second;
3378 }
3379
3380 // Emit a trap instruction if we are required to do so.
3381 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3382 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3383 Chain = DAG.getNode(Opcode: ISD::TRAP, DL: getCurSDLoc(), VT: MVT::Other, Operand: Chain);
3384
3385 DAG.setRoot(Chain);
3386}
3387
3388/// visitBitTestHeader - This function emits necessary code to produce value
3389/// suitable for "bit tests"
3390void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
3391 MachineBasicBlock *SwitchBB) {
3392 SDLoc dl = getCurSDLoc();
3393
3394 // Subtract the minimum value.
3395 SDValue SwitchOp = getValue(V: B.SValue);
3396 EVT VT = SwitchOp.getValueType();
3397 SDValue RangeSub =
3398 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp, N2: DAG.getConstant(Val: B.First, DL: dl, VT));
3399
3400 // Determine the type of the test operands.
3401 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3402 bool UsePtrType = false;
3403 if (!TLI.isTypeLegal(VT)) {
3404 UsePtrType = true;
3405 } else {
3406 for (const BitTestCase &Case : B.Cases)
3407 if (!isUIntN(N: VT.getSizeInBits(), x: Case.Mask)) {
3408 // Switch table case range are encoded into series of masks.
3409 // Just use pointer type, it's guaranteed to fit.
3410 UsePtrType = true;
3411 break;
3412 }
3413 }
3414 SDValue Sub = RangeSub;
3415 if (UsePtrType) {
3416 VT = TLI.getPointerTy(DL: DAG.getDataLayout());
3417 Sub = DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT);
3418 }
3419
3420 B.RegVT = VT.getSimpleVT();
3421 B.Reg = FuncInfo.CreateReg(VT: B.RegVT);
3422 SDValue CopyTo = DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: B.Reg, N: Sub);
3423
3424 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3425
3426 if (!B.FallthroughUnreachable)
3427 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
3428 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
3429 SwitchBB->normalizeSuccProbs();
3430
3431 SDValue Root = CopyTo;
3432 if (!B.FallthroughUnreachable) {
3433 // Conditional branch to the default block.
3434 SDValue RangeCmp = DAG.getSetCC(DL: dl,
3435 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3436 VT: RangeSub.getValueType()),
3437 LHS: RangeSub, RHS: DAG.getConstant(Val: B.Range, DL: dl, VT: RangeSub.getValueType()),
3438 Cond: ISD::SETUGT);
3439
3440 Root = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: Root, N2: RangeCmp,
3441 N3: DAG.getBasicBlock(MBB: B.Default));
3442 }
3443
3444 // Avoid emitting unnecessary branches to the next block.
3445 if (MBB != NextBlock(MBB: SwitchBB))
3446 Root = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: Root, N2: DAG.getBasicBlock(MBB));
3447
3448 DAG.setRoot(Root);
3449}
3450
3451/// visitBitTestCase - this function produces one "bit test"
3452void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
3453 MachineBasicBlock *NextMBB,
3454 BranchProbability BranchProbToNext,
3455 Register Reg, BitTestCase &B,
3456 MachineBasicBlock *SwitchBB) {
3457 SDLoc dl = getCurSDLoc();
3458 MVT VT = BB.RegVT;
3459 SDValue ShiftOp = DAG.getCopyFromReg(Chain: getControlRoot(), dl, Reg, VT);
3460 SDValue Cmp;
3461 unsigned PopCount = llvm::popcount(Value: B.Mask);
3462 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3463 if (PopCount == 1) {
3464 // Testing for a single bit; just compare the shift count with what it
3465 // would need to be to shift a 1 bit in that position.
3466 Cmp = DAG.getSetCC(
3467 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3468 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_zero(Val: B.Mask), DL: dl, VT),
3469 Cond: ISD::SETEQ);
3470 } else if (PopCount == BB.Range) {
3471 // There is only one zero bit in the range, test for it directly.
3472 Cmp = DAG.getSetCC(
3473 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3474 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_one(Value: B.Mask), DL: dl, VT), Cond: ISD::SETNE);
3475 } else {
3476 // Make desired shift
3477 SDValue SwitchVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT,
3478 N1: DAG.getConstant(Val: 1, DL: dl, VT), N2: ShiftOp);
3479
3480 // Emit bit tests and jumps
3481 SDValue AndOp = DAG.getNode(Opcode: ISD::AND, DL: dl,
3482 VT, N1: SwitchVal, N2: DAG.getConstant(Val: B.Mask, DL: dl, VT));
3483 Cmp = DAG.getSetCC(
3484 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3485 LHS: AndOp, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
3486 }
3487
3488 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3489 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
3490 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3491 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
3492 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3493 // one as they are relative probabilities (and thus work more like weights),
3494 // and hence we need to normalize them to let the sum of them become one.
3495 SwitchBB->normalizeSuccProbs();
3496
3497 SDValue BrAnd = DAG.getNode(Opcode: ISD::BRCOND, DL: dl,
3498 VT: MVT::Other, N1: getControlRoot(),
3499 N2: Cmp, N3: DAG.getBasicBlock(MBB: B.TargetBB));
3500
3501 // Avoid emitting unnecessary branches to the next block.
3502 if (NextMBB != NextBlock(MBB: SwitchBB))
3503 BrAnd = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrAnd,
3504 N2: DAG.getBasicBlock(MBB: NextMBB));
3505
3506 DAG.setRoot(BrAnd);
3507}
3508
3509void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3510 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3511
3512 // Retrieve successors. Look through artificial IR level blocks like
3513 // catchswitch for successors.
3514 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
3515 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3516 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(BB: EHPadBB);
3517
3518 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3519 // have to do anything here to lower funclet bundles.
3520 failForInvalidBundles(I, Name: "invokes",
3521 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3522 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3523 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3524 LLVMContext::OB_clang_arc_attachedcall,
3525 LLVMContext::OB_kcfi});
3526
3527 const Value *Callee(I.getCalledOperand());
3528 const Function *Fn = dyn_cast<Function>(Val: Callee);
3529 if (isa<InlineAsm>(Val: Callee))
3530 visitInlineAsm(Call: I, EHPadBB);
3531 else if (Fn && Fn->isIntrinsic()) {
3532 switch (Fn->getIntrinsicID()) {
3533 default:
3534 llvm_unreachable("Cannot invoke this intrinsic");
3535 case Intrinsic::donothing:
3536 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3537 case Intrinsic::seh_try_begin:
3538 case Intrinsic::seh_scope_begin:
3539 case Intrinsic::seh_try_end:
3540 case Intrinsic::seh_scope_end:
3541 if (EHPadMBB)
3542 // a block referenced by EH table
3543 // so dtor-funclet not removed by opts
3544 EHPadMBB->setMachineBlockAddressTaken();
3545 break;
3546 case Intrinsic::experimental_patchpoint_void:
3547 case Intrinsic::experimental_patchpoint:
3548 visitPatchpoint(CB: I, EHPadBB);
3549 break;
3550 case Intrinsic::experimental_gc_statepoint:
3551 LowerStatepoint(I: cast<GCStatepointInst>(Val: I), EHPadBB);
3552 break;
3553 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3554 // but these intrinsics are special because they can be invoked, so we
3555 // manually lower it to a DAG node here.
3556 case Intrinsic::wasm_throw: {
3557 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3558 std::array<SDValue, 4> Ops = {
3559 getControlRoot(), // inchain for the terminator node
3560 DAG.getTargetConstant(Val: Intrinsic::wasm_throw, DL: getCurSDLoc(),
3561 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3562 getValue(V: I.getArgOperand(i: 0)), // tag
3563 getValue(V: I.getArgOperand(i: 1)) // thrown value
3564 };
3565 SDVTList VTs = DAG.getVTList(VTs: ArrayRef<EVT>({MVT::Other})); // outchain
3566 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3567 break;
3568 }
3569 case Intrinsic::wasm_rethrow: {
3570 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3571 std::array<SDValue, 2> Ops = {
3572 getControlRoot(), // inchain for the terminator node
3573 DAG.getTargetConstant(Val: Intrinsic::wasm_rethrow, DL: getCurSDLoc(),
3574 VT: TLI.getPointerTy(DL: DAG.getDataLayout()))};
3575 SDVTList VTs = DAG.getVTList(VTs: ArrayRef<EVT>({MVT::Other})); // outchain
3576 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3577 break;
3578 }
3579 // EH labels mark it as an invoke site, like a call.
3580 case Intrinsic::bpf_unwind_exit: {
3581 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3582 MCSymbol *BeginLabel;
3583 std::array<SDValue, 2> Ops = {
3584 lowerStartEH(Chain: getControlRoot(), EHPadBB, BeginLabel),
3585 DAG.getTargetConstant(Val: Intrinsic::bpf_unwind_exit, DL: getCurSDLoc(),
3586 VT: TLI.getPointerTy(DL: DAG.getDataLayout()))};
3587 SDValue Chain =
3588 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VT: MVT::Other, Ops);
3589 DAG.setRoot(lowerEndEH(Chain, II: &I, EHPadBB, BeginLabel));
3590 break;
3591 }
3592 }
3593 } else if (I.hasDeoptState()) {
3594 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3595 // Eventually we will support lowering the @llvm.experimental.deoptimize
3596 // intrinsic, and right now there are no plans to support other intrinsics
3597 // with deopt state.
3598 LowerCallSiteWithDeoptBundle(Call: &I, Callee: getValue(V: Callee), EHPadBB);
3599 } else if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
3600 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), EHPadBB);
3601 } else {
3602 LowerCallTo(CB: I, Callee: getValue(V: Callee), IsTailCall: false, IsMustTailCall: false, EHPadBB);
3603 }
3604
3605 // If the value of the invoke is used outside of its defining block, make it
3606 // available as a virtual register.
3607 // We already took care of the exported value for the statepoint instruction
3608 // during call to the LowerStatepoint.
3609 if (!isa<GCStatepointInst>(Val: I)) {
3610 CopyToExportRegsIfNeeded(V: &I);
3611 }
3612
3613 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3614 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3615 BranchProbability EHPadBBProb =
3616 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3617 : BranchProbability::getZero();
3618 findUnwindDestinations(FuncInfo, EHPadBB, Prob: EHPadBBProb, UnwindDests);
3619
3620 // Update successor info.
3621 addSuccessorWithProb(Src: InvokeMBB, Dst: Return);
3622 for (auto &UnwindDest : UnwindDests) {
3623 UnwindDest.first->setIsEHPad();
3624 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3625 }
3626 InvokeMBB->normalizeSuccProbs();
3627
3628 // Drop into normal successor.
3629 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other, N1: getControlRoot(),
3630 N2: DAG.getBasicBlock(MBB: Return)));
3631}
3632
3633/// The intrinsics currently supported by callbr are implicit control flow
3634/// intrinsics such as amdgcn.kill.
3635/// - they should be called (no "dontcall-" attributes)
3636/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
3637/// - they do not need custom argument handling (no
3638/// TLI.CollectTargetIntrinsicOperands())
3639void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
3640#ifndef NDEBUG
3641 SmallVector<TargetLowering::IntrinsicInfo, 2> Infos;
3642 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3643 Infos, I, DAG.getMachineFunction(), I.getIntrinsicID());
3644 assert(Infos.empty() && "Intrinsic touches memory");
3645#endif
3646
3647 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
3648
3649 SmallVector<SDValue, 8> Ops =
3650 getTargetIntrinsicOperands(I, HasChain, OnlyLoad);
3651 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
3652
3653 // Create the node.
3654 SDValue Result =
3655 getTargetNonMemIntrinsicNode(IntrinsicVT: *I.getType(), HasChain, Ops, VTs);
3656 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
3657
3658 setValue(V: &I, NewN: Result);
3659}
3660
3661void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3662 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3663
3664 if (I.isInlineAsm()) {
3665 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3666 // have to do anything here to lower funclet bundles.
3667 failForInvalidBundles(I, Name: "callbrs",
3668 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_funclet});
3669 visitInlineAsm(Call: I);
3670 } else {
3671 assert(!I.hasOperandBundles() &&
3672 "Can't have operand bundles for intrinsics");
3673 visitCallBrIntrinsic(I);
3674 }
3675 CopyToExportRegsIfNeeded(V: &I);
3676
3677 // Retrieve successors.
3678 SmallPtrSet<BasicBlock *, 8> Dests;
3679 Dests.insert(Ptr: I.getDefaultDest());
3680 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getDefaultDest());
3681
3682 // Update successor info.
3683 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3684 // TODO: For most of the cases where there is an intrinsic callbr, we're
3685 // having exactly one indirect target, which will be unreachable. As soon as
3686 // this changes, we might need to enhance
3687 // Target->setIsInlineAsmBrIndirectTarget or add something similar for
3688 // intrinsic indirect branches.
3689 if (I.isInlineAsm()) {
3690 for (BasicBlock *Dest : I.getIndirectDests()) {
3691 MachineBasicBlock *Target = FuncInfo.getMBB(BB: Dest);
3692 Target->setIsInlineAsmBrIndirectTarget();
3693 // If we introduce a type of asm goto statement that is permitted to use
3694 // an indirect call instruction to jump to its labels, then we should add
3695 // a call to Target->setMachineBlockAddressTaken() here, to mark the
3696 // target block as requiring a BTI.
3697
3698 Target->setLabelMustBeEmitted();
3699 // Don't add duplicate machine successors.
3700 if (Dests.insert(Ptr: Dest).second)
3701 addSuccessorWithProb(Src: CallBrMBB, Dst: Target, Prob: BranchProbability::getZero());
3702 }
3703 }
3704 CallBrMBB->normalizeSuccProbs();
3705
3706 // Drop into default successor.
3707 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(),
3708 VT: MVT::Other, N1: getControlRoot(),
3709 N2: DAG.getBasicBlock(MBB: Return)));
3710}
3711
3712void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3713 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3714}
3715
3716void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3717 assert(FuncInfo.MBB->isEHPad() &&
3718 "Call to landingpad not in landing pad!");
3719
3720 // If there aren't registers to copy the values into (e.g., during SjLj
3721 // exceptions), then don't bother to create these DAG nodes.
3722 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3723 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3724 if (TLI.getExceptionPointerRegister(EH: FuncInfo.ExceptionModel, PersonalityFn) ==
3725 0 &&
3726 TLI.getExceptionSelectorRegister(EH: FuncInfo.ExceptionModel,
3727 PersonalityFn) == 0)
3728 return;
3729
3730 // If landingpad's return type is token type, we don't create DAG nodes
3731 // for its exception pointer and selector value. The extraction of exception
3732 // pointer or selector value from token type landingpads is not currently
3733 // supported.
3734 if (LP.getType()->isTokenTy())
3735 return;
3736
3737 // LangRef leaves the result type target-specific, so diagnose types this
3738 // lowering cannot represent instead of asserting.
3739 SDLoc dl = getCurSDLoc();
3740 if (!isExceptionPointerAndSelectorType(Ty: LP.getType())) {
3741 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3742 *LP.getFunction(),
3743 "landingpad result type must be a struct of an exception pointer and "
3744 "an integer selector",
3745 dl.getDebugLoc()));
3746 setValueToPoison(V: &LP, dl);
3747 return;
3748 }
3749
3750 SmallVector<EVT, 2> ValueVTs;
3751 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: LP.getType(), ValueVTs);
3752 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3753
3754 // Get the two live-in registers as SDValues. The physregs have already been
3755 // copied into virtual registers.
3756 SDValue Ops[2];
3757 if (FuncInfo.ExceptionPointerVirtReg) {
3758 Ops[0] = DAG.getZExtOrTrunc(
3759 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3760 Reg: FuncInfo.ExceptionPointerVirtReg,
3761 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3762 DL: dl, VT: ValueVTs[0]);
3763 } else {
3764 Ops[0] = DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3765 }
3766 Ops[1] = DAG.getZExtOrTrunc(
3767 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3768 Reg: FuncInfo.ExceptionSelectorVirtReg,
3769 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3770 DL: dl, VT: ValueVTs[1]);
3771
3772 // Merge into one.
3773 SDValue Res = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
3774 VTList: DAG.getVTList(VTs: ValueVTs), Ops);
3775 setValue(V: &LP, NewN: Res);
3776}
3777
3778void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3779 MachineBasicBlock *Last) {
3780 // Update JTCases.
3781 for (JumpTableBlock &JTB : SL->JTCases)
3782 if (JTB.first.HeaderBB == First)
3783 JTB.first.HeaderBB = Last;
3784
3785 // Update BitTestCases.
3786 for (BitTestBlock &BTB : SL->BitTestCases)
3787 if (BTB.Parent == First)
3788 BTB.Parent = Last;
3789}
3790
3791void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3792 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3793
3794 // Update machine-CFG edges with unique successors.
3795 SmallPtrSet<BasicBlock *, 32> Done;
3796 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3797 BasicBlock *BB = I.getSuccessor(i);
3798 bool Inserted = Done.insert(Ptr: BB).second;
3799 if (!Inserted)
3800 continue;
3801
3802 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3803 addSuccessorWithProb(Src: IndirectBrMBB, Dst: Succ);
3804 }
3805 IndirectBrMBB->normalizeSuccProbs();
3806
3807 DAG.setRoot(DAG.getNode(Opcode: ISD::BRIND, DL: getCurSDLoc(),
3808 VT: MVT::Other, N1: getControlRoot(),
3809 N2: getValue(V: I.getAddress())));
3810}
3811
3812void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3813 if (!I.shouldLowerToTrap(TrapUnreachable: DAG.getTarget().Options.TrapUnreachable,
3814 NoTrapAfterNoreturn: DAG.getTarget().Options.NoTrapAfterNoreturn))
3815 return;
3816
3817 DAG.setRoot(DAG.getNode(Opcode: ISD::TRAP, DL: getCurSDLoc(), VT: MVT::Other, Operand: DAG.getRoot()));
3818}
3819
3820void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3821 SDNodeFlags Flags;
3822 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3823 Flags.copyFMF(FPMO: *FPOp);
3824
3825 SDValue Op = getValue(V: I.getOperand(i: 0));
3826 SDValue UnNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op.getValueType(),
3827 Operand: Op, Flags);
3828 setValue(V: &I, NewN: UnNodeValue);
3829}
3830
3831void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3832 SDNodeFlags Flags;
3833 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(Val: &I)) {
3834 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3835 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3836 }
3837 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(Val: &I))
3838 Flags.setExact(ExactOp->isExact());
3839 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(Val: &I))
3840 Flags.setDisjoint(DisjointOp->isDisjoint());
3841 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3842 Flags.copyFMF(FPMO: *FPOp);
3843
3844 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3845 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3846 SDValue BinNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(),
3847 N1: Op1, N2: Op2, Flags);
3848 setValue(V: &I, NewN: BinNodeValue);
3849}
3850
3851void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3852 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3853 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3854
3855 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3856 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
3857
3858 // Coerce the shift amount to the right type if we can. This exposes the
3859 // truncate or zext to optimization early.
3860 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3861 assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3862 "Unexpected shift type");
3863 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
3864 }
3865
3866 bool nuw = false;
3867 bool nsw = false;
3868 bool exact = false;
3869
3870 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3871
3872 if (const OverflowingBinaryOperator *OFBinOp =
3873 dyn_cast<const OverflowingBinaryOperator>(Val: &I)) {
3874 nuw = OFBinOp->hasNoUnsignedWrap();
3875 nsw = OFBinOp->hasNoSignedWrap();
3876 }
3877 if (const PossiblyExactOperator *ExactOp =
3878 dyn_cast<const PossiblyExactOperator>(Val: &I))
3879 exact = ExactOp->isExact();
3880 }
3881 SDNodeFlags Flags;
3882 Flags.setExact(exact);
3883 Flags.setNoSignedWrap(nsw);
3884 Flags.setNoUnsignedWrap(nuw);
3885 SDValue Res = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1, N2: Op2,
3886 Flags);
3887 setValue(V: &I, NewN: Res);
3888}
3889
3890void SelectionDAGBuilder::visitSDiv(const User &I) {
3891 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3892 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3893
3894 SDNodeFlags Flags;
3895 Flags.setExact(isa<PossiblyExactOperator>(Val: &I) &&
3896 cast<PossiblyExactOperator>(Val: &I)->isExact());
3897 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SDIV, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1,
3898 N2: Op2, Flags));
3899}
3900
3901void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3902 ICmpInst::Predicate predicate = I.getPredicate();
3903 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3904 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3905 ISD::CondCode Opcode = getICmpCondCode(Pred: predicate);
3906
3907 auto &TLI = DAG.getTargetLoweringInfo();
3908 EVT MemVT =
3909 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
3910
3911 // If a pointer's DAG type is larger than its memory type then the DAG values
3912 // are zero-extended. This breaks signed comparisons so truncate back to the
3913 // underlying type before doing the compare.
3914 if (Op1.getValueType() != MemVT) {
3915 Op1 = DAG.getPtrExtOrTrunc(Op: Op1, DL: getCurSDLoc(), VT: MemVT);
3916 Op2 = DAG.getPtrExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: MemVT);
3917 }
3918
3919 SDNodeFlags Flags;
3920 Flags.setSameSign(I.hasSameSign());
3921
3922 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3923 Ty: I.getType());
3924 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Opcode,
3925 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3926}
3927
3928void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3929 FCmpInst::Predicate predicate = I.getPredicate();
3930 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3931 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3932
3933 ISD::CondCode Condition = getFCmpCondCode(Pred: predicate);
3934 auto *FPMO = cast<FPMathOperator>(Val: &I);
3935 if (FPMO->hasNoNaNs() ||
3936 (DAG.isKnownNeverNaN(Op: Op1) && DAG.isKnownNeverNaN(Op: Op2)))
3937 Condition = getFCmpCodeWithoutNaN(CC: Condition);
3938
3939 SDNodeFlags Flags;
3940 Flags.copyFMF(FPMO: *FPMO);
3941
3942 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3943 Ty: I.getType());
3944 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Condition,
3945 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3946}
3947
3948// Check if the condition of the select has one use or two users that are both
3949// selects with the same condition.
3950static bool hasOnlySelectUsers(const Value *Cond) {
3951 return llvm::all_of(Range: Cond->users(), P: [](const Value *V) {
3952 return isa<SelectInst>(Val: V);
3953 });
3954}
3955
3956void SelectionDAGBuilder::visitSelect(const User &I) {
3957 SmallVector<EVT, 4> ValueVTs;
3958 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
3959 ValueVTs);
3960 unsigned NumValues = ValueVTs.size();
3961 if (NumValues == 0) return;
3962
3963 SmallVector<SDValue, 4> Values(NumValues);
3964 SDValue Cond = getValue(V: I.getOperand(i: 0));
3965 SDValue LHSVal = getValue(V: I.getOperand(i: 1));
3966 SDValue RHSVal = getValue(V: I.getOperand(i: 2));
3967 SmallVector<SDValue, 1> BaseOps(1, Cond);
3968 ISD::NodeType OpCode =
3969 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3970
3971 bool IsUnaryAbs = false;
3972 bool Negate = false;
3973
3974 SDNodeFlags Flags;
3975 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3976 Flags.copyFMF(FPMO: *FPOp);
3977
3978 Flags.setUnpredictable(
3979 cast<SelectInst>(Val: I).getMetadata(KindID: LLVMContext::MD_unpredictable));
3980
3981 // Min/max matching is only viable if all output VTs are the same.
3982 if (all_equal(Range&: ValueVTs)) {
3983 EVT VT = ValueVTs[0];
3984 LLVMContext &Ctx = *DAG.getContext();
3985 auto &TLI = DAG.getTargetLoweringInfo();
3986
3987 // We care about the legality of the operation after it has been type
3988 // legalized.
3989 while (TLI.getTypeAction(Context&: Ctx, VT) != TargetLoweringBase::TypeLegal)
3990 VT = TLI.getTypeToTransformTo(Context&: Ctx, VT);
3991
3992 // If the vselect is legal, assume we want to leave this as a vector setcc +
3993 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3994 // min/max is legal on the scalar type.
3995 bool UseScalarMinMax = VT.isVector() &&
3996 !TLI.isOperationLegalOrCustom(Op: ISD::VSELECT, VT);
3997
3998 // ValueTracking's select pattern matching does not account for -0.0,
3999 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
4000 // -0.0 is less than +0.0.
4001 const Value *LHS, *RHS;
4002 auto SPR = matchSelectPattern(V: &I, LHS, RHS);
4003 ISD::NodeType Opc = ISD::DELETED_NODE;
4004 switch (SPR.Flavor) {
4005 case SPF_UMAX: Opc = ISD::UMAX; break;
4006 case SPF_UMIN: Opc = ISD::UMIN; break;
4007 case SPF_SMAX: Opc = ISD::SMAX; break;
4008 case SPF_SMIN: Opc = ISD::SMIN; break;
4009 case SPF_FMINNUM:
4010 if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
4011 break;
4012
4013 switch (SPR.NaNBehavior) {
4014 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4015 case SPNB_RETURNS_ANY:
4016 case SPNB_RETURNS_NAN:
4017 break;
4018 case SPNB_RETURNS_OTHER:
4019 Opc = ISD::FMINIMUMNUM;
4020 Flags.setNoSignedZeros(true);
4021 break;
4022 }
4023 break;
4024 case SPF_FMAXNUM:
4025 if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
4026 break;
4027
4028 switch (SPR.NaNBehavior) {
4029 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4030 case SPNB_RETURNS_NAN:
4031 case SPNB_RETURNS_ANY:
4032 break;
4033 case SPNB_RETURNS_OTHER:
4034 Opc = ISD::FMAXIMUMNUM;
4035 Flags.setNoSignedZeros(true);
4036 break;
4037 }
4038 break;
4039 case SPF_NABS:
4040 Negate = true;
4041 [[fallthrough]];
4042 case SPF_ABS:
4043 IsUnaryAbs = true;
4044 Opc = ISD::ABS;
4045 break;
4046 default: break;
4047 }
4048
4049 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
4050 (TLI.isOperationLegalOrCustom(Op: Opc, VT) ||
4051 (UseScalarMinMax &&
4052 TLI.isOperationLegalOrCustom(Op: Opc, VT: VT.getScalarType()))) &&
4053 // If the underlying comparison instruction is used by any other
4054 // instruction, the consumed instructions won't be destroyed, so it is
4055 // not profitable to convert to a min/max.
4056 hasOnlySelectUsers(Cond: cast<SelectInst>(Val: I).getCondition())) {
4057 OpCode = Opc;
4058 LHSVal = getValue(V: LHS);
4059 RHSVal = getValue(V: RHS);
4060 BaseOps.clear();
4061 }
4062
4063 if (IsUnaryAbs) {
4064 OpCode = Opc;
4065 LHSVal = getValue(V: LHS);
4066 BaseOps.clear();
4067 }
4068 }
4069
4070 if (IsUnaryAbs) {
4071 for (unsigned i = 0; i != NumValues; ++i) {
4072 SDLoc dl = getCurSDLoc();
4073 EVT VT = LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i);
4074 Values[i] =
4075 DAG.getNode(Opcode: OpCode, DL: dl, VT, Operand: LHSVal.getValue(R: LHSVal.getResNo() + i));
4076 if (Negate)
4077 Values[i] = DAG.getNegative(Val: Values[i], DL: dl, VT);
4078 }
4079 } else {
4080 for (unsigned i = 0; i != NumValues; ++i) {
4081 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
4082 Ops.push_back(Elt: SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
4083 Ops.push_back(Elt: SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
4084 Values[i] = DAG.getNode(
4085 Opcode: OpCode, DL: getCurSDLoc(),
4086 VT: LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i), Ops, Flags);
4087 }
4088 }
4089
4090 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4091 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
4092}
4093
4094void SelectionDAGBuilder::visitTrunc(const User &I) {
4095 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
4096 SDValue N = getValue(V: I.getOperand(i: 0));
4097 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4098 Ty: I.getType());
4099 SDNodeFlags Flags;
4100 if (auto *Trunc = dyn_cast<TruncInst>(Val: &I)) {
4101 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
4102 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
4103 }
4104
4105 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4106}
4107
4108void SelectionDAGBuilder::visitZExt(const User &I) {
4109 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4110 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
4111 SDValue N = getValue(V: I.getOperand(i: 0));
4112 auto &TLI = DAG.getTargetLoweringInfo();
4113 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4114
4115 SDNodeFlags Flags;
4116 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(Val: &I))
4117 Flags.setNonNeg(PNI->hasNonNeg());
4118
4119 // Eagerly use nonneg information to canonicalize towards sign_extend if
4120 // that is the target's preference.
4121 // TODO: Let the target do this later.
4122 if (Flags.hasNonNeg() &&
4123 TLI.isSExtCheaperThanZExt(FromTy: N.getValueType(), ToTy: DestVT)) {
4124 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4125 return;
4126 }
4127
4128 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4129}
4130
4131void SelectionDAGBuilder::visitSExt(const User &I) {
4132 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4133 // SExt also can't be a cast to bool for same reason. So, nothing much to do
4134 SDValue N = getValue(V: I.getOperand(i: 0));
4135 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4136 Ty: I.getType());
4137 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4138}
4139
4140void SelectionDAGBuilder::visitFPTrunc(const User &I) {
4141 // FPTrunc is never a no-op cast, no need to check
4142 SDValue N = getValue(V: I.getOperand(i: 0));
4143 SDLoc dl = getCurSDLoc();
4144 SDNodeFlags Flags;
4145 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
4146 Flags.copyFMF(FPMO: *FPOp);
4147 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4148 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4149 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: DestVT, N1: N,
4150 N2: DAG.getTargetConstant(
4151 Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4152 Flags));
4153}
4154
4155void SelectionDAGBuilder::visitFPExt(const User &I) {
4156 // FPExt is never a no-op cast, no need to check
4157 SDValue N = getValue(V: I.getOperand(i: 0));
4158 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4159 Ty: I.getType());
4160 SDNodeFlags Flags;
4161 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
4162 Flags.copyFMF(FPMO: *FPOp);
4163 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4164}
4165
4166void SelectionDAGBuilder::visitFPToUI(const User &I) {
4167 // FPToUI is never a no-op cast, no need to check
4168 SDValue N = getValue(V: I.getOperand(i: 0));
4169 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4170 Ty: I.getType());
4171 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4172}
4173
4174void SelectionDAGBuilder::visitFPToSI(const User &I) {
4175 // FPToSI is never a no-op cast, no need to check
4176 SDValue N = getValue(V: I.getOperand(i: 0));
4177 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4178 Ty: I.getType());
4179 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4180}
4181
4182void SelectionDAGBuilder::visitUIToFP(const User &I) {
4183 // UIToFP is never a no-op cast, no need to check
4184 SDValue N = getValue(V: I.getOperand(i: 0));
4185 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4186 Ty: I.getType());
4187 SDNodeFlags Flags;
4188 Flags.setNonNeg(cast<PossiblyNonNegInst>(Val: &I)->hasNonNeg());
4189 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
4190
4191 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4192}
4193
4194void SelectionDAGBuilder::visitSIToFP(const User &I) {
4195 // SIToFP is never a no-op cast, no need to check
4196 SDValue N = getValue(V: I.getOperand(i: 0));
4197 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4198 Ty: I.getType());
4199 SDNodeFlags Flags;
4200 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
4201
4202 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4203}
4204
4205void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
4206 SDValue N = getValue(V: I.getOperand(i: 0));
4207 // By definition the type of the ptrtoaddr must be equal to the address type.
4208 const auto &TLI = DAG.getTargetLoweringInfo();
4209 EVT AddrVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4210 // The address width must be smaller or equal to the pointer representation
4211 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).
4212 N = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: AddrVT, Operand: N);
4213 setValue(V: &I, NewN: N);
4214}
4215
4216void SelectionDAGBuilder::visitPtrToInt(const User &I) {
4217 // What to do depends on the size of the integer and the size of the pointer.
4218 // We can either truncate, zero extend, or no-op, accordingly.
4219 SDValue N = getValue(V: I.getOperand(i: 0));
4220 auto &TLI = DAG.getTargetLoweringInfo();
4221 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4222 Ty: I.getType());
4223 EVT PtrMemVT =
4224 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i: 0)->getType());
4225 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
4226 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
4227 setValue(V: &I, NewN: N);
4228}
4229
4230void SelectionDAGBuilder::visitIntToPtr(const User &I) {
4231 // What to do depends on the size of the integer and the size of the pointer.
4232 // We can either truncate, zero extend, or no-op, accordingly.
4233 SDValue N = getValue(V: I.getOperand(i: 0));
4234 auto &TLI = DAG.getTargetLoweringInfo();
4235 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4236 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4237 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
4238 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
4239 setValue(V: &I, NewN: N);
4240}
4241
4242void SelectionDAGBuilder::visitBitCast(const User &I) {
4243 SDValue N = getValue(V: I.getOperand(i: 0));
4244 SDLoc dl = getCurSDLoc();
4245 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4246 Ty: I.getType());
4247
4248 // BitCast assures us that source and destination are the same size so this is
4249 // either a BITCAST or a no-op.
4250 if (DestVT != N.getValueType())
4251 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITCAST, DL: dl,
4252 VT: DestVT, Operand: N)); // convert types.
4253 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
4254 // might fold any kind of constant expression to an integer constant and that
4255 // is not what we are looking for. Only recognize a bitcast of a genuine
4256 // constant integer as an opaque constant.
4257 else if(ConstantInt *C = dyn_cast<ConstantInt>(Val: I.getOperand(i: 0)))
4258 setValue(V: &I, NewN: DAG.getConstant(Val: C->getValue(), DL: dl, VT: DestVT, /*isTarget=*/false,
4259 /*isOpaque*/true));
4260 else
4261 setValue(V: &I, NewN: N); // noop cast.
4262}
4263
4264void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
4265 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4266 const Value *SV = I.getOperand(i: 0);
4267 SDValue N = getValue(V: SV);
4268 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4269
4270 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
4271 unsigned DestAS = I.getType()->getPointerAddressSpace();
4272
4273 if (!TM.isNoopAddrSpaceCast(DL: DAG.getDataLayout(), SrcAS, DestAS)) {
4274 SDNodeFlags Flags;
4275 if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &I))
4276 Flags.setNonNull(ASC->hasNonNull());
4277 N = DAG.getAddrSpaceCast(dl: getCurSDLoc(), VT: DestVT, Ptr: N, SrcAS, DestAS, Flags);
4278 }
4279
4280 setValue(V: &I, NewN: N);
4281}
4282
4283void SelectionDAGBuilder::visitInsertElement(const User &I) {
4284 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4285 SDValue InVec = getValue(V: I.getOperand(i: 0));
4286 SDValue InVal = getValue(V: I.getOperand(i: 1));
4287 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 2)), DL: getCurSDLoc(),
4288 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
4289 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: getCurSDLoc(),
4290 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
4291 N1: InVec, N2: InVal, N3: InIdx));
4292}
4293
4294void SelectionDAGBuilder::visitBitInsert(const User &I) {
4295 SDValue Base = getValue(V: I.getOperand(i: 0));
4296 SDValue Val = getValue(V: I.getOperand(i: 1));
4297 SDValue Offset = getValue(V: I.getOperand(i: 2));
4298 EVT BaseVT = Base.getValueType();
4299 EVT ValVT = Val.getValueType();
4300 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4301 SDLoc dl = getCurSDLoc();
4302
4303 assert(BaseVT.getScalarSizeInBits() >= ValVT.getScalarSizeInBits() &&
4304 "bitinsert val wider than base should be rejected by verifier");
4305
4306 // If Val is a float, cast it to an integer of the same bitwidth
4307 // so DAG.getZExtOrTrunc can process it safely.
4308 if (!ValVT.isInteger()) {
4309 ValVT = ValVT.changeTypeToInteger();
4310 Val = DAG.getBitcast(VT: ValVT, V: Val);
4311 }
4312
4313 // Legalize shift amount to the target's shift amount type.
4314 EVT ShiftAmtTy = TLI.getShiftAmountTy(LHSTy: BaseVT, DL: DAG.getDataLayout());
4315 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Op: Offset, DL: dl, VT: ShiftAmtTy);
4316
4317 unsigned BaseBitWidth = BaseVT.getScalarSizeInBits();
4318 unsigned ValBitWidth = ValVT.getScalarSizeInBits();
4319 APInt InsertMask = APInt::getLowBitsSet(numBits: BaseBitWidth, loBitsSet: ValBitWidth);
4320 SDValue ShiftedMask =
4321 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: BaseVT, N1: DAG.getConstant(Val: InsertMask, DL: dl, VT: BaseVT),
4322 N2: LegalShiftAmount);
4323 SDValue ClearMask = DAG.getNOT(DL: dl, Val: ShiftedMask, VT: BaseVT);
4324 SDValue ClearedBase = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BaseVT, N1: Base, N2: ClearMask);
4325
4326 SDValue ExtVal = DAG.getZExtOrTrunc(Op: Val, DL: dl, VT: BaseVT);
4327 SDValue ShiftedVal =
4328 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: BaseVT, N1: ExtVal, N2: LegalShiftAmount);
4329 SDValue Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BaseVT, N1: ClearedBase, N2: ShiftedVal);
4330 setValue(V: &I, NewN: Result);
4331}
4332
4333void SelectionDAGBuilder::visitBitExtract(const User &I) {
4334 SDValue Src = getValue(V: I.getOperand(i: 0));
4335 SDValue Offset = getValue(V: I.getOperand(i: 1));
4336 EVT SrcVT = Src.getValueType();
4337 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4338 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4339 SDLoc dl = getCurSDLoc();
4340
4341 assert(ResultVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() &&
4342 "bitextract result wider than source should be rejected by verifier");
4343
4344 // Legalize shift amount to the target's shift amount type.
4345 EVT ShiftAmtTy = TLI.getShiftAmountTy(LHSTy: SrcVT, DL: DAG.getDataLayout());
4346 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Op: Offset, DL: dl, VT: ShiftAmtTy);
4347
4348 // Shift right by Offset - brings target field to bit 0
4349 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Src, N2: LegalShiftAmount);
4350
4351 SDValue Result;
4352 if (!ResultVT.isInteger()) {
4353 // Drop into the integer domain to safely truncate the shifted bits
4354 EVT IntResultVT = ResultVT.changeTypeToInteger();
4355 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: IntResultVT, Operand: Shifted);
4356 Result = DAG.getBitcast(VT: ResultVT, V: Result);
4357 } else {
4358 // Normal integer path
4359 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResultVT, Operand: Shifted);
4360 }
4361
4362 setValue(V: &I, NewN: Result);
4363}
4364
4365void SelectionDAGBuilder::visitExtractElement(const User &I) {
4366 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4367 SDValue InVec = getValue(V: I.getOperand(i: 0));
4368 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 1)), DL: getCurSDLoc(),
4369 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
4370 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: getCurSDLoc(),
4371 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
4372 N1: InVec, N2: InIdx));
4373}
4374
4375void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4376 SDValue Src1 = getValue(V: I.getOperand(i: 0));
4377 SDValue Src2 = getValue(V: I.getOperand(i: 1));
4378 ArrayRef<int> Mask;
4379 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &I))
4380 Mask = SVI->getShuffleMask();
4381 else
4382 Mask = cast<ConstantExpr>(Val: I).getShuffleMask();
4383 SDLoc DL = getCurSDLoc();
4384 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4385 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4386 EVT SrcVT = Src1.getValueType();
4387
4388 if (all_of(Range&: Mask, P: equal_to(Arg: 0)) && VT.isScalableVector()) {
4389 // Canonical splat form of first element of first input vector.
4390 SDValue FirstElt =
4391 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SrcVT.getScalarType(), N1: Src1,
4392 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4393 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: FirstElt));
4394 return;
4395 }
4396
4397 // For now, we only handle splats for scalable vectors.
4398 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4399 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4400 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4401
4402 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4403 unsigned MaskNumElts = Mask.size();
4404
4405 if (SrcNumElts == MaskNumElts) {
4406 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask));
4407 return;
4408 }
4409
4410 // Normalize the shuffle vector since mask and vector length don't match.
4411 if (SrcNumElts < MaskNumElts) {
4412 // Mask is longer than the source vectors. We can use concatenate vector to
4413 // make the mask and vectors lengths match.
4414
4415 if (MaskNumElts % SrcNumElts == 0) {
4416 // Mask length is a multiple of the source vector length.
4417 // Check if the shuffle is some kind of concatenation of the input
4418 // vectors.
4419 unsigned NumConcat = MaskNumElts / SrcNumElts;
4420 bool IsConcat = true;
4421 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4422 for (unsigned i = 0; i != MaskNumElts; ++i) {
4423 int Idx = Mask[i];
4424 if (Idx < 0)
4425 continue;
4426 // Ensure the indices in each SrcVT sized piece are sequential and that
4427 // the same source is used for the whole piece.
4428 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4429 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4430 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4431 IsConcat = false;
4432 break;
4433 }
4434 // Remember which source this index came from.
4435 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4436 }
4437
4438 // The shuffle is concatenating multiple vectors together. Just emit
4439 // a CONCAT_VECTORS operation.
4440 if (IsConcat) {
4441 SmallVector<SDValue, 8> ConcatOps;
4442 for (auto Src : ConcatSrcs) {
4443 if (Src < 0)
4444 ConcatOps.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
4445 else if (Src == 0)
4446 ConcatOps.push_back(Elt: Src1);
4447 else
4448 ConcatOps.push_back(Elt: Src2);
4449 }
4450 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps));
4451 return;
4452 }
4453 }
4454
4455 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
4456 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4457 EVT PaddedVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(),
4458 NumElements: PaddedMaskNumElts);
4459
4460 // Pad both vectors with undefs to make them the same length as the mask.
4461 SDValue UndefVal = DAG.getUNDEF(VT: SrcVT);
4462
4463 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4464 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4465 MOps1[0] = Src1;
4466 MOps2[0] = Src2;
4467
4468 Src1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps1);
4469 Src2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps2);
4470
4471 // Readjust mask for new input vector length.
4472 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4473 for (unsigned i = 0; i != MaskNumElts; ++i) {
4474 int Idx = Mask[i];
4475 if (Idx >= (int)SrcNumElts)
4476 Idx -= SrcNumElts - PaddedMaskNumElts;
4477 MappedOps[i] = Idx;
4478 }
4479
4480 SDValue Result = DAG.getVectorShuffle(VT: PaddedVT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps);
4481
4482 // If the concatenated vector was padded, extract a subvector with the
4483 // correct number of elements.
4484 if (MaskNumElts != PaddedMaskNumElts)
4485 Result = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Result,
4486 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4487
4488 setValue(V: &I, NewN: Result);
4489 return;
4490 }
4491
4492 assert(SrcNumElts > MaskNumElts);
4493
4494 // Analyze the access pattern of the vector to see if we can extract
4495 // two subvectors and do the shuffle.
4496 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4497 bool CanExtract = true;
4498 for (int Idx : Mask) {
4499 unsigned Input = 0;
4500 if (Idx < 0)
4501 continue;
4502
4503 if (Idx >= (int)SrcNumElts) {
4504 Input = 1;
4505 Idx -= SrcNumElts;
4506 }
4507
4508 // If all the indices come from the same MaskNumElts sized portion of
4509 // the sources we can use extract. Also make sure the extract wouldn't
4510 // extract past the end of the source.
4511 int NewStartIdx = alignDown(Value: Idx, Align: MaskNumElts);
4512 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4513 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4514 CanExtract = false;
4515 // Make sure we always update StartIdx as we use it to track if all
4516 // elements are undef.
4517 StartIdx[Input] = NewStartIdx;
4518 }
4519
4520 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4521 setValue(V: &I, NewN: DAG.getUNDEF(VT)); // Vectors are not used.
4522 return;
4523 }
4524 if (CanExtract) {
4525 // Extract appropriate subvector and generate a vector shuffle
4526 for (unsigned Input = 0; Input < 2; ++Input) {
4527 SDValue &Src = Input == 0 ? Src1 : Src2;
4528 if (StartIdx[Input] < 0)
4529 Src = DAG.getUNDEF(VT);
4530 else {
4531 Src = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Src,
4532 N2: DAG.getVectorIdxConstant(Val: StartIdx[Input], DL));
4533 }
4534 }
4535
4536 // Calculate new mask.
4537 SmallVector<int, 8> MappedOps(Mask);
4538 for (int &Idx : MappedOps) {
4539 if (Idx >= (int)SrcNumElts)
4540 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4541 else if (Idx >= 0)
4542 Idx -= StartIdx[0];
4543 }
4544
4545 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps));
4546 return;
4547 }
4548
4549 // We can't use either concat vectors or extract subvectors so fall back to
4550 // replacing the shuffle with extract and build vector.
4551 // to insert and build vector.
4552 EVT EltVT = VT.getVectorElementType();
4553 SmallVector<SDValue,8> Ops;
4554 for (int Idx : Mask) {
4555 SDValue Res;
4556
4557 if (Idx < 0) {
4558 Res = DAG.getUNDEF(VT: EltVT);
4559 } else {
4560 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4561 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4562
4563 Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Src,
4564 N2: DAG.getVectorIdxConstant(Val: Idx, DL));
4565 }
4566
4567 Ops.push_back(Elt: Res);
4568 }
4569
4570 setValue(V: &I, NewN: DAG.getBuildVector(VT, DL, Ops));
4571}
4572
4573void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4574 ArrayRef<unsigned> Indices = I.getIndices();
4575 const Value *Op0 = I.getOperand(i_nocapture: 0);
4576 const Value *Op1 = I.getOperand(i_nocapture: 1);
4577 Type *AggTy = I.getType();
4578 Type *ValTy = Op1->getType();
4579 bool IntoUndef = isa<UndefValue>(Val: Op0);
4580 bool FromUndef = isa<UndefValue>(Val: Op1);
4581
4582 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4583
4584 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4585 SmallVector<EVT, 4> AggValueVTs;
4586 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: AggTy, ValueVTs&: AggValueVTs);
4587 SmallVector<EVT, 4> ValValueVTs;
4588 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4589
4590 unsigned NumAggValues = AggValueVTs.size();
4591 unsigned NumValValues = ValValueVTs.size();
4592 SmallVector<SDValue, 4> Values(NumAggValues);
4593
4594 // Ignore an insertvalue that produces an empty object
4595 if (!NumAggValues) {
4596 setValue(V: &I, NewN: DAG.getUNDEF(VT: MVT(MVT::Other)));
4597 return;
4598 }
4599
4600 SDValue Agg = getValue(V: Op0);
4601 unsigned i = 0;
4602 // Copy the beginning value(s) from the original aggregate.
4603 for (; i != LinearIndex; ++i)
4604 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4605 SDValue(Agg.getNode(), Agg.getResNo() + i);
4606 // Copy values from the inserted value(s).
4607 if (NumValValues) {
4608 SDValue Val = getValue(V: Op1);
4609 for (; i != LinearIndex + NumValValues; ++i)
4610 Values[i] = FromUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4611 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4612 }
4613 // Copy remaining value(s) from the original aggregate.
4614 for (; i != NumAggValues; ++i)
4615 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4616 SDValue(Agg.getNode(), Agg.getResNo() + i);
4617
4618 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4619 VTList: DAG.getVTList(VTs: AggValueVTs), Ops: Values));
4620}
4621
4622void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4623 ArrayRef<unsigned> Indices = I.getIndices();
4624 const Value *Op0 = I.getOperand(i_nocapture: 0);
4625 Type *AggTy = Op0->getType();
4626 Type *ValTy = I.getType();
4627 bool OutOfUndef = isa<UndefValue>(Val: Op0);
4628
4629 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4630
4631 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4632 SmallVector<EVT, 4> ValValueVTs;
4633 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4634
4635 unsigned NumValValues = ValValueVTs.size();
4636
4637 // Ignore a extractvalue that produces an empty object
4638 if (!NumValValues) {
4639 setValue(V: &I, NewN: DAG.getUNDEF(VT: MVT(MVT::Other)));
4640 return;
4641 }
4642
4643 SmallVector<SDValue, 4> Values(NumValValues);
4644
4645 SDValue Agg = getValue(V: Op0);
4646 // Copy out the selected value(s).
4647 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4648 Values[i - LinearIndex] =
4649 OutOfUndef ?
4650 DAG.getUNDEF(VT: Agg.getNode()->getValueType(ResNo: Agg.getResNo() + i)) :
4651 SDValue(Agg.getNode(), Agg.getResNo() + i);
4652
4653 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4654 VTList: DAG.getVTList(VTs: ValValueVTs), Ops: Values));
4655}
4656
4657void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4658 Value *Op0 = I.getOperand(i: 0);
4659 // Note that the pointer operand may be a vector of pointers. Take the scalar
4660 // element which holds a pointer.
4661 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4662 SDValue N = getValue(V: Op0);
4663 SDLoc dl = getCurSDLoc();
4664 auto &TLI = DAG.getTargetLoweringInfo();
4665 GEPNoWrapFlags NW = cast<GEPOperator>(Val: I).getNoWrapFlags();
4666
4667 // For a vector GEP, keep the prefix scalar as long as possible, then
4668 // convert any scalars encountered after the first vector operand to vectors.
4669 bool IsVectorGEP = I.getType()->isVectorTy();
4670 ElementCount VectorElementCount =
4671 IsVectorGEP ? cast<VectorType>(Val: I.getType())->getElementCount()
4672 : ElementCount::getFixed(MinVal: 0);
4673
4674 for (gep_type_iterator GTI = gep_type_begin(GEP: &I), E = gep_type_end(GEP: &I);
4675 GTI != E; ++GTI) {
4676 const Value *Idx = GTI.getOperand();
4677 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4678 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
4679 if (Field) {
4680 // N = N + Offset
4681 uint64_t Offset =
4682 DAG.getDataLayout().getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
4683
4684 // In an inbounds GEP with an offset that is nonnegative even when
4685 // interpreted as signed, assume there is no unsigned overflow.
4686 SDNodeFlags Flags;
4687 if (NW.hasNoUnsignedWrap() ||
4688 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4689 Flags |= SDNodeFlags::NoUnsignedWrap;
4690 Flags.setInBounds(NW.isInBounds());
4691
4692 N = DAG.getMemBasePlusOffset(
4693 Base: N, Offset: DAG.getConstant(Val: Offset, DL: dl, VT: N.getValueType()), DL: dl, Flags);
4694 }
4695 } else {
4696 // IdxSize is the width of the arithmetic according to IR semantics.
4697 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4698 // (and fix up the result later).
4699 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4700 MVT IdxTy = MVT::getIntegerVT(BitWidth: IdxSize);
4701 TypeSize ElementSize =
4702 GTI.getSequentialElementStride(DL: DAG.getDataLayout());
4703 // We intentionally mask away the high bits here; ElementSize may not
4704 // fit in IdxTy.
4705 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4706 /*isSigned=*/false, /*implicitTrunc=*/true);
4707 bool ElementScalable = ElementSize.isScalable();
4708
4709 // If this is a scalar constant or a splat vector of constants,
4710 // handle it quickly.
4711 const auto *C = dyn_cast<Constant>(Val: Idx);
4712 if (C && isa<VectorType>(Val: C->getType()))
4713 C = C->getSplatValue();
4714
4715 const auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
4716 if (CI && CI->isZero())
4717 continue;
4718 if (CI && !ElementScalable) {
4719 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(width: IdxSize);
4720 LLVMContext &Context = *DAG.getContext();
4721 SDValue OffsVal;
4722 if (N.getValueType().isVector())
4723 OffsVal = DAG.getConstant(
4724 Val: Offs, DL: dl, VT: EVT::getVectorVT(Context, VT: IdxTy, EC: VectorElementCount));
4725 else
4726 OffsVal = DAG.getConstant(Val: Offs, DL: dl, VT: IdxTy);
4727
4728 // In an inbounds GEP with an offset that is nonnegative even when
4729 // interpreted as signed, assume there is no unsigned overflow.
4730 SDNodeFlags Flags;
4731 if (NW.hasNoUnsignedWrap() ||
4732 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4733 Flags.setNoUnsignedWrap(true);
4734 Flags.setInBounds(NW.isInBounds());
4735
4736 OffsVal = DAG.getSExtOrTrunc(Op: OffsVal, DL: dl, VT: N.getValueType());
4737
4738 N = DAG.getMemBasePlusOffset(Base: N, Offset: OffsVal, DL: dl, Flags);
4739 continue;
4740 }
4741
4742 // N = N + Idx * ElementMul;
4743 SDValue IdxN = getValue(V: Idx);
4744
4745 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
4746 if (N.getValueType().isVector()) {
4747 EVT VT = EVT::getVectorVT(Context&: *Context, VT: IdxN.getValueType(),
4748 EC: VectorElementCount);
4749 IdxN = DAG.getSplat(VT, DL: dl, Op: IdxN);
4750 } else {
4751 EVT VT =
4752 EVT::getVectorVT(Context&: *Context, VT: N.getValueType(), EC: VectorElementCount);
4753 N = DAG.getSplat(VT, DL: dl, Op: N);
4754 }
4755 }
4756
4757 // If the index is smaller or larger than intptr_t, truncate or extend
4758 // it.
4759 IdxN = DAG.getSExtOrTrunc(Op: IdxN, DL: dl, VT: N.getValueType());
4760
4761 SDNodeFlags ScaleFlags;
4762 // The multiplication of an index by the type size does not wrap the
4763 // pointer index type in a signed sense (mul nsw).
4764 ScaleFlags.setNoSignedWrap(NW.hasNoUnsignedSignedWrap());
4765
4766 // The multiplication of an index by the type size does not wrap the
4767 // pointer index type in an unsigned sense (mul nuw).
4768 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4769
4770 if (ElementScalable) {
4771 EVT VScaleTy = N.getValueType().getScalarType();
4772 SDValue VScale = DAG.getNode(
4773 Opcode: ISD::VSCALE, DL: dl, VT: VScaleTy,
4774 Operand: DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl, VT: VScaleTy));
4775 if (N.getValueType().isVector())
4776 VScale = DAG.getSplatVector(VT: N.getValueType(), DL: dl, Op: VScale);
4777 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: VScale,
4778 Flags: ScaleFlags);
4779 } else {
4780 // If this is a multiply by a power of two, turn it into a shl
4781 // immediately. This is a very common case.
4782 if (ElementMul != 1) {
4783 if (ElementMul.isPowerOf2()) {
4784 unsigned Amt = ElementMul.logBase2();
4785 IdxN = DAG.getNode(
4786 Opcode: ISD::SHL, DL: dl, VT: N.getValueType(), N1: IdxN,
4787 N2: DAG.getShiftAmountConstant(Val: Amt, VT: N.getValueType(), DL: dl),
4788 Flags: ScaleFlags);
4789 } else {
4790 SDValue Scale = DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl,
4791 VT: IdxN.getValueType());
4792 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: Scale,
4793 Flags: ScaleFlags);
4794 }
4795 }
4796 }
4797
4798 // The successive addition of the current address, truncated to the
4799 // pointer index type and interpreted as an unsigned number, and each
4800 // offset, also interpreted as an unsigned number, does not wrap the
4801 // pointer index type (add nuw).
4802 SDNodeFlags AddFlags;
4803 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4804 AddFlags.setInBounds(NW.isInBounds());
4805
4806 N = DAG.getMemBasePlusOffset(Base: N, Offset: IdxN, DL: dl, Flags: AddFlags);
4807 }
4808 }
4809
4810 if (IsVectorGEP && !N.getValueType().isVector()) {
4811 EVT VT = EVT::getVectorVT(Context&: *Context, VT: N.getValueType(), EC: VectorElementCount);
4812 N = DAG.getSplat(VT, DL: dl, Op: N);
4813 }
4814
4815 MVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout(), AS);
4816 MVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout(), AS);
4817 if (IsVectorGEP) {
4818 PtrTy = MVT::getVectorVT(VT: PtrTy, EC: VectorElementCount);
4819 PtrMemTy = MVT::getVectorVT(VT: PtrMemTy, EC: VectorElementCount);
4820 }
4821
4822 if (PtrMemTy != PtrTy && !cast<GEPOperator>(Val: I).isInBounds())
4823 N = DAG.getPtrExtendInReg(Op: N, DL: dl, VT: PtrMemTy);
4824
4825 setValue(V: &I, NewN: N);
4826}
4827
4828void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4829 // If this is a fixed sized alloca in the entry block of the function,
4830 // allocate it statically on the stack.
4831 if (FuncInfo.StaticAllocaMap.count(Val: &I))
4832 return; // getValue will auto-populate this.
4833
4834 SDLoc dl = getCurSDLoc();
4835 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4836 auto &DL = DAG.getDataLayout();
4837 TypeSize TySize = I.getAllocationBaseSize(DL);
4838 MaybeAlign Alignment = I.getAlign();
4839
4840 SDValue AllocSize = getValue(V: I.getArraySize());
4841
4842 EVT IntPtr = TLI.getPointerTy(DL, AS: I.getAddressSpace());
4843 if (AllocSize.getValueType() != IntPtr)
4844 AllocSize = DAG.getZExtOrTrunc(Op: AllocSize, DL: dl, VT: IntPtr);
4845
4846 AllocSize = DAG.getNode(
4847 Opcode: ISD::MUL, DL: dl, VT: IntPtr, N1: AllocSize,
4848 N2: DAG.getZExtOrTrunc(Op: DAG.getTypeSize(DL: dl, VT: MVT::i64, TS: TySize), DL: dl, VT: IntPtr));
4849
4850 // Handle alignment. If the requested alignment is less than or equal to
4851 // the stack alignment, ignore it. If the size is greater than or equal to
4852 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4853 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4854 if (*Alignment <= StackAlign)
4855 Alignment = std::nullopt;
4856
4857 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4858 // Round the size of the allocation up to the stack alignment size
4859 // by add SA-1 to the size. This doesn't overflow because we're computing
4860 // an address inside an alloca.
4861 AllocSize = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4862 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT: IntPtr),
4863 Flags: SDNodeFlags::NoUnsignedWrap);
4864
4865 // Mask out the low bits for alignment purposes.
4866 AllocSize = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4867 N2: DAG.getSignedConstant(Val: ~StackAlignMask, DL: dl, VT: IntPtr));
4868
4869 SDValue Ops[] = {
4870 getRoot(), AllocSize,
4871 DAG.getConstant(Val: Alignment ? Alignment->value() : 0, DL: dl, VT: IntPtr)};
4872 SDVTList VTs = DAG.getVTList(VT1: AllocSize.getValueType(), VT2: MVT::Other);
4873 SDValue DSA = DAG.getNode(Opcode: ISD::DYNAMIC_STACKALLOC, DL: dl, VTList: VTs, Ops);
4874 setValue(V: &I, NewN: DSA);
4875 DAG.setRoot(DSA.getValue(R: 1));
4876
4877 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4878}
4879
4880static const MDNode *getRangeMetadata(const Instruction &I) {
4881 return I.getMetadata(KindID: LLVMContext::MD_range);
4882}
4883
4884static std::optional<ConstantRange> getRange(const Instruction &I) {
4885 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4886 if (std::optional<ConstantRange> CR = CB->getRange())
4887 return CR;
4888 if (const MDNode *Range = getRangeMetadata(I))
4889 return getConstantRangeFromMetadata(RangeMD: *Range);
4890 return std::nullopt;
4891}
4892
4893static FPClassTest getNoFPClass(const Instruction &I) {
4894 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4895 return CB->getRetNoFPClass();
4896 return fcNone;
4897}
4898
4899void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4900 if (I.isAtomic())
4901 return visitAtomicLoad(I);
4902
4903 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4904 const Value *SV = I.getOperand(i_nocapture: 0);
4905 if (TLI.supportSwiftError()) {
4906 // Swifterror values can come from either a function parameter with
4907 // swifterror attribute or an alloca with swifterror attribute.
4908 if (const Argument *Arg = dyn_cast<Argument>(Val: SV)) {
4909 if (Arg->hasSwiftErrorAttr())
4910 return visitLoadFromSwiftError(I);
4911 }
4912
4913 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: SV)) {
4914 if (Alloca->isSwiftError())
4915 return visitLoadFromSwiftError(I);
4916 }
4917 }
4918
4919 SDValue Ptr = getValue(V: SV);
4920
4921 Type *Ty = I.getType();
4922 SmallVector<EVT, 4> ValueVTs, MemVTs;
4923 SmallVector<TypeSize, 4> Offsets;
4924 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty, ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
4925 unsigned NumValues = ValueVTs.size();
4926 if (NumValues == 0)
4927 return;
4928
4929 Align Alignment = I.getAlign();
4930 AAMDNodes AAInfo = I.getAAMetadata();
4931 const MDNode *Ranges = getRangeMetadata(I);
4932 const MDNode *MemCacheHint = getMemCacheHint(I);
4933 bool isVolatile = I.isVolatile();
4934 MachineMemOperand::Flags MMOFlags =
4935 TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
4936
4937 SDValue Root;
4938 bool ConstantMemory = false;
4939 if (isVolatile)
4940 // Serialize volatile loads with other side effects.
4941 Root = getRoot();
4942 else if (NumValues > MaxParallelChains)
4943 Root = getMemoryRoot();
4944 else if (BatchAA &&
4945 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(
4946 SV,
4947 LocationSize::precise(Value: DAG.getDataLayout().getTypeStoreSize(Ty)),
4948 AAInfo))) {
4949 // Do not serialize (non-volatile) loads of constant memory with anything.
4950 Root = DAG.getEntryNode();
4951 ConstantMemory = true;
4952 MMOFlags |= MachineMemOperand::MOInvariant;
4953 } else {
4954 // Do not serialize non-volatile loads against each other.
4955 Root = DAG.getRoot();
4956 }
4957
4958 SDLoc dl = getCurSDLoc();
4959
4960 if (isVolatile)
4961 Root = TLI.prepareVolatileOrAtomicLoad(Chain: Root, DL: dl, DAG);
4962
4963 SmallVector<SDValue, 4> Values(NumValues);
4964 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
4965
4966 unsigned ChainI = 0;
4967 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4968 // Serializing loads here may result in excessive register pressure, and
4969 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4970 // could recover a bit by hoisting nodes upward in the chain by recognizing
4971 // they are side-effect free or do not alias. The optimizer should really
4972 // avoid this case by converting large object/array copies to llvm.memcpy
4973 // (MaxParallelChains should always remain as failsafe).
4974 if (ChainI == MaxParallelChains) {
4975 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4976 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4977 Ops: ArrayRef(Chains.data(), ChainI));
4978 Root = Chain;
4979 ChainI = 0;
4980 }
4981
4982 // TODO: MachinePointerInfo only supports a fixed length offset.
4983 MachinePointerInfo PtrInfo =
4984 !Offsets[i].isScalable() || Offsets[i].isZero()
4985 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4986 : MachinePointerInfo();
4987
4988 SDValue A = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
4989 SDValue L =
4990 DAG.getLoad(VT: MemVTs[i], dl, Chain: Root, Ptr: A, PtrInfo, Alignment, MMOFlags,
4991 Metadata: MMOMetadata(AAInfo, Ranges, MemCacheHint));
4992 Chains[ChainI] = L.getValue(R: 1);
4993
4994 if (MemVTs[i] != ValueVTs[i])
4995 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT: ValueVTs[i]);
4996
4997 if (MDNode *NoFPClassMD = I.getMetadata(KindID: LLVMContext::MD_nofpclass)) {
4998 uint64_t FPTestInt =
4999 cast<ConstantInt>(
5000 Val: cast<ConstantAsMetadata>(Val: NoFPClassMD->getOperand(I: 0))->getValue())
5001 ->getZExtValue();
5002 if (FPTestInt != fcNone) {
5003 SDValue FPTestConst =
5004 DAG.getTargetConstant(Val: FPTestInt, DL: SDLoc(), VT: MVT::i32);
5005 L = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: L.getValueType(), N1: L,
5006 N2: FPTestConst);
5007 }
5008 }
5009 Values[i] = L;
5010 }
5011
5012 if (!ConstantMemory) {
5013 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
5014 Ops: ArrayRef(Chains.data(), ChainI));
5015 if (isVolatile)
5016 DAG.setRoot(Chain);
5017 else
5018 PendingLoads.push_back(Elt: Chain);
5019 }
5020
5021 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
5022 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
5023}
5024
5025void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
5026 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5027 "call visitStoreToSwiftError when backend supports swifterror");
5028
5029 SmallVector<EVT, 4> ValueVTs;
5030 SmallVector<uint64_t, 4> Offsets;
5031 const Value *SrcV = I.getOperand(i_nocapture: 0);
5032 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
5033 Ty: SrcV->getType(), ValueVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
5034 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5035 "expect a single EVT for swifterror");
5036
5037 SDValue Src = getValue(V: SrcV);
5038 // Create a virtual register, then update the virtual register.
5039 Register VReg =
5040 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
5041 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
5042 // Chain can be getRoot or getControlRoot.
5043 SDValue CopyNode = DAG.getCopyToReg(Chain: getRoot(), dl: getCurSDLoc(), Reg: VReg,
5044 N: SDValue(Src.getNode(), Src.getResNo()));
5045 DAG.setRoot(CopyNode);
5046}
5047
5048void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
5049 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5050 "call visitLoadFromSwiftError when backend supports swifterror");
5051
5052 assert(!I.isVolatile() &&
5053 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
5054 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
5055 "Support volatile, non temporal, invariant for load_from_swift_error");
5056
5057 const Value *SV = I.getOperand(i_nocapture: 0);
5058 Type *Ty = I.getType();
5059 assert(
5060 (!BatchAA ||
5061 !BatchAA->pointsToConstantMemory(MemoryLocation(
5062 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
5063 I.getAAMetadata()))) &&
5064 "load_from_swift_error should not be constant memory");
5065
5066 SmallVector<EVT, 4> ValueVTs;
5067 SmallVector<uint64_t, 4> Offsets;
5068 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty,
5069 ValueVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
5070 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5071 "expect a single EVT for swifterror");
5072
5073 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
5074 SDValue L = DAG.getCopyFromReg(
5075 Chain: getRoot(), dl: getCurSDLoc(),
5076 Reg: SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), VT: ValueVTs[0]);
5077
5078 setValue(V: &I, NewN: L);
5079}
5080
5081void SelectionDAGBuilder::visitStore(const StoreInst &I) {
5082 if (I.isAtomic())
5083 return visitAtomicStore(I);
5084
5085 const Value *SrcV = I.getOperand(i_nocapture: 0);
5086 const Value *PtrV = I.getOperand(i_nocapture: 1);
5087
5088 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5089 if (TLI.supportSwiftError()) {
5090 // Swifterror values can come from either a function parameter with
5091 // swifterror attribute or an alloca with swifterror attribute.
5092 if (const Argument *Arg = dyn_cast<Argument>(Val: PtrV)) {
5093 if (Arg->hasSwiftErrorAttr())
5094 return visitStoreToSwiftError(I);
5095 }
5096
5097 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: PtrV)) {
5098 if (Alloca->isSwiftError())
5099 return visitStoreToSwiftError(I);
5100 }
5101 }
5102
5103 SmallVector<EVT, 4> ValueVTs, MemVTs;
5104 SmallVector<TypeSize, 4> Offsets;
5105 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
5106 Ty: SrcV->getType(), ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
5107 unsigned NumValues = ValueVTs.size();
5108 if (NumValues == 0)
5109 return;
5110
5111 // Get the lowered operands. Note that we do this after
5112 // checking if NumResults is zero, because with zero results
5113 // the operands won't have values in the map.
5114 SDValue Src = getValue(V: SrcV);
5115 SDValue Ptr = getValue(V: PtrV);
5116
5117 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
5118 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
5119 SDLoc dl = getCurSDLoc();
5120 Align Alignment = I.getAlign();
5121 AAMDNodes AAInfo = I.getAAMetadata();
5122 const MDNode *MemCacheHint = getMemCacheHint(I, OperandNo: I.getPointerOperandIndex());
5123
5124 auto MMOFlags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
5125
5126 unsigned ChainI = 0;
5127 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
5128 // See visitLoad comments.
5129 if (ChainI == MaxParallelChains) {
5130 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
5131 Ops: ArrayRef(Chains.data(), ChainI));
5132 Root = Chain;
5133 ChainI = 0;
5134 }
5135
5136 // TODO: MachinePointerInfo only supports a fixed length offset.
5137 MachinePointerInfo PtrInfo =
5138 !Offsets[i].isScalable() || Offsets[i].isZero()
5139 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
5140 : MachinePointerInfo();
5141
5142 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
5143 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
5144 if (MemVTs[i] != ValueVTs[i])
5145 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVTs[i]);
5146 SDValue St =
5147 DAG.getStore(Chain: Root, dl, Val, Ptr: Add, PtrInfo, Alignment, MMOFlags,
5148 Metadata: MMOMetadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
5149 Chains[ChainI] = St;
5150 }
5151
5152 SDValue StoreNode = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
5153 Ops: ArrayRef(Chains.data(), ChainI));
5154 setValue(V: &I, NewN: StoreNode);
5155 DAG.setRoot(StoreNode);
5156}
5157
5158void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
5159 bool IsCompressing) {
5160 SDLoc sdl = getCurSDLoc();
5161
5162 Value *Src0Operand = I.getArgOperand(i: 0);
5163 Value *PtrOperand = I.getArgOperand(i: 1);
5164 Value *MaskOperand = I.getArgOperand(i: 2);
5165 Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
5166
5167 SDValue Ptr = getValue(V: PtrOperand);
5168 SDValue Src0 = getValue(V: Src0Operand);
5169 SDValue Mask = getValue(V: MaskOperand);
5170 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
5171
5172 EVT VT = Src0.getValueType();
5173
5174 const auto &TLI = DAG.getTargetLoweringInfo();
5175
5176 auto MMOFlags = MachineMemOperand::MOStore;
5177 MMOFlags |= TLI.getTargetMMOFlags(I);
5178 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5179 MMOFlags |= MachineMemOperand::MONonTemporal;
5180
5181 const MDNode *MemCacheHint = getMemCacheHint(I, /*OperandNo=*/1);
5182
5183 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5184 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5185 Size: LocationSize::upperBound(Value: VT.getStoreSize()), BaseAlignment: Alignment,
5186 Metadata: MMOMetadata(I.getAAMetadata(), /*Ranges=*/nullptr, MemCacheHint));
5187
5188 SDValue StoreNode =
5189 !IsCompressing && TTI->hasConditionalLoadStoreForType(
5190 Ty: I.getArgOperand(i: 0)->getType(), /*IsStore=*/true)
5191 ? TLI.visitMaskedStore(DAG, DL: sdl, Chain: getMemoryRoot(), MMO, Ptr, Val: Src0,
5192 Mask)
5193 : DAG.getMaskedStore(Chain: getMemoryRoot(), dl: sdl, Val: Src0, Base: Ptr, Offset, Mask,
5194 MemVT: VT, MMO, AM: ISD::UNINDEXED, /*Truncating=*/IsTruncating: false,
5195 IsCompressing);
5196 DAG.setRoot(StoreNode);
5197 setValue(V: &I, NewN: StoreNode);
5198}
5199
5200// Get a uniform base for the Gather/Scatter intrinsic.
5201// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
5202// We try to represent it as a base pointer + vector of indices.
5203// Usually, the vector of pointers comes from a 'getelementptr' instruction.
5204// The first operand of the GEP may be a single pointer or a vector of pointers
5205// Example:
5206// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
5207// or
5208// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
5209// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
5210//
5211// When the first GEP operand is a single pointer - it is the uniform base we
5212// are looking for. If first operand of the GEP is a splat vector - we
5213// extract the splat value and use it as a uniform base.
5214// In all other cases the function returns 'false'.
5215static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
5216 SDValue &Scale, SelectionDAGBuilder *SDB,
5217 const BasicBlock *CurBB, uint64_t ElemSize) {
5218 SelectionDAG& DAG = SDB->DAG;
5219 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5220 const DataLayout &DL = DAG.getDataLayout();
5221
5222 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
5223
5224 // Handle splat constant pointer.
5225 if (auto *C = dyn_cast<Constant>(Val: Ptr)) {
5226 C = C->getSplatValue();
5227 if (!C)
5228 return false;
5229
5230 Base = SDB->getValue(V: C);
5231
5232 ElementCount NumElts = cast<VectorType>(Val: Ptr->getType())->getElementCount();
5233 EVT VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TLI.getPointerTy(DL), EC: NumElts);
5234 Index = DAG.getConstant(Val: 0, DL: SDB->getCurSDLoc(), VT);
5235 Scale = DAG.getTargetConstant(Val: 1, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
5236 return true;
5237 }
5238
5239 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
5240 if (!GEP || GEP->getParent() != CurBB)
5241 return false;
5242
5243 if (GEP->getNumOperands() != 2)
5244 return false;
5245
5246 const Value *BasePtr = GEP->getPointerOperand();
5247 const Value *IndexVal = GEP->getOperand(i_nocapture: GEP->getNumOperands() - 1);
5248
5249 // Make sure the base is scalar and the index is a vector.
5250 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
5251 return false;
5252
5253 TypeSize ScaleVal = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
5254 if (ScaleVal.isScalable())
5255 return false;
5256
5257 // Target may not support the required addressing mode.
5258 if (ScaleVal != 1 &&
5259 !TLI.isLegalScaleForGatherScatter(Scale: ScaleVal.getFixedValue(), ElemSize))
5260 return false;
5261
5262 Base = SDB->getValue(V: BasePtr);
5263 Index = SDB->getValue(V: IndexVal);
5264
5265 Scale =
5266 DAG.getTargetConstant(Val: ScaleVal, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
5267 return true;
5268}
5269
5270void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
5271 SDLoc sdl = getCurSDLoc();
5272
5273 // llvm.masked.scatter.*(Src0, Ptrs, Mask)
5274 const Value *Ptr = I.getArgOperand(i: 1);
5275 SDValue Src0 = getValue(V: I.getArgOperand(i: 0));
5276 SDValue Mask = getValue(V: I.getArgOperand(i: 2));
5277 EVT VT = Src0.getValueType();
5278 Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
5279 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5280
5281 SDValue Base;
5282 SDValue Index;
5283 SDValue Scale;
5284 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
5285 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5286
5287 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5288 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5289 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOStore,
5290 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, Metadata: I.getAAMetadata());
5291 if (!UniformBase) {
5292 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5293 Index = getValue(V: Ptr);
5294 Scale =
5295 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5296 }
5297
5298 EVT IdxVT = Index.getValueType();
5299 EVT EltTy = IdxVT.getVectorElementType();
5300 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5301 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
5302 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5303 }
5304
5305 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
5306 SDValue Scatter = DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: VT, dl: sdl,
5307 Ops, MMO, IndexType: ISD::SIGNED_SCALED, IsTruncating: false);
5308 DAG.setRoot(Scatter);
5309 setValue(V: &I, NewN: Scatter);
5310}
5311
5312void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
5313 SDLoc sdl = getCurSDLoc();
5314
5315 Value *PtrOperand = I.getArgOperand(i: 0);
5316 Value *MaskOperand = I.getArgOperand(i: 1);
5317 Value *Src0Operand = I.getArgOperand(i: 2);
5318 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5319
5320 SDValue Ptr = getValue(V: PtrOperand);
5321 SDValue Src0 = getValue(V: Src0Operand);
5322 SDValue Mask = getValue(V: MaskOperand);
5323 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
5324
5325 EVT VT = Src0.getValueType();
5326 AAMDNodes AAInfo = I.getAAMetadata();
5327 const MDNode *Ranges = getRangeMetadata(I);
5328 const MDNode *MemCacheHint = getMemCacheHint(I, /*OperandNo=*/0);
5329
5330 // Do not serialize masked loads of constant memory with anything.
5331 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
5332 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
5333
5334 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5335
5336 const auto &TLI = DAG.getTargetLoweringInfo();
5337
5338 auto MMOFlags = MachineMemOperand::MOLoad;
5339 MMOFlags |= TLI.getTargetMMOFlags(I);
5340 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5341 MMOFlags |= MachineMemOperand::MONonTemporal;
5342 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
5343 MMOFlags |= MachineMemOperand::MOInvariant;
5344
5345 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5346 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5347 Size: LocationSize::upperBound(Value: VT.getStoreSize()), BaseAlignment: Alignment,
5348 Metadata: MMOMetadata(AAInfo, Ranges, MemCacheHint));
5349
5350 // The Load/Res may point to different values and both of them are output
5351 // variables.
5352 SDValue Load;
5353 SDValue Res;
5354 if (!IsExpanding &&
5355 TTI->hasConditionalLoadStoreForType(Ty: Src0Operand->getType(),
5356 /*IsStore=*/false))
5357 Res = TLI.visitMaskedLoad(DAG, DL: sdl, Chain: InChain, MMO, NewLoad&: Load, Ptr, PassThru: Src0, Mask);
5358 else
5359 Res = Load =
5360 DAG.getMaskedLoad(VT, dl: sdl, Chain: InChain, Base: Ptr, Offset, Mask, Src0, MemVT: VT, MMO,
5361 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5362 if (AddToChain)
5363 PendingLoads.push_back(Elt: Load.getValue(R: 1));
5364 setValue(V: &I, NewN: Res);
5365}
5366
5367void SelectionDAGBuilder::visitSpeculativeLoad(const CallInst &I) {
5368 SDLoc sdl = getCurSDLoc();
5369 Value *PtrOperand = I.getArgOperand(i: 0);
5370 // The remaining arguments (num_accessible_bytes or oracle function + args)
5371 // are IR-level semantics only; they are not needed at codegen.
5372 SDValue Ptr = getValue(V: PtrOperand);
5373
5374 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5375 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5376 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5377 AAMDNodes AAInfo = I.getAAMetadata();
5378
5379 SDValue InChain = DAG.getRoot();
5380
5381 // Use MOLoad but NOT MODereferenceable - the memory may not be
5382 // fully dereferenceable.
5383 auto MMOFlags = MachineMemOperand::MOLoad;
5384 MMOFlags |= TLI.getTargetMMOFlags(I);
5385 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5386 MMOFlags |= MachineMemOperand::MONonTemporal;
5387 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
5388 MMOFlags |= MachineMemOperand::MOInvariant;
5389
5390 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5391 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5392 Size: LocationSize::precise(Value: VT.getStoreSize()), BaseAlignment: Alignment, Metadata: AAInfo);
5393
5394 SDValue Load = DAG.getLoad(VT, dl: sdl, Chain: InChain, Ptr, MMO);
5395 PendingLoads.push_back(Elt: Load.getValue(R: 1));
5396 setValue(V: &I, NewN: Load);
5397}
5398
5399void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5400 SDLoc sdl = getCurSDLoc();
5401
5402 // @llvm.masked.gather.*(Ptrs, Mask, Src0)
5403 const Value *Ptr = I.getArgOperand(i: 0);
5404 SDValue Src0 = getValue(V: I.getArgOperand(i: 2));
5405 SDValue Mask = getValue(V: I.getArgOperand(i: 1));
5406
5407 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5408 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5409 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5410
5411 const MDNode *Ranges = getRangeMetadata(I);
5412
5413 SDValue Root = DAG.getRoot();
5414 SDValue Base;
5415 SDValue Index;
5416 SDValue Scale;
5417 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
5418 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5419 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5420 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5421 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOLoad,
5422 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
5423 Metadata: MMOMetadata(I.getAAMetadata(), Ranges));
5424
5425 if (!UniformBase) {
5426 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5427 Index = getValue(V: Ptr);
5428 Scale =
5429 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5430 }
5431
5432 EVT IdxVT = Index.getValueType();
5433 EVT EltTy = IdxVT.getVectorElementType();
5434 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5435 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
5436 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5437 }
5438
5439 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5440 SDValue Gather =
5441 DAG.getMaskedGather(VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), MemVT: VT, dl: sdl, Ops, MMO,
5442 IndexType: ISD::SIGNED_SCALED, ExtTy: ISD::NON_EXTLOAD);
5443
5444 PendingLoads.push_back(Elt: Gather.getValue(R: 1));
5445 setValue(V: &I, NewN: Gather);
5446}
5447
5448void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5449 SDLoc dl = getCurSDLoc();
5450 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5451 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5452 SyncScope::ID SSID = I.getSyncScopeID();
5453
5454 SDValue InChain = getRoot();
5455
5456 MVT MemVT = getValue(V: I.getCompareOperand()).getSimpleValueType();
5457 SDVTList VTs = DAG.getVTList(VT1: MemVT, VT2: MVT::i1, VT3: MVT::Other);
5458
5459 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5460 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5461
5462 MachineFunction &MF = DAG.getMachineFunction();
5463 const MDNode *MemCacheHint = getMemCacheHint(I);
5464 MachineMemOperand *MMO = MF.getMachineMemOperand(
5465 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5466 BaseAlignment: I.getAlign(), Metadata: MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5467 SSID, Ordering: SuccessOrdering, FailureOrdering);
5468
5469 SDValue L = DAG.getAtomicCmpSwap(Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
5470 dl, MemVT, VTs, Chain: InChain,
5471 Ptr: getValue(V: I.getPointerOperand()),
5472 Cmp: getValue(V: I.getCompareOperand()),
5473 Swp: getValue(V: I.getNewValOperand()), MMO);
5474
5475 SDValue OutChain = L.getValue(R: 2);
5476
5477 setValue(V: &I, NewN: L);
5478 DAG.setRoot(OutChain);
5479}
5480
5481void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5482 SDLoc dl = getCurSDLoc();
5483 ISD::NodeType NT = getAtomicRMWOpcode(Op: I.getOperation());
5484 AtomicOrdering Ordering = I.getOrdering();
5485 SyncScope::ID SSID = I.getSyncScopeID();
5486
5487 SDValue InChain = getRoot();
5488
5489 auto MemVT = getValue(V: I.getValOperand()).getSimpleValueType();
5490 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5491 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5492
5493 MachineFunction &MF = DAG.getMachineFunction();
5494 const MDNode *MemCacheHint = getMemCacheHint(I);
5495 MachineMemOperand *MMO = MF.getMachineMemOperand(
5496 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5497 BaseAlignment: I.getAlign(), Metadata: MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5498 SSID, Ordering);
5499
5500 SDValue L =
5501 DAG.getAtomic(Opcode: NT, dl, MemVT, Chain: InChain,
5502 Ptr: getValue(V: I.getPointerOperand()), Val: getValue(V: I.getValOperand()),
5503 MMO);
5504
5505 SDValue OutChain = L.getValue(R: 1);
5506
5507 setValue(V: &I, NewN: L);
5508 DAG.setRoot(OutChain);
5509}
5510
5511void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5512 SDLoc dl = getCurSDLoc();
5513 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5514 SDValue Ops[3];
5515 Ops[0] = getRoot();
5516 Ops[1] = DAG.getTargetConstant(Val: (unsigned)I.getOrdering(), DL: dl,
5517 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5518 Ops[2] = DAG.getTargetConstant(Val: I.getSyncScopeID(), DL: dl,
5519 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5520 SDValue N = DAG.getNode(Opcode: ISD::ATOMIC_FENCE, DL: dl, VT: MVT::Other, Ops);
5521 setValue(V: &I, NewN: N);
5522 DAG.setRoot(N);
5523}
5524
5525void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5526 SDLoc dl = getCurSDLoc();
5527 AtomicOrdering Order = I.getOrdering();
5528 SyncScope::ID SSID = I.getSyncScopeID();
5529
5530 SDValue InChain = getRoot();
5531
5532 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5533 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5534 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5535
5536 if (!TLI.isAtomicAlignmentSupported(Alignment: I.getAlign(), SizeInBytes: MemVT.getSizeInBits() / 8))
5537 report_fatal_error(reason: "Cannot generate unaligned atomic load");
5538
5539 auto Flags = TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
5540
5541 const MDNode *Ranges = getRangeMetadata(I);
5542 const MDNode *MemCacheHint = getMemCacheHint(I);
5543 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5544 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5545 BaseAlignment: I.getAlign(), Metadata: MMOMetadata(AAMDNodes(), Ranges, MemCacheHint), SSID,
5546 Ordering: Order);
5547
5548 InChain = TLI.prepareVolatileOrAtomicLoad(Chain: InChain, DL: dl, DAG);
5549
5550 SDValue Ptr = getValue(V: I.getPointerOperand());
5551 SDValue L =
5552 DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT, VT: MemVT, Chain: InChain, Ptr, MMO);
5553
5554 SDValue OutChain = L.getValue(R: 1);
5555 if (MemVT != VT)
5556 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT);
5557
5558 setValue(V: &I, NewN: L);
5559 DAG.setRoot(OutChain);
5560}
5561
5562void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5563 SDLoc dl = getCurSDLoc();
5564
5565 AtomicOrdering Ordering = I.getOrdering();
5566 SyncScope::ID SSID = I.getSyncScopeID();
5567
5568 SDValue InChain = getRoot();
5569
5570 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5571 EVT MemVT =
5572 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getValueOperand()->getType());
5573
5574 if (!TLI.isAtomicAlignmentSupported(Alignment: I.getAlign(), SizeInBytes: MemVT.getSizeInBits() / 8))
5575 report_fatal_error(reason: "Cannot generate unaligned atomic store");
5576
5577 auto Flags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
5578
5579 MachineFunction &MF = DAG.getMachineFunction();
5580 const MDNode *MemCacheHint = getMemCacheHint(I, OperandNo: I.getPointerOperandIndex());
5581 MachineMemOperand *MMO = MF.getMachineMemOperand(
5582 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5583 BaseAlignment: I.getAlign(), Metadata: MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5584 SSID, Ordering);
5585
5586 SDValue Val = getValue(V: I.getValueOperand());
5587 if (Val.getValueType() != MemVT)
5588 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVT);
5589 SDValue Ptr = getValue(V: I.getPointerOperand());
5590
5591 SDValue OutChain =
5592 DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT, Chain: InChain, Ptr: Val, Val: Ptr, MMO);
5593
5594 setValue(V: &I, NewN: OutChain);
5595 DAG.setRoot(OutChain);
5596}
5597
5598/// Check if this intrinsic call depends on the chain (1st return value)
5599/// and if it only *loads* memory.
5600/// Ignore the callsite's attributes. A specific call site may be marked with
5601/// readnone, but the lowering code will expect the chain based on the
5602/// definition.
5603std::pair<bool, bool>
5604SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {
5605 const Function *F = I.getCalledFunction();
5606 bool HasChain = !F->doesNotAccessMemory();
5607 bool OnlyLoad =
5608 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5609
5610 return {HasChain, OnlyLoad};
5611}
5612
5613SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
5614 const CallBase &I, bool HasChain, bool OnlyLoad,
5615 TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
5616 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5617
5618 // Build the operand list.
5619 SmallVector<SDValue, 8> Ops;
5620 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5621 if (OnlyLoad) {
5622 // We don't need to serialize loads against other loads.
5623 Ops.push_back(Elt: DAG.getRoot());
5624 } else {
5625 Ops.push_back(Elt: getRoot());
5626 }
5627 }
5628
5629 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5630 if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||
5631 TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)
5632 Ops.push_back(Elt: DAG.getTargetConstant(Val: I.getIntrinsicID(), DL: getCurSDLoc(),
5633 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
5634
5635 // Add all operands of the call to the operand list.
5636 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5637 const Value *Arg = I.getArgOperand(i);
5638 if (!I.paramHasAttr(ArgNo: i, Kind: Attribute::ImmArg)) {
5639 Ops.push_back(Elt: getValue(V: Arg));
5640 continue;
5641 }
5642
5643 // Use TargetConstant instead of a regular constant for immarg.
5644 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: Arg->getType(), AllowUnknown: true);
5645 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg)) {
5646 assert(CI->getBitWidth() <= 64 &&
5647 "large intrinsic immediates not handled");
5648 Ops.push_back(Elt: DAG.getTargetConstant(Val: *CI, DL: SDLoc(), VT));
5649 } else {
5650 Ops.push_back(
5651 Elt: DAG.getTargetConstantFP(Val: *cast<ConstantFP>(Val: Arg), DL: SDLoc(), VT));
5652 }
5653 }
5654
5655 if (std::optional<OperandBundleUse> Bundle =
5656 I.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
5657 auto *Sym = Bundle->Inputs[0].get();
5658 SDValue SDSym = getValue(V: Sym);
5659 SDSym = DAG.getDeactivationSymbol(GV: cast<GlobalValue>(Val: Sym));
5660 Ops.push_back(Elt: SDSym);
5661 }
5662
5663 if (std::optional<OperandBundleUse> Bundle =
5664 I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
5665 Value *Token = Bundle->Inputs[0].get();
5666 SDValue ConvControlToken = getValue(V: Token);
5667 assert(Ops.back().getValueType() != MVT::Glue &&
5668 "Did not expect another glue node here.");
5669 ConvControlToken =
5670 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: {}, VT: MVT::Glue, Operand: ConvControlToken);
5671 Ops.push_back(Elt: ConvControlToken);
5672 }
5673
5674 return Ops;
5675}
5676
5677SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
5678 bool HasChain) {
5679 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5680
5681 SmallVector<EVT, 4> ValueVTs;
5682 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
5683
5684 if (HasChain)
5685 ValueVTs.push_back(Elt: MVT::Other);
5686
5687 return DAG.getVTList(VTs: ValueVTs);
5688}
5689
5690/// Get an INTRINSIC node for a target intrinsic which does not touch memory.
5691SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5692 const Type &IntrinsicVT, bool HasChain, ArrayRef<SDValue> Ops,
5693 const SDVTList &VTs) {
5694 if (!HasChain)
5695 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5696 if (!IntrinsicVT.isVoidTy())
5697 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5698 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops);
5699}
5700
5701/// Set root, convert return type if necessary and check alignment.
5702SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
5703 bool HasChain,
5704 bool OnlyLoad,
5705 SDValue Result) {
5706 if (HasChain) {
5707 SDValue Chain = Result.getValue(R: Result.getNode()->getNumValues() - 1);
5708 if (OnlyLoad)
5709 PendingLoads.push_back(Elt: Chain);
5710 else
5711 DAG.setRoot(Chain);
5712 }
5713
5714 if (I.getType()->isVoidTy())
5715 return Result;
5716
5717 if (MaybeAlign Alignment = I.getRetAlign(); InsertAssertAlign && Alignment) {
5718 // Insert `assertalign` node if there's an alignment.
5719 Result = DAG.getAssertAlign(DL: getCurSDLoc(), V: Result, A: Alignment.valueOrOne());
5720 } else if (!isa<VectorType>(Val: I.getType())) {
5721 Result = lowerRangeToAssertZExt(DAG, I, Op: Result);
5722 }
5723
5724 return Result;
5725}
5726
5727/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5728/// node.
5729void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5730 unsigned Intrinsic) {
5731 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
5732 Intrinsic::ID IntrinsicID = static_cast<Intrinsic::ID>(Intrinsic);
5733
5734 if (!DAG.getMachineFunction().getSubtarget().isIntrinsicSupported(
5735 IntrinsicID: Intrinsic)) {
5736 SDLoc DL = getCurSDLoc();
5737 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupportedTargetIntrinsic(
5738 *I.getFunction(), IntrinsicID, DL.getDebugLoc()));
5739
5740 // The intrinsic is not available on this subtarget. Preserve the chain for
5741 // side-effecting intrinsics and lower any result to poison so that
5742 // compilation can continue and collect further diagnostics.
5743 if (HasChain && !OnlyLoad)
5744 DAG.setRoot(getRoot());
5745
5746 setValueToPoison(V: &I, dl: DL);
5747 return;
5748 }
5749
5750 // Infos is set by getTgtMemIntrinsic.
5751 SmallVector<TargetLowering::IntrinsicInfo> Infos;
5752 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5753 TLI.getTgtMemIntrinsic(Infos, I, MF&: DAG.getMachineFunction(), Intrinsic);
5754 // Use the first (primary) info determines the node opcode.
5755 TargetLowering::IntrinsicInfo *Info = !Infos.empty() ? &Infos[0] : nullptr;
5756
5757 SmallVector<SDValue, 8> Ops =
5758 getTargetIntrinsicOperands(I, HasChain, OnlyLoad, TgtMemIntrinsicInfo: Info);
5759 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
5760
5761 // Propagate fast-math-flags from IR to node(s).
5762 SDNodeFlags Flags;
5763 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
5764 Flags.copyFMF(FPMO: *FPMO);
5765 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5766
5767 // Create the node.
5768 SDValue Result;
5769
5770 // In some cases, custom collection of operands from CallInst I may be needed.
5771 TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);
5772 if (!Infos.empty()) {
5773 // This is target intrinsic that touches memory
5774 // Create MachineMemOperands for each memory access described by the target.
5775 MachineFunction &MF = DAG.getMachineFunction();
5776 SmallVector<MachineMemOperand *> MMOs;
5777 for (const auto &Info : Infos) {
5778 // TODO: We currently just fallback to address space 0 if
5779 // getTgtMemIntrinsic didn't yield anything useful.
5780 MachinePointerInfo MPI;
5781 if (Info.ptrVal)
5782 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5783 else if (Info.fallbackAddressSpace)
5784 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5785 EVT MemVT = Info.memVT;
5786 LocationSize Size = LocationSize::precise(Value: Info.size);
5787 if (Size.hasValue() && !Size.getValue())
5788 Size = LocationSize::precise(Value: MemVT.getStoreSize());
5789 Align Alignment = Info.align.value_or(u: DAG.getEVTAlign(MemoryVT: MemVT));
5790 MachineMemOperand *MMO = MF.getMachineMemOperand(
5791 PtrInfo: MPI, F: Info.flags, Size, BaseAlignment: Alignment, Metadata: I.getAAMetadata(), SSID: Info.ssid,
5792 Ordering: Info.order, FailureOrdering: Info.failureOrder);
5793 MMOs.push_back(Elt: MMO);
5794 }
5795
5796 Result = DAG.getMemIntrinsicNode(Opcode: Info->opc, dl: getCurSDLoc(), VTList: VTs, Ops,
5797 MemVT: Info->memVT, MMOs);
5798 } else {
5799 Result = getTargetNonMemIntrinsicNode(IntrinsicVT: *I.getType(), HasChain, Ops, VTs);
5800 }
5801
5802 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
5803
5804 setValue(V: &I, NewN: Result);
5805}
5806
5807/// GetSignificand - Get the significand and build it into a floating-point
5808/// number with exponent of 1:
5809///
5810/// Op = (Op & 0x007fffff) | 0x3f800000;
5811///
5812/// where Op is the hexadecimal representation of floating point value.
5813static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
5814 SDValue t1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Op,
5815 N2: DAG.getConstant(Val: 0x007fffff, DL: dl, VT: MVT::i32));
5816 SDValue t2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: t1,
5817 N2: DAG.getConstant(Val: 0x3f800000, DL: dl, VT: MVT::i32));
5818 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: t2);
5819}
5820
5821/// GetExponent - Get the exponent:
5822///
5823/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5824///
5825/// where Op is the hexadecimal representation of floating point value.
5826static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
5827 const TargetLowering &TLI, const SDLoc &dl) {
5828 SDValue t0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Op,
5829 N2: DAG.getConstant(Val: 0x7f800000, DL: dl, VT: MVT::i32));
5830 SDValue t1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: t0,
5831 N2: DAG.getShiftAmountConstant(Val: 23, VT: MVT::i32, DL: dl));
5832 SDValue t2 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: t1,
5833 N2: DAG.getConstant(Val: 127, DL: dl, VT: MVT::i32));
5834 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::f32, Operand: t2);
5835}
5836
5837/// getF32Constant - Get 32-bit floating point constant.
5838static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
5839 const SDLoc &dl) {
5840 return DAG.getConstantFP(Val: APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), DL: dl,
5841 VT: MVT::f32);
5842}
5843
5844static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
5845 SelectionDAG &DAG) {
5846 // TODO: What fast-math-flags should be set on the floating-point nodes?
5847
5848 // IntegerPartOfX = ((int32_t)(t0);
5849 SDValue IntegerPartOfX = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: t0);
5850
5851 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5852 SDValue t1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::f32, Operand: IntegerPartOfX);
5853 SDValue X = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0, N2: t1);
5854
5855 // IntegerPartOfX <<= 23;
5856 IntegerPartOfX = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: IntegerPartOfX,
5857 N2: DAG.getShiftAmountConstant(Val: 23, VT: MVT::i32, DL: dl));
5858
5859 SDValue TwoToFractionalPartOfX;
5860 if (LimitFloatPrecision <= 6) {
5861 // For floating-point precision of 6:
5862 //
5863 // TwoToFractionalPartOfX =
5864 // 0.997535578f +
5865 // (0.735607626f + 0.252464424f * x) * x;
5866 //
5867 // error 0.0144103317, which is 6 bits
5868 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5869 N2: getF32Constant(DAG, Flt: 0x3e814304, dl));
5870 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5871 N2: getF32Constant(DAG, Flt: 0x3f3c50c8, dl));
5872 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5873 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5874 N2: getF32Constant(DAG, Flt: 0x3f7f5e7e, dl));
5875 } else if (LimitFloatPrecision <= 12) {
5876 // For floating-point precision of 12:
5877 //
5878 // TwoToFractionalPartOfX =
5879 // 0.999892986f +
5880 // (0.696457318f +
5881 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5882 //
5883 // error 0.000107046256, which is 13 to 14 bits
5884 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5885 N2: getF32Constant(DAG, Flt: 0x3da235e3, dl));
5886 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5887 N2: getF32Constant(DAG, Flt: 0x3e65b8f3, dl));
5888 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5889 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5890 N2: getF32Constant(DAG, Flt: 0x3f324b07, dl));
5891 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5892 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5893 N2: getF32Constant(DAG, Flt: 0x3f7ff8fd, dl));
5894 } else { // LimitFloatPrecision <= 18
5895 // For floating-point precision of 18:
5896 //
5897 // TwoToFractionalPartOfX =
5898 // 0.999999982f +
5899 // (0.693148872f +
5900 // (0.240227044f +
5901 // (0.554906021e-1f +
5902 // (0.961591928e-2f +
5903 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5904 // error 2.47208000*10^(-7), which is better than 18 bits
5905 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5906 N2: getF32Constant(DAG, Flt: 0x3924b03e, dl));
5907 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5908 N2: getF32Constant(DAG, Flt: 0x3ab24b87, dl));
5909 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5910 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5911 N2: getF32Constant(DAG, Flt: 0x3c1d8c17, dl));
5912 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5913 SDValue t7 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5914 N2: getF32Constant(DAG, Flt: 0x3d634a1d, dl));
5915 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5916 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
5917 N2: getF32Constant(DAG, Flt: 0x3e75fe14, dl));
5918 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
5919 SDValue t11 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t10,
5920 N2: getF32Constant(DAG, Flt: 0x3f317234, dl));
5921 SDValue t12 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t11, N2: X);
5922 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t12,
5923 N2: getF32Constant(DAG, Flt: 0x3f800000, dl));
5924 }
5925
5926 // Add the exponent into the result in integer domain.
5927 SDValue t13 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: TwoToFractionalPartOfX);
5928 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32,
5929 Operand: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: t13, N2: IntegerPartOfX));
5930}
5931
5932/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5933/// limited-precision mode.
5934static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5935 const TargetLowering &TLI, SDNodeFlags Flags) {
5936 if (Op.getValueType() == MVT::f32 &&
5937 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5938
5939 // Put the exponent in the right bit position for later addition to the
5940 // final result:
5941 //
5942 // t0 = Op * log2(e)
5943
5944 // TODO: What fast-math-flags should be set here?
5945 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Op,
5946 N2: DAG.getConstantFP(Val: numbers::log2ef, DL: dl, VT: MVT::f32));
5947 return getLimitedPrecisionExp2(t0, dl, DAG);
5948 }
5949
5950 // No special expansion.
5951 return DAG.getNode(Opcode: ISD::FEXP, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5952}
5953
5954/// expandLog - Lower a log intrinsic. Handles the special sequences for
5955/// limited-precision mode.
5956static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5957 const TargetLowering &TLI, SDNodeFlags Flags) {
5958 // TODO: What fast-math-flags should be set on the floating-point nodes?
5959
5960 if (Op.getValueType() == MVT::f32 &&
5961 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5962 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
5963
5964 // Scale the exponent by log(2).
5965 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
5966 SDValue LogOfExponent =
5967 DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Exp,
5968 N2: DAG.getConstantFP(Val: numbers::ln2f, DL: dl, VT: MVT::f32));
5969
5970 // Get the significand and build it into a floating-point number with
5971 // exponent of 1.
5972 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5973
5974 SDValue LogOfMantissa;
5975 if (LimitFloatPrecision <= 6) {
5976 // For floating-point precision of 6:
5977 //
5978 // LogofMantissa =
5979 // -1.1609546f +
5980 // (1.4034025f - 0.23903021f * x) * x;
5981 //
5982 // error 0.0034276066, which is better than 8 bits
5983 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5984 N2: getF32Constant(DAG, Flt: 0xbe74c456, dl));
5985 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5986 N2: getF32Constant(DAG, Flt: 0x3fb3a2b1, dl));
5987 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5988 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5989 N2: getF32Constant(DAG, Flt: 0x3f949a29, dl));
5990 } else if (LimitFloatPrecision <= 12) {
5991 // For floating-point precision of 12:
5992 //
5993 // LogOfMantissa =
5994 // -1.7417939f +
5995 // (2.8212026f +
5996 // (-1.4699568f +
5997 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5998 //
5999 // error 0.000061011436, which is 14 bits
6000 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6001 N2: getF32Constant(DAG, Flt: 0xbd67b6d6, dl));
6002 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6003 N2: getF32Constant(DAG, Flt: 0x3ee4f4b8, dl));
6004 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6005 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6006 N2: getF32Constant(DAG, Flt: 0x3fbc278b, dl));
6007 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6008 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
6009 N2: getF32Constant(DAG, Flt: 0x40348e95, dl));
6010 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6011 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
6012 N2: getF32Constant(DAG, Flt: 0x3fdef31a, dl));
6013 } else { // LimitFloatPrecision <= 18
6014 // For floating-point precision of 18:
6015 //
6016 // LogOfMantissa =
6017 // -2.1072184f +
6018 // (4.2372794f +
6019 // (-3.7029485f +
6020 // (2.2781945f +
6021 // (-0.87823314f +
6022 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
6023 //
6024 // error 0.0000023660568, which is better than 18 bits
6025 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6026 N2: getF32Constant(DAG, Flt: 0xbc91e5ac, dl));
6027 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6028 N2: getF32Constant(DAG, Flt: 0x3e4350aa, dl));
6029 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6030 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6031 N2: getF32Constant(DAG, Flt: 0x3f60d3e3, dl));
6032 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6033 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
6034 N2: getF32Constant(DAG, Flt: 0x4011cdf0, dl));
6035 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6036 SDValue t7 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
6037 N2: getF32Constant(DAG, Flt: 0x406cfd1c, dl));
6038 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
6039 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
6040 N2: getF32Constant(DAG, Flt: 0x408797cb, dl));
6041 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
6042 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t10,
6043 N2: getF32Constant(DAG, Flt: 0x4006dcab, dl));
6044 }
6045
6046 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: LogOfMantissa);
6047 }
6048
6049 // No special expansion.
6050 return DAG.getNode(Opcode: ISD::FLOG, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6051}
6052
6053/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
6054/// limited-precision mode.
6055static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
6056 const TargetLowering &TLI, SDNodeFlags Flags) {
6057 // TODO: What fast-math-flags should be set on the floating-point nodes?
6058
6059 if (Op.getValueType() == MVT::f32 &&
6060 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
6061 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
6062
6063 // Get the exponent.
6064 SDValue LogOfExponent = GetExponent(DAG, Op: Op1, TLI, dl);
6065
6066 // Get the significand and build it into a floating-point number with
6067 // exponent of 1.
6068 SDValue X = GetSignificand(DAG, Op: Op1, dl);
6069
6070 // Different possible minimax approximations of significand in
6071 // floating-point for various degrees of accuracy over [1,2].
6072 SDValue Log2ofMantissa;
6073 if (LimitFloatPrecision <= 6) {
6074 // For floating-point precision of 6:
6075 //
6076 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
6077 //
6078 // error 0.0049451742, which is more than 7 bits
6079 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6080 N2: getF32Constant(DAG, Flt: 0xbeb08fe0, dl));
6081 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6082 N2: getF32Constant(DAG, Flt: 0x40019463, dl));
6083 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6084 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6085 N2: getF32Constant(DAG, Flt: 0x3fd6633d, dl));
6086 } else if (LimitFloatPrecision <= 12) {
6087 // For floating-point precision of 12:
6088 //
6089 // Log2ofMantissa =
6090 // -2.51285454f +
6091 // (4.07009056f +
6092 // (-2.12067489f +
6093 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
6094 //
6095 // error 0.0000876136000, which is better than 13 bits
6096 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6097 N2: getF32Constant(DAG, Flt: 0xbda7262e, dl));
6098 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6099 N2: getF32Constant(DAG, Flt: 0x3f25280b, dl));
6100 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6101 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6102 N2: getF32Constant(DAG, Flt: 0x4007b923, dl));
6103 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6104 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
6105 N2: getF32Constant(DAG, Flt: 0x40823e2f, dl));
6106 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6107 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
6108 N2: getF32Constant(DAG, Flt: 0x4020d29c, dl));
6109 } else { // LimitFloatPrecision <= 18
6110 // For floating-point precision of 18:
6111 //
6112 // Log2ofMantissa =
6113 // -3.0400495f +
6114 // (6.1129976f +
6115 // (-5.3420409f +
6116 // (3.2865683f +
6117 // (-1.2669343f +
6118 // (0.27515199f -
6119 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
6120 //
6121 // error 0.0000018516, which is better than 18 bits
6122 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6123 N2: getF32Constant(DAG, Flt: 0xbcd2769e, dl));
6124 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6125 N2: getF32Constant(DAG, Flt: 0x3e8ce0b9, dl));
6126 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6127 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6128 N2: getF32Constant(DAG, Flt: 0x3fa22ae7, dl));
6129 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6130 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
6131 N2: getF32Constant(DAG, Flt: 0x40525723, dl));
6132 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6133 SDValue t7 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
6134 N2: getF32Constant(DAG, Flt: 0x40aaf200, dl));
6135 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
6136 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
6137 N2: getF32Constant(DAG, Flt: 0x40c39dad, dl));
6138 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
6139 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t10,
6140 N2: getF32Constant(DAG, Flt: 0x4042902c, dl));
6141 }
6142
6143 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: Log2ofMantissa);
6144 }
6145
6146 // No special expansion.
6147 return DAG.getNode(Opcode: ISD::FLOG2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6148}
6149
6150/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
6151/// limited-precision mode.
6152static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
6153 const TargetLowering &TLI, SDNodeFlags Flags) {
6154 // TODO: What fast-math-flags should be set on the floating-point nodes?
6155
6156 if (Op.getValueType() == MVT::f32 &&
6157 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
6158 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
6159
6160 // Scale the exponent by log10(2) [0.30102999f].
6161 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
6162 SDValue LogOfExponent = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Exp,
6163 N2: getF32Constant(DAG, Flt: 0x3e9a209a, dl));
6164
6165 // Get the significand and build it into a floating-point number with
6166 // exponent of 1.
6167 SDValue X = GetSignificand(DAG, Op: Op1, dl);
6168
6169 SDValue Log10ofMantissa;
6170 if (LimitFloatPrecision <= 6) {
6171 // For floating-point precision of 6:
6172 //
6173 // Log10ofMantissa =
6174 // -0.50419619f +
6175 // (0.60948995f - 0.10380950f * x) * x;
6176 //
6177 // error 0.0014886165, which is 6 bits
6178 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6179 N2: getF32Constant(DAG, Flt: 0xbdd49a13, dl));
6180 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6181 N2: getF32Constant(DAG, Flt: 0x3f1c0789, dl));
6182 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6183 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6184 N2: getF32Constant(DAG, Flt: 0x3f011300, dl));
6185 } else if (LimitFloatPrecision <= 12) {
6186 // For floating-point precision of 12:
6187 //
6188 // Log10ofMantissa =
6189 // -0.64831180f +
6190 // (0.91751397f +
6191 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
6192 //
6193 // error 0.00019228036, which is better than 12 bits
6194 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6195 N2: getF32Constant(DAG, Flt: 0x3d431f31, dl));
6196 SDValue t1 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0,
6197 N2: getF32Constant(DAG, Flt: 0x3ea21fb2, dl));
6198 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6199 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
6200 N2: getF32Constant(DAG, Flt: 0x3f6ae232, dl));
6201 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6202 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t4,
6203 N2: getF32Constant(DAG, Flt: 0x3f25f7c3, dl));
6204 } else { // LimitFloatPrecision <= 18
6205 // For floating-point precision of 18:
6206 //
6207 // Log10ofMantissa =
6208 // -0.84299375f +
6209 // (1.5327582f +
6210 // (-1.0688956f +
6211 // (0.49102474f +
6212 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
6213 //
6214 // error 0.0000037995730, which is better than 18 bits
6215 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6216 N2: getF32Constant(DAG, Flt: 0x3c5d51ce, dl));
6217 SDValue t1 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0,
6218 N2: getF32Constant(DAG, Flt: 0x3e00685a, dl));
6219 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6220 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
6221 N2: getF32Constant(DAG, Flt: 0x3efb6798, dl));
6222 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6223 SDValue t5 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t4,
6224 N2: getF32Constant(DAG, Flt: 0x3f88d192, dl));
6225 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6226 SDValue t7 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
6227 N2: getF32Constant(DAG, Flt: 0x3fc4316c, dl));
6228 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
6229 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t8,
6230 N2: getF32Constant(DAG, Flt: 0x3f57ce70, dl));
6231 }
6232
6233 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: Log10ofMantissa);
6234 }
6235
6236 // No special expansion.
6237 return DAG.getNode(Opcode: ISD::FLOG10, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6238}
6239
6240/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
6241/// limited-precision mode.
6242static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
6243 const TargetLowering &TLI, SDNodeFlags Flags) {
6244 if (Op.getValueType() == MVT::f32 &&
6245 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
6246 return getLimitedPrecisionExp2(t0: Op, dl, DAG);
6247
6248 // No special expansion.
6249 return DAG.getNode(Opcode: ISD::FEXP2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6250}
6251
6252/// visitPow - Lower a pow intrinsic. Handles the special sequences for
6253/// limited-precision mode with x == 10.0f.
6254static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
6255 SelectionDAG &DAG, const TargetLowering &TLI,
6256 SDNodeFlags Flags) {
6257 bool IsExp10 = false;
6258 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
6259 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
6260 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
6261 APFloat Ten(10.0f);
6262 IsExp10 = LHSC->isExactlyValue(V: Ten);
6263 }
6264 }
6265
6266 // TODO: What fast-math-flags should be set on the FMUL node?
6267 if (IsExp10) {
6268 // Put the exponent in the right bit position for later addition to the
6269 // final result:
6270 //
6271 // #define LOG2OF10 3.3219281f
6272 // t0 = Op * LOG2OF10;
6273 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: RHS,
6274 N2: getF32Constant(DAG, Flt: 0x40549a78, dl));
6275 return getLimitedPrecisionExp2(t0, dl, DAG);
6276 }
6277
6278 // No special expansion.
6279 return DAG.getNode(Opcode: ISD::FPOW, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags);
6280}
6281
6282/// ExpandPowI - Expand a llvm.powi intrinsic.
6283static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
6284 SelectionDAG &DAG) {
6285 // If RHS is a constant, we can expand this out to a multiplication tree if
6286 // it's beneficial on the target, otherwise we end up lowering to a call to
6287 // __powidf2 (for example).
6288 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
6289 unsigned Val = RHSC->getSExtValue();
6290
6291 // powi(x, 0) -> 1.0
6292 if (Val == 0)
6293 return DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType());
6294
6295 if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
6296 Exponent: Val, OptForSize: DAG.shouldOptForSize())) {
6297 // Get the exponent as a positive value.
6298 if ((int)Val < 0)
6299 Val = -Val;
6300 // We use the simple binary decomposition method to generate the multiply
6301 // sequence. There are more optimal ways to do this (for example,
6302 // powi(x,15) generates one more multiply than it should), but this has
6303 // the benefit of being both really simple and much better than a libcall.
6304 SDValue Res; // Logically starts equal to 1.0
6305 SDValue CurSquare = LHS;
6306 // TODO: Intrinsics should have fast-math-flags that propagate to these
6307 // nodes.
6308 while (Val) {
6309 if (Val & 1) {
6310 if (Res.getNode())
6311 Res =
6312 DAG.getNode(Opcode: ISD::FMUL, DL, VT: Res.getValueType(), N1: Res, N2: CurSquare);
6313 else
6314 Res = CurSquare; // 1.0*CurSquare.
6315 }
6316
6317 CurSquare = DAG.getNode(Opcode: ISD::FMUL, DL, VT: CurSquare.getValueType(),
6318 N1: CurSquare, N2: CurSquare);
6319 Val >>= 1;
6320 }
6321
6322 // If the original was negative, invert the result, producing 1/(x*x*x).
6323 if (RHSC->getSExtValue() < 0)
6324 Res = DAG.getNode(Opcode: ISD::FDIV, DL, VT: LHS.getValueType(),
6325 N1: DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType()), N2: Res);
6326 return Res;
6327 }
6328 }
6329
6330 // Otherwise, expand to a libcall.
6331 return DAG.getNode(Opcode: ISD::FPOWI, DL, VT: LHS.getValueType(), N1: LHS, N2: RHS);
6332}
6333
6334static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
6335 SDValue LHS, SDValue RHS, SDValue Scale,
6336 SelectionDAG &DAG, const TargetLowering &TLI) {
6337 EVT VT = LHS.getValueType();
6338 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
6339 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
6340 LLVMContext &Ctx = *DAG.getContext();
6341
6342 // If the type is legal but the operation isn't, this node might survive all
6343 // the way to operation legalization. If we end up there and we do not have
6344 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
6345 // node.
6346
6347 // Coax the legalizer into expanding the node during type legalization instead
6348 // by bumping the size by one bit. This will force it to Promote, enabling the
6349 // early expansion and avoiding the need to expand later.
6350
6351 // We don't have to do this if Scale is 0; that can always be expanded, unless
6352 // it's a saturating signed operation. Those can experience true integer
6353 // division overflow, a case which we must avoid.
6354
6355 // FIXME: We wouldn't have to do this (or any of the early
6356 // expansion/promotion) if it was possible to expand a libcall of an
6357 // illegal type during operation legalization. But it's not, so things
6358 // get a bit hacky.
6359 unsigned ScaleInt = Scale->getAsZExtVal();
6360 if ((ScaleInt > 0 || (Saturating && Signed)) &&
6361 (TLI.isTypeLegal(VT) ||
6362 (VT.isVector() && TLI.isTypeLegal(VT: VT.getVectorElementType())))) {
6363 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
6364 Op: Opcode, VT, Scale: ScaleInt);
6365 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
6366 EVT PromVT;
6367 if (VT.isScalarInteger())
6368 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits() + 1);
6369 else if (VT.isVector()) {
6370 PromVT = VT.getVectorElementType();
6371 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PromVT.getSizeInBits() + 1);
6372 PromVT = EVT::getVectorVT(Context&: Ctx, VT: PromVT, EC: VT.getVectorElementCount());
6373 } else
6374 llvm_unreachable("Wrong VT for DIVFIX?");
6375 LHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL, VT: PromVT);
6376 RHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL, VT: PromVT);
6377 EVT ShiftTy = TLI.getShiftAmountTy(LHSTy: PromVT, DL: DAG.getDataLayout());
6378 // For saturating operations, we need to shift up the LHS to get the
6379 // proper saturation width, and then shift down again afterwards.
6380 if (Saturating)
6381 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: PromVT, N1: LHS,
6382 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
6383 SDValue Res = DAG.getNode(Opcode, DL, VT: PromVT, N1: LHS, N2: RHS, N3: Scale);
6384 if (Saturating)
6385 Res = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: PromVT, N1: Res,
6386 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
6387 return DAG.getZExtOrTrunc(Op: Res, DL, VT);
6388 }
6389 }
6390
6391 return DAG.getNode(Opcode, DL, VT, N1: LHS, N2: RHS, N3: Scale);
6392}
6393
6394// getUnderlyingArgRegs - Find underlying registers used for a truncated,
6395// bitcasted, or split argument. Returns a list of <Register, size in bits>
6396static void
6397getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
6398 const SDValue &N) {
6399 switch (N.getOpcode()) {
6400 case ISD::CopyFromReg: {
6401 SDValue Op = N.getOperand(i: 1);
6402 Regs.emplace_back(Args: cast<RegisterSDNode>(Val&: Op)->getReg(),
6403 Args: Op.getValueType().getSizeInBits());
6404 return;
6405 }
6406 case ISD::BITCAST:
6407 case ISD::AssertZext:
6408 case ISD::AssertSext:
6409 case ISD::TRUNCATE:
6410 getUnderlyingArgRegs(Regs, N: N.getOperand(i: 0));
6411 return;
6412 case ISD::BUILD_PAIR:
6413 case ISD::BUILD_VECTOR:
6414 case ISD::CONCAT_VECTORS:
6415 for (SDValue Op : N->op_values())
6416 getUnderlyingArgRegs(Regs, N: Op);
6417 return;
6418 default:
6419 return;
6420 }
6421}
6422
6423/// If the DbgValueInst is a dbg_value of a function argument, create the
6424/// corresponding DBG_VALUE machine instruction for it now. At the end of
6425/// instruction selection, they will be inserted to the entry BB.
6426/// We don't currently support this for variadic dbg_values, as they shouldn't
6427/// appear for function arguments or in the prologue.
6428bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6429 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6430 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6431 const Argument *Arg = dyn_cast<Argument>(Val: V);
6432 if (!Arg)
6433 return false;
6434
6435 MachineFunction &MF = DAG.getMachineFunction();
6436 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6437
6438 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6439 // we've been asked to pursue.
6440 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6441 bool Indirect) {
6442 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6443 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6444 // pointing at the VReg, which will be patched up later.
6445 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_INSTR_REF);
6446 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
6447 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6448 /* isKill */ false, /* isDead */ false,
6449 /* isUndef */ false, /* isEarlyClobber */ false,
6450 /* SubReg */ 0, /* isDebug */ true)});
6451
6452 auto *NewDIExpr = FragExpr;
6453 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6454 // the DIExpression.
6455 if (Indirect)
6456 NewDIExpr = DIExpression::prepend(Expr: FragExpr, Flags: DIExpression::DerefBefore);
6457 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
6458 NewDIExpr = DIExpression::prependOpcodes(Expr: NewDIExpr, Ops);
6459 return BuildMI(MF, DL, MCID: Inst, IsIndirect: false, MOs, Variable, Expr: NewDIExpr);
6460 } else {
6461 // Create a completely standard DBG_VALUE.
6462 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_VALUE);
6463 return BuildMI(MF, DL, MCID: Inst, IsIndirect: Indirect, Reg, Variable, Expr: FragExpr);
6464 }
6465 };
6466
6467 if (Kind == FuncArgumentDbgValueKind::Value) {
6468 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6469 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6470 // the entry block.
6471 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6472 if (!IsInEntryBlock)
6473 return false;
6474
6475 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6476 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6477 // variable that also is a param.
6478 //
6479 // Although, if we are at the top of the entry block already, we can still
6480 // emit using ArgDbgValue. This might catch some situations when the
6481 // dbg.value refers to an argument that isn't used in the entry block, so
6482 // any CopyToReg node would be optimized out and the only way to express
6483 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6484 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6485 // we should only emit as ArgDbgValue if the Variable is an argument to the
6486 // current function, and the dbg.value intrinsic is found in the entry
6487 // block.
6488 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6489 !DL->getInlinedAt();
6490 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6491 if (!IsInPrologue && !VariableIsFunctionInputArg)
6492 return false;
6493
6494 // Here we assume that a function argument on IR level only can be used to
6495 // describe one input parameter on source level. If we for example have
6496 // source code like this
6497 //
6498 // struct A { long x, y; };
6499 // void foo(struct A a, long b) {
6500 // ...
6501 // b = a.x;
6502 // ...
6503 // }
6504 //
6505 // and IR like this
6506 //
6507 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6508 // entry:
6509 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6510 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6511 // call void @llvm.dbg.value(metadata i32 %b, "b",
6512 // ...
6513 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6514 // ...
6515 //
6516 // then the last dbg.value is describing a parameter "b" using a value that
6517 // is an argument. But since we already has used %a1 to describe a parameter
6518 // we should not handle that last dbg.value here (that would result in an
6519 // incorrect hoisting of the DBG_VALUE to the function entry).
6520 // Notice that we allow one dbg.value per IR level argument, to accommodate
6521 // for the situation with fragments above.
6522 // If there is no node for the value being handled, we return true to skip
6523 // the normal generation of debug info, as it would kill existing debug
6524 // info for the parameter in case of duplicates.
6525 if (VariableIsFunctionInputArg) {
6526 unsigned ArgNo = Arg->getArgNo();
6527 if (ArgNo >= FuncInfo.DescribedArgs.size())
6528 FuncInfo.DescribedArgs.resize(N: ArgNo + 1, t: false);
6529 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(Idx: ArgNo))
6530 return !NodeMap[V].getNode();
6531 FuncInfo.DescribedArgs.set(ArgNo);
6532 }
6533 }
6534
6535 bool IsIndirect = false;
6536 std::optional<MachineOperand> Op;
6537 // Some arguments' frame index is recorded during argument lowering.
6538 int FI = FuncInfo.getArgumentFrameIndex(A: Arg);
6539 if (FI != std::numeric_limits<int>::max())
6540 Op = MachineOperand::CreateFI(Idx: FI);
6541
6542 SmallVector<std::pair<Register, TypeSize>, 8> ArgRegsAndSizes;
6543 if (!Op && N.getNode()) {
6544 getUnderlyingArgRegs(Regs&: ArgRegsAndSizes, N);
6545 Register Reg;
6546 if (ArgRegsAndSizes.size() == 1)
6547 Reg = ArgRegsAndSizes.front().first;
6548
6549 if (Reg && Reg.isVirtual()) {
6550 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6551 Register PR = RegInfo.getLiveInPhysReg(VReg: Reg);
6552 if (PR)
6553 Reg = PR;
6554 }
6555 if (Reg) {
6556 Op = MachineOperand::CreateReg(Reg, isDef: false);
6557 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6558 }
6559 }
6560
6561 if (!Op && N.getNode()) {
6562 // Check if frame index is available.
6563 SDValue LCandidate = peekThroughBitcasts(V: N);
6564 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(Val: LCandidate.getNode()))
6565 if (FrameIndexSDNode *FINode =
6566 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
6567 Op = MachineOperand::CreateFI(Idx: FINode->getIndex());
6568 }
6569
6570 if (!Op) {
6571 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6572 auto splitMultiRegDbgValue =
6573 [&](ArrayRef<std::pair<Register, TypeSize>> SplitRegs) -> bool {
6574 unsigned Offset = 0;
6575 for (const auto &[Reg, RegSizeInBits] : SplitRegs) {
6576 // FIXME: Scalable sizes are not supported in fragment expressions.
6577 if (RegSizeInBits.isScalable())
6578 return false;
6579
6580 // If the expression is already a fragment, the current register
6581 // offset+size might extend beyond the fragment. In this case, only
6582 // the register bits that are inside the fragment are relevant.
6583 int RegFragmentSizeInBits = RegSizeInBits.getFixedValue();
6584 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6585 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6586 // The register is entirely outside the expression fragment,
6587 // so is irrelevant for debug info.
6588 if (Offset >= ExprFragmentSizeInBits)
6589 break;
6590 // The register is partially outside the expression fragment, only
6591 // the low bits within the fragment are relevant for debug info.
6592 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6593 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6594 }
6595 }
6596
6597 auto FragmentExpr = DIExpression::createFragmentExpression(
6598 Expr, OffsetInBits: Offset, SizeInBits: RegFragmentSizeInBits);
6599 Offset += RegSizeInBits.getFixedValue();
6600 // If a valid fragment expression cannot be created, the variable's
6601 // correct value cannot be determined and so it is set as poison.
6602 if (!FragmentExpr) {
6603 SDDbgValue *SDV = DAG.getConstantDbgValue(
6604 Var: Variable, Expr, C: PoisonValue::get(T: V->getType()), DL, O: SDNodeOrder);
6605 DAG.AddDbgValue(DB: SDV, isParameter: false);
6606 continue;
6607 }
6608 MachineInstr *NewMI = MakeVRegDbgValue(
6609 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6610 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6611 }
6612
6613 return true;
6614 };
6615
6616 // Check if ValueMap has reg number.
6617 DenseMap<const Value *, Register>::const_iterator
6618 VMI = FuncInfo.ValueMap.find(Val: V);
6619 if (VMI != FuncInfo.ValueMap.end()) {
6620 const auto &TLI = DAG.getTargetLoweringInfo();
6621 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6622 V->getType(), std::nullopt);
6623 if (RFV.occupiesMultipleRegs())
6624 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6625
6626 Op = MachineOperand::CreateReg(Reg: VMI->second, isDef: false);
6627 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6628 } else if (ArgRegsAndSizes.size() > 1) {
6629 // This was split due to the calling convention, and no virtual register
6630 // mapping exists for the value.
6631 return splitMultiRegDbgValue(ArgRegsAndSizes);
6632 }
6633 }
6634
6635 if (!Op)
6636 return false;
6637
6638 assert(Variable->isValidLocationForIntrinsic(DL) &&
6639 "Expected inlined-at fields to agree");
6640 MachineInstr *NewMI = nullptr;
6641
6642 if (Op->isReg())
6643 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6644 else
6645 NewMI = BuildMI(MF, DL, MCID: TII->get(Opcode: TargetOpcode::DBG_VALUE), IsIndirect: true, MOs: *Op,
6646 Variable, Expr);
6647
6648 // Otherwise, use ArgDbgValues.
6649 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6650 return true;
6651}
6652
6653/// Return the appropriate SDDbgValue based on N.
6654SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6655 DILocalVariable *Variable,
6656 DIExpression *Expr,
6657 const DebugLoc &dl,
6658 unsigned DbgSDNodeOrder) {
6659 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
6660 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6661 // stack slot locations.
6662 //
6663 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6664 // debug values here after optimization:
6665 //
6666 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6667 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6668 //
6669 // Both describe the direct values of their associated variables.
6670 return DAG.getFrameIndexDbgValue(Var: Variable, Expr, FI: FISDN->getIndex(),
6671 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6672 }
6673 return DAG.getDbgValue(Var: Variable, Expr, N: N.getNode(), R: N.getResNo(),
6674 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6675}
6676
6677static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6678 switch (Intrinsic) {
6679 case Intrinsic::smul_fix:
6680 return ISD::SMULFIX;
6681 case Intrinsic::umul_fix:
6682 return ISD::UMULFIX;
6683 case Intrinsic::smul_fix_sat:
6684 return ISD::SMULFIXSAT;
6685 case Intrinsic::umul_fix_sat:
6686 return ISD::UMULFIXSAT;
6687 case Intrinsic::sdiv_fix:
6688 return ISD::SDIVFIX;
6689 case Intrinsic::udiv_fix:
6690 return ISD::UDIVFIX;
6691 case Intrinsic::sdiv_fix_sat:
6692 return ISD::SDIVFIXSAT;
6693 case Intrinsic::udiv_fix_sat:
6694 return ISD::UDIVFIXSAT;
6695 default:
6696 llvm_unreachable("Unhandled fixed point intrinsic");
6697 }
6698}
6699
6700/// Given a @llvm.call.preallocated.setup, return the corresponding
6701/// preallocated call.
6702static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6703 assert(cast<CallBase>(PreallocatedSetup)
6704 ->getCalledFunction()
6705 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6706 "expected call_preallocated_setup Value");
6707 for (const auto *U : PreallocatedSetup->users()) {
6708 auto *UseCall = cast<CallBase>(Val: U);
6709 const Function *Fn = UseCall->getCalledFunction();
6710 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6711 return UseCall;
6712 }
6713 }
6714 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6715}
6716
6717/// If DI is a debug value with an EntryValue expression, lower it using the
6718/// corresponding physical register of the associated Argument value
6719/// (guaranteed to exist by the verifier).
6720bool SelectionDAGBuilder::visitEntryValueDbgValue(
6721 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6722 DIExpression *Expr, DebugLoc DbgLoc) {
6723 if (!Expr->isEntryValue() || !hasSingleElement(C&: Values))
6724 return false;
6725
6726 // These properties are guaranteed by the verifier.
6727 const Argument *Arg = cast<Argument>(Val: Values[0]);
6728 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6729
6730 auto ArgIt = FuncInfo.ValueMap.find(Val: Arg);
6731 if (ArgIt == FuncInfo.ValueMap.end()) {
6732 LLVM_DEBUG(
6733 dbgs() << "Dropping dbg.value: expression is entry_value but "
6734 "couldn't find an associated register for the Argument\n");
6735 return true;
6736 }
6737 Register ArgVReg = ArgIt->getSecond();
6738
6739 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6740 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6741 SDDbgValue *SDV = DAG.getVRegDbgValue(
6742 Var: Variable, Expr, VReg: PhysReg, IsIndirect: false /*IsIndidrect*/, DL: DbgLoc, O: SDNodeOrder);
6743 DAG.AddDbgValue(DB: SDV, isParameter: false /*treat as dbg.declare byval parameter*/);
6744 return true;
6745 }
6746 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6747 "couldn't find a physical register\n");
6748 return true;
6749}
6750
6751/// Lower the call to the specified intrinsic function.
6752void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6753 unsigned Intrinsic) {
6754 SDLoc sdl = getCurSDLoc();
6755 switch (Intrinsic) {
6756 case Intrinsic::experimental_convergence_anchor:
6757 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_ANCHOR, DL: sdl, VT: MVT::Untyped));
6758 break;
6759 case Intrinsic::experimental_convergence_entry:
6760 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_ENTRY, DL: sdl, VT: MVT::Untyped));
6761 break;
6762 case Intrinsic::experimental_convergence_loop: {
6763 auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
6764 auto *Token = Bundle->Inputs[0].get();
6765 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_LOOP, DL: sdl, VT: MVT::Untyped,
6766 Operand: getValue(V: Token)));
6767 break;
6768 }
6769 }
6770}
6771
6772void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6773 unsigned IntrinsicID) {
6774 // For now, we're only lowering an 'add' histogram.
6775 // We can add others later, e.g. saturating adds, min/max.
6776 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6777 "Tried to lower unsupported histogram type");
6778 SDLoc sdl = getCurSDLoc();
6779 Value *Ptr = I.getOperand(i_nocapture: 0);
6780 SDValue Inc = getValue(V: I.getOperand(i_nocapture: 1));
6781 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
6782
6783 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6784 EVT VT = Inc.getValueType();
6785 Align Alignment = DAG.getEVTAlign(MemoryVT: VT);
6786
6787 const MDNode *Ranges = getRangeMetadata(I);
6788
6789 SDValue Root = DAG.getRoot();
6790 SDValue Base;
6791 SDValue Index;
6792 SDValue Scale;
6793 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
6794 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
6795
6796 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6797
6798 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6799 PtrInfo: MachinePointerInfo(AS),
6800 F: MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
6801 Size: MemoryLocation::UnknownSize, BaseAlignment: Alignment,
6802 Metadata: MMOMetadata(I.getAAMetadata(), Ranges));
6803
6804 if (!UniformBase) {
6805 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6806 Index = getValue(V: Ptr);
6807 Scale =
6808 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6809 }
6810
6811 EVT IdxVT = Index.getValueType();
6812
6813 // Avoid using e.g. i32 as index type when the increment must be performed
6814 // on i64's.
6815 bool MustExtendIndex = VT.getScalarSizeInBits() > IdxVT.getScalarSizeInBits();
6816 EVT EltTy = MustExtendIndex ? VT : IdxVT.getVectorElementType();
6817 if (MustExtendIndex || TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
6818 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
6819 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
6820 }
6821
6822 SDValue ID = DAG.getTargetConstant(Val: IntrinsicID, DL: sdl, VT: MVT::i32);
6823
6824 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6825 SDValue Histogram = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT: VT, dl: sdl,
6826 Ops, MMO, IndexType: ISD::SIGNED_SCALED);
6827
6828 setValue(V: &I, NewN: Histogram);
6829 DAG.setRoot(Histogram);
6830}
6831
6832void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6833 unsigned Intrinsic) {
6834 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6835 "Tried lowering invalid vector extract last");
6836 SDLoc sdl = getCurSDLoc();
6837 const DataLayout &Layout = DAG.getDataLayout();
6838 SDValue Data = getValue(V: I.getOperand(i_nocapture: 0));
6839 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
6840
6841 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6842 EVT ResVT = TLI.getValueType(DL: Layout, Ty: I.getType());
6843
6844 EVT ExtVT = TLI.getVectorIdxTy(DL: Layout);
6845 SDValue Idx = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL: sdl, VT: ExtVT, Operand: Mask);
6846 SDValue Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: sdl, VT: ResVT, N1: Data, N2: Idx);
6847
6848 Value *Default = I.getOperand(i_nocapture: 2);
6849 if (!isa<PoisonValue>(Val: Default) && !isa<UndefValue>(Val: Default)) {
6850 SDValue PassThru = getValue(V: Default);
6851 EVT BoolVT = Mask.getValueType().getScalarType();
6852 SDValue AnyActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: sdl, VT: BoolVT, Operand: Mask);
6853 Result = DAG.getSelect(DL: sdl, VT: ResVT, Cond: AnyActive, LHS: Result, RHS: PassThru);
6854 }
6855
6856 setValue(V: &I, NewN: Result);
6857}
6858
6859/// Lower the call to the specified intrinsic function.
6860void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6861 unsigned Intrinsic) {
6862 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6863 SDLoc sdl = getCurSDLoc();
6864 DebugLoc dl = getCurDebugLoc();
6865 SDValue Res;
6866
6867 SDNodeFlags Flags;
6868 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
6869 Flags.copyFMF(FPMO: *FPOp);
6870
6871 switch (Intrinsic) {
6872 default:
6873 // By default, turn this into a target intrinsic node.
6874 visitTargetIntrinsic(I, Intrinsic);
6875 return;
6876 case Intrinsic::vscale: {
6877 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6878 setValue(V: &I, NewN: DAG.getVScale(DL: sdl, VT, MulImm: APInt(VT.getSizeInBits(), 1)));
6879 return;
6880 }
6881 case Intrinsic::vastart: visitVAStart(I); return;
6882 case Intrinsic::vaend: visitVAEnd(I); return;
6883 case Intrinsic::vacopy: visitVACopy(I); return;
6884 case Intrinsic::returnaddress:
6885 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::RETURNADDR, DL: sdl,
6886 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
6887 Operand: getValue(V: I.getArgOperand(i: 0))));
6888 return;
6889 case Intrinsic::addressofreturnaddress:
6890 setValue(V: &I,
6891 NewN: DAG.getNode(Opcode: ISD::ADDROFRETURNADDR, DL: sdl,
6892 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6893 return;
6894 case Intrinsic::sponentry:
6895 setValue(V: &I,
6896 NewN: DAG.getNode(Opcode: ISD::SPONENTRY, DL: sdl,
6897 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6898 return;
6899 case Intrinsic::frameaddress:
6900 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FRAMEADDR, DL: sdl,
6901 VT: TLI.getFrameIndexTy(DL: DAG.getDataLayout()),
6902 Operand: getValue(V: I.getArgOperand(i: 0))));
6903 return;
6904 case Intrinsic::read_volatile_register:
6905 case Intrinsic::read_register: {
6906 Value *Reg = I.getArgOperand(i: 0);
6907 SDValue Chain = getRoot();
6908 SDValue RegName =
6909 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6910 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6911 Res = DAG.getNode(Opcode: ISD::READ_REGISTER, DL: sdl,
6912 VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), N1: Chain, N2: RegName);
6913 setValue(V: &I, NewN: Res);
6914 DAG.setRoot(Res.getValue(R: 1));
6915 return;
6916 }
6917 case Intrinsic::write_register: {
6918 Value *Reg = I.getArgOperand(i: 0);
6919 Value *RegValue = I.getArgOperand(i: 1);
6920 SDValue Chain = getRoot();
6921 SDValue RegName =
6922 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6923 DAG.setRoot(DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL: sdl, VT: MVT::Other, N1: Chain,
6924 N2: RegName, N3: getValue(V: RegValue)));
6925 return;
6926 }
6927 case Intrinsic::write_volatile_register: {
6928 Value *Reg = I.getArgOperand(i: 0);
6929 Value *RegValue = I.getArgOperand(i: 1);
6930 SDValue Chain = getRoot();
6931 const MDNode *MD = cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata());
6932 SDValue RegName = DAG.getMDNode(MD);
6933 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: RegValue->getType());
6934 SDValue WriteChain = DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL: sdl, VT: MVT::Other,
6935 N1: Chain, N2: RegName, N3: getValue(V: RegValue));
6936 // FAKE_USE of the physical register marks it live after the WRITE_REGISTER,
6937 // preventing the backend from dead-eliminating the write. This is
6938 // preferred over READ_REGISTER, which would emit extra register copies
6939 // (e.g. fmov xN, dN for FP/SIMD registers).
6940 const MDString *RegStr = cast<MDString>(Val: MD->getOperand(I: 0));
6941 LLT Ty = VT.isSimple() ? getLLTForMVT(Ty: VT.getSimpleVT()) : LLT();
6942 const MachineFunction &MF = DAG.getMachineFunction();
6943 Register PhysReg =
6944 TLI.getRegisterByName(RegName: RegStr->getString().data(), Ty, MF);
6945 if (PhysReg.isValid()) {
6946 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6947 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: PhysReg);
6948 MVT RegVT = *TRI->legalclasstypes_begin(RC: *RC);
6949 DAG.setRoot(DAG.getNode(Opcode: ISD::FAKE_USE, DL: sdl, VT: MVT::Other,
6950 Ops: {WriteChain, DAG.getRegister(Reg: PhysReg, VT: RegVT)}));
6951 } else {
6952 DAG.setRoot(WriteChain);
6953 }
6954 return;
6955 }
6956 case Intrinsic::memcpy:
6957 case Intrinsic::memcpy_inline: {
6958 const auto &MCI = cast<MemCpyInst>(Val: I);
6959 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6960 SDValue Src = getValue(V: I.getArgOperand(i: 1));
6961 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6962 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6963 "memcpy_inline needs constant size");
6964 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6965 Align DstAlign = MCI.getDestAlign().valueOrOne();
6966 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6967 bool isVol = MCI.isVolatile();
6968 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6969 SDValue MC = DAG.getMemcpy(Chain: Root, dl: sdl, Dst, Src, Size, DstAlign, SrcAlign,
6970 isVol, AlwaysInline: MCI.isForceInlined(), CI: &I, OverrideTailCall: std::nullopt,
6971 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
6972 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)),
6973 AAInfo: I.getAAMetadata(), BatchAA);
6974 updateDAGForMaybeTailCall(MaybeTC: MC);
6975 return;
6976 }
6977 case Intrinsic::memset:
6978 case Intrinsic::memset_inline: {
6979 const auto &MSII = cast<MemSetInst>(Val: I);
6980 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6981 SDValue Value = getValue(V: I.getArgOperand(i: 1));
6982 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6983 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6984 "memset_inline needs constant size");
6985 // @llvm.memset defines 0 and 1 to both mean no alignment.
6986 Align DstAlign = MSII.getDestAlign().valueOrOne();
6987 bool isVol = MSII.isVolatile();
6988 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6989 SDValue MC = DAG.getMemset(
6990 Chain: Root, dl: sdl, Dst, Src: Value, Size, Alignment: DstAlign, isVol, AlwaysInline: MSII.isForceInlined(),
6991 CI: &I, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)), AAInfo: I.getAAMetadata());
6992 updateDAGForMaybeTailCall(MaybeTC: MC);
6993 return;
6994 }
6995 case Intrinsic::memmove: {
6996 const auto &MMI = cast<MemMoveInst>(Val: I);
6997 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
6998 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
6999 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7000 // @llvm.memmove defines 0 and 1 to both mean no alignment.
7001 Align DstAlign = MMI.getDestAlign().valueOrOne();
7002 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
7003 bool isVol = MMI.isVolatile();
7004 SDValue Root = isVol ? getRoot() : getMemoryRoot();
7005 SDValue MM = DAG.getMemmove(
7006 Chain: Root, dl: sdl, Dst: Op1, Src: Op2, Size: Op3, DstAlign, SrcAlign, isVol, CI: &I,
7007 /* OverrideTailCall */ std::nullopt,
7008 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
7009 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)), AAInfo: I.getAAMetadata(), BatchAA);
7010 updateDAGForMaybeTailCall(MaybeTC: MM);
7011 return;
7012 }
7013 case Intrinsic::memcpy_element_unordered_atomic: {
7014 auto &MI = cast<AnyMemCpyInst>(Val: I);
7015 SDValue Dst = getValue(V: MI.getRawDest());
7016 SDValue Src = getValue(V: MI.getRawSource());
7017 SDValue Length = getValue(V: MI.getLength());
7018
7019 Type *LengthTy = MI.getLength()->getType();
7020 unsigned ElemSz = MI.getElementSizeInBytes();
7021 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
7022 SDValue MC =
7023 DAG.getAtomicMemcpy(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
7024 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
7025 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
7026 updateDAGForMaybeTailCall(MaybeTC: MC);
7027 return;
7028 }
7029 case Intrinsic::memmove_element_unordered_atomic: {
7030 auto &MI = cast<AnyMemMoveInst>(Val: I);
7031 SDValue Dst = getValue(V: MI.getRawDest());
7032 SDValue Src = getValue(V: MI.getRawSource());
7033 SDValue Length = getValue(V: MI.getLength());
7034
7035 Type *LengthTy = MI.getLength()->getType();
7036 unsigned ElemSz = MI.getElementSizeInBytes();
7037 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
7038 SDValue MC =
7039 DAG.getAtomicMemmove(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
7040 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
7041 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
7042 updateDAGForMaybeTailCall(MaybeTC: MC);
7043 return;
7044 }
7045 case Intrinsic::memset_element_unordered_atomic: {
7046 auto &MI = cast<AnyMemSetInst>(Val: I);
7047 SDValue Dst = getValue(V: MI.getRawDest());
7048 SDValue Val = getValue(V: MI.getValue());
7049 SDValue Length = getValue(V: MI.getLength());
7050
7051 Type *LengthTy = MI.getLength()->getType();
7052 unsigned ElemSz = MI.getElementSizeInBytes();
7053 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
7054 SDValue MC =
7055 DAG.getAtomicMemset(Chain: getRoot(), dl: sdl, Dst, Value: Val, Size: Length, SizeTy: LengthTy, ElemSz,
7056 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()));
7057 updateDAGForMaybeTailCall(MaybeTC: MC);
7058 return;
7059 }
7060 case Intrinsic::call_preallocated_setup: {
7061 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: &I);
7062 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
7063 SDValue Res = DAG.getNode(Opcode: ISD::PREALLOCATED_SETUP, DL: sdl, VT: MVT::Other,
7064 N1: getRoot(), N2: SrcValue);
7065 setValue(V: &I, NewN: Res);
7066 DAG.setRoot(Res);
7067 return;
7068 }
7069 case Intrinsic::call_preallocated_arg: {
7070 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: I.getOperand(i_nocapture: 0));
7071 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
7072 SDValue Ops[3];
7073 Ops[0] = getRoot();
7074 Ops[1] = SrcValue;
7075 Ops[2] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 1)), DL: sdl,
7076 VT: MVT::i32); // arg index
7077 SDValue Res = DAG.getNode(
7078 Opcode: ISD::PREALLOCATED_ARG, DL: sdl,
7079 VTList: DAG.getVTList(VT1: TLI.getPointerTy(DL: DAG.getDataLayout()), VT2: MVT::Other), Ops);
7080 setValue(V: &I, NewN: Res);
7081 DAG.setRoot(Res.getValue(R: 1));
7082 return;
7083 }
7084
7085 case Intrinsic::eh_typeid_for: {
7086 // Find the type id for the given typeinfo.
7087 GlobalValue *GV = ExtractTypeInfo(V: I.getArgOperand(i: 0));
7088 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(TI: GV);
7089 Res = DAG.getConstant(Val: TypeID, DL: sdl, VT: MVT::i32);
7090 setValue(V: &I, NewN: Res);
7091 return;
7092 }
7093
7094 case Intrinsic::eh_return_i32:
7095 case Intrinsic::eh_return_i64:
7096 DAG.getMachineFunction().setCallsEHReturn(true);
7097 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_RETURN, DL: sdl,
7098 VT: MVT::Other,
7099 N1: getControlRoot(),
7100 N2: getValue(V: I.getArgOperand(i: 0)),
7101 N3: getValue(V: I.getArgOperand(i: 1))));
7102 return;
7103 case Intrinsic::eh_unwind_init:
7104 DAG.getMachineFunction().setCallsUnwindInit(true);
7105 return;
7106 case Intrinsic::eh_dwarf_cfa:
7107 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EH_DWARF_CFA, DL: sdl,
7108 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
7109 Operand: getValue(V: I.getArgOperand(i: 0))));
7110 return;
7111 case Intrinsic::eh_sjlj_callsite: {
7112 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 0));
7113 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
7114
7115 FuncInfo.setCurrentCallSite(CI->getZExtValue());
7116 return;
7117 }
7118 case Intrinsic::eh_sjlj_functioncontext: {
7119 // Get and store the index of the function context.
7120 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7121 AllocaInst *FnCtx =
7122 cast<AllocaInst>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
7123 int FI = FuncInfo.StaticAllocaMap[FnCtx];
7124 MFI.setFunctionContextIndex(FI);
7125 return;
7126 }
7127 case Intrinsic::eh_sjlj_setjmp: {
7128 SDValue Ops[2];
7129 Ops[0] = getRoot();
7130 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7131 SDValue Op = DAG.getNode(Opcode: ISD::EH_SJLJ_SETJMP, DL: sdl,
7132 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
7133 setValue(V: &I, NewN: Op.getValue(R: 0));
7134 DAG.setRoot(Op.getValue(R: 1));
7135 return;
7136 }
7137 case Intrinsic::eh_sjlj_longjmp:
7138 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_SJLJ_LONGJMP, DL: sdl, VT: MVT::Other,
7139 N1: getRoot(), N2: getValue(V: I.getArgOperand(i: 0))));
7140 return;
7141 case Intrinsic::eh_sjlj_setup_dispatch:
7142 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_SJLJ_SETUP_DISPATCH, DL: sdl, VT: MVT::Other,
7143 Operand: getRoot()));
7144 return;
7145 case Intrinsic::masked_gather:
7146 visitMaskedGather(I);
7147 return;
7148 case Intrinsic::masked_load:
7149 visitMaskedLoad(I);
7150 return;
7151 case Intrinsic::masked_scatter:
7152 visitMaskedScatter(I);
7153 return;
7154 case Intrinsic::masked_store:
7155 visitMaskedStore(I);
7156 return;
7157 case Intrinsic::masked_expandload:
7158 visitMaskedLoad(I, IsExpanding: true /* IsExpanding */);
7159 return;
7160 case Intrinsic::masked_compressstore:
7161 visitMaskedStore(I, IsCompressing: true /* IsCompressing */);
7162 return;
7163 case Intrinsic::speculative_load:
7164 visitSpeculativeLoad(I);
7165 return;
7166 case Intrinsic::powi:
7167 setValue(V: &I, NewN: ExpandPowI(DL: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
7168 RHS: getValue(V: I.getArgOperand(i: 1)), DAG));
7169 return;
7170 case Intrinsic::log:
7171 setValue(V: &I, NewN: expandLog(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7172 return;
7173 case Intrinsic::log2:
7174 setValue(V: &I,
7175 NewN: expandLog2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7176 return;
7177 case Intrinsic::log10:
7178 setValue(V: &I,
7179 NewN: expandLog10(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7180 return;
7181 case Intrinsic::exp:
7182 setValue(V: &I, NewN: expandExp(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7183 return;
7184 case Intrinsic::exp2:
7185 setValue(V: &I,
7186 NewN: expandExp2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7187 return;
7188 case Intrinsic::pow:
7189 setValue(V: &I, NewN: expandPow(dl: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
7190 RHS: getValue(V: I.getArgOperand(i: 1)), DAG, TLI, Flags));
7191 return;
7192 case Intrinsic::sqrt:
7193 case Intrinsic::fabs:
7194 case Intrinsic::sin:
7195 case Intrinsic::cos:
7196 case Intrinsic::tan:
7197 case Intrinsic::asin:
7198 case Intrinsic::acos:
7199 case Intrinsic::atan:
7200 case Intrinsic::sinh:
7201 case Intrinsic::cosh:
7202 case Intrinsic::tanh:
7203 case Intrinsic::exp10:
7204 case Intrinsic::floor:
7205 case Intrinsic::ceil:
7206 case Intrinsic::trunc:
7207 case Intrinsic::rint:
7208 case Intrinsic::nearbyint:
7209 case Intrinsic::round:
7210 case Intrinsic::roundeven:
7211 case Intrinsic::canonicalize: {
7212 unsigned Opcode;
7213 // clang-format off
7214 switch (Intrinsic) {
7215 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7216 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
7217 case Intrinsic::fabs: Opcode = ISD::FABS; break;
7218 case Intrinsic::sin: Opcode = ISD::FSIN; break;
7219 case Intrinsic::cos: Opcode = ISD::FCOS; break;
7220 case Intrinsic::tan: Opcode = ISD::FTAN; break;
7221 case Intrinsic::asin: Opcode = ISD::FASIN; break;
7222 case Intrinsic::acos: Opcode = ISD::FACOS; break;
7223 case Intrinsic::atan: Opcode = ISD::FATAN; break;
7224 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
7225 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
7226 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
7227 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
7228 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
7229 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
7230 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
7231 case Intrinsic::rint: Opcode = ISD::FRINT; break;
7232 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
7233 case Intrinsic::round: Opcode = ISD::FROUND; break;
7234 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
7235 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
7236 }
7237 // clang-format on
7238
7239 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl,
7240 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7241 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
7242 return;
7243 }
7244 case Intrinsic::atan2:
7245 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FATAN2, DL: sdl,
7246 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7247 N1: getValue(V: I.getArgOperand(i: 0)),
7248 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7249 return;
7250 case Intrinsic::lround:
7251 case Intrinsic::llround:
7252 case Intrinsic::lrint:
7253 case Intrinsic::llrint: {
7254 unsigned Opcode;
7255 // clang-format off
7256 switch (Intrinsic) {
7257 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7258 case Intrinsic::lround: Opcode = ISD::LROUND; break;
7259 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
7260 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
7261 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
7262 }
7263 // clang-format on
7264
7265 EVT RetVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7266 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VT: RetVT,
7267 Operand: getValue(V: I.getArgOperand(i: 0))));
7268 return;
7269 }
7270 case Intrinsic::minnum:
7271 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINNUM, DL: sdl,
7272 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7273 N1: getValue(V: I.getArgOperand(i: 0)),
7274 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7275 return;
7276 case Intrinsic::maxnum:
7277 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXNUM, DL: sdl,
7278 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7279 N1: getValue(V: I.getArgOperand(i: 0)),
7280 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7281 return;
7282 case Intrinsic::minimum:
7283 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUM, DL: sdl,
7284 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7285 N1: getValue(V: I.getArgOperand(i: 0)),
7286 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7287 return;
7288 case Intrinsic::maximum:
7289 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUM, DL: sdl,
7290 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7291 N1: getValue(V: I.getArgOperand(i: 0)),
7292 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7293 return;
7294 case Intrinsic::minimumnum:
7295 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUMNUM, DL: sdl,
7296 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7297 N1: getValue(V: I.getArgOperand(i: 0)),
7298 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7299 return;
7300 case Intrinsic::maximumnum:
7301 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUMNUM, DL: sdl,
7302 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7303 N1: getValue(V: I.getArgOperand(i: 0)),
7304 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7305 return;
7306 case Intrinsic::copysign:
7307 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: sdl,
7308 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7309 N1: getValue(V: I.getArgOperand(i: 0)),
7310 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7311 return;
7312 case Intrinsic::ldexp:
7313 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FLDEXP, DL: sdl,
7314 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7315 N1: getValue(V: I.getArgOperand(i: 0)),
7316 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7317 return;
7318 case Intrinsic::modf:
7319 case Intrinsic::sincos:
7320 case Intrinsic::sincospi:
7321 case Intrinsic::frexp: {
7322 unsigned Opcode;
7323 switch (Intrinsic) {
7324 default:
7325 llvm_unreachable("unexpected intrinsic");
7326 case Intrinsic::sincos:
7327 Opcode = ISD::FSINCOS;
7328 break;
7329 case Intrinsic::sincospi:
7330 Opcode = ISD::FSINCOSPI;
7331 break;
7332 case Intrinsic::modf:
7333 Opcode = ISD::FMODF;
7334 break;
7335 case Intrinsic::frexp:
7336 Opcode = ISD::FFREXP;
7337 break;
7338 }
7339 SmallVector<EVT, 2> ValueVTs;
7340 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
7341 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
7342 setValue(
7343 V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: getValue(V: I.getArgOperand(i: 0)), Flags));
7344 return;
7345 }
7346 case Intrinsic::arithmetic_fence: {
7347 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: sdl,
7348 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7349 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
7350 return;
7351 }
7352 case Intrinsic::fma:
7353 setValue(V: &I, NewN: DAG.getNode(
7354 Opcode: ISD::FMA, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7355 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)),
7356 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7357 return;
7358#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7359 case Intrinsic::INTRINSIC:
7360#include "llvm/IR/ConstrainedOps.def"
7361 visitConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: I));
7362 return;
7363#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7364#include "llvm/IR/VPIntrinsics.def"
7365 visitVectorPredicationIntrinsic(VPIntrin: cast<VPIntrinsic>(Val: I));
7366 return;
7367 case Intrinsic::fptrunc_round: {
7368 // Get the last argument, the metadata and convert it to an integer in the
7369 // call
7370 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7371 std::optional<RoundingMode> RoundMode =
7372 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
7373
7374 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7375
7376 // Propagate fast-math-flags from IR to node(s).
7377 SDNodeFlags Flags;
7378 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
7379 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
7380
7381 SDValue Result;
7382 Result = DAG.getNode(
7383 Opcode: ISD::FPTRUNC_ROUND, DL: sdl, VT, N1: getValue(V: I.getArgOperand(i: 0)),
7384 N2: DAG.getTargetConstant(Val: (int)*RoundMode, DL: sdl, VT: MVT::i32));
7385 setValue(V: &I, NewN: Result);
7386
7387 return;
7388 }
7389 case Intrinsic::fmuladd: {
7390 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7391 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
7392 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMA, DL: sdl,
7393 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7394 N1: getValue(V: I.getArgOperand(i: 0)),
7395 N2: getValue(V: I.getArgOperand(i: 1)),
7396 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7397 } else if (TLI.isOperationLegalOrCustom(Op: ISD::FMULADD, VT)) {
7398 // TODO: Support splitting the vector.
7399 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMULADD, DL: sdl,
7400 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7401 N1: getValue(V: I.getArgOperand(i: 0)),
7402 N2: getValue(V: I.getArgOperand(i: 1)),
7403 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7404 } else {
7405 // TODO: Intrinsic calls should have fast-math-flags.
7406 SDValue Mul = DAG.getNode(
7407 Opcode: ISD::FMUL, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7408 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)), Flags);
7409 SDValue Add = DAG.getNode(Opcode: ISD::FADD, DL: sdl,
7410 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7411 N1: Mul, N2: getValue(V: I.getArgOperand(i: 2)), Flags);
7412 setValue(V: &I, NewN: Add);
7413 }
7414 return;
7415 }
7416 case Intrinsic::fptosi_sat: {
7417 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7418 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT_SAT, DL: sdl, VT,
7419 N1: getValue(V: I.getArgOperand(i: 0)),
7420 N2: DAG.getValueType(VT.getScalarType())));
7421 return;
7422 }
7423 case Intrinsic::fptoui_sat: {
7424 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7425 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT_SAT, DL: sdl, VT,
7426 N1: getValue(V: I.getArgOperand(i: 0)),
7427 N2: DAG.getValueType(VT.getScalarType())));
7428 return;
7429 }
7430 case Intrinsic::convert_from_arbitrary_fp: {
7431 // Extract format metadata and convert to semantics enum.
7432 EVT DstVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7433 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7434 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
7435 const fltSemantics *SrcSem =
7436 APFloatBase::getArbitraryFPSemantics(Format: FormatStr);
7437 if (!SrcSem) {
7438 DAG.getContext()->emitError(
7439 ErrorStr: "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7440 "'");
7441 setValue(V: &I, NewN: DAG.getPOISON(VT: DstVT));
7442 return;
7443 }
7444 APFloatBase::Semantics SemEnum = APFloatBase::SemanticsToEnum(Sem: *SrcSem);
7445
7446 SDValue IntVal = getValue(V: I.getArgOperand(i: 0));
7447
7448 // Emit ISD::CONVERT_FROM_ARBITRARY_FP node.
7449 SDValue SemConst =
7450 DAG.getTargetConstant(Val: static_cast<int>(SemEnum), DL: sdl, VT: MVT::i32);
7451 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: sdl, VT: DstVT, N1: IntVal,
7452 N2: SemConst));
7453 return;
7454 }
7455 case Intrinsic::convert_to_arbitrary_fp: {
7456 // Extract format metadata and convert to semantics enum.
7457 EVT DstVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7458 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7459 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
7460 const fltSemantics *DstSem =
7461 APFloatBase::getArbitraryFPSemantics(Format: FormatStr);
7462 if (!DstSem) {
7463 DAG.getContext()->emitError(
7464 ErrorStr: "convert_to_arbitrary_fp: not implemented format '" + FormatStr +
7465 "'");
7466 setValue(V: &I, NewN: DAG.getPOISON(VT: DstVT));
7467 return;
7468 }
7469 APFloatBase::Semantics SemEnum = APFloatBase::SemanticsToEnum(Sem: *DstSem);
7470
7471 Metadata *RoundMD =
7472 cast<MetadataAsValue>(Val: I.getArgOperand(i: 2))->getMetadata();
7473 StringRef RoundStr = cast<MDString>(Val: RoundMD)->getString();
7474 std::optional<RoundingMode> RoundMode = convertStrToRoundingMode(RoundStr);
7475 assert(RoundMode && *RoundMode != RoundingMode::Dynamic &&
7476 "Dynamic rounding mode should have been rejected by the verifier");
7477
7478 uint64_t Saturate =
7479 cast<ConstantInt>(Val: I.getArgOperand(i: 3))->getZExtValue() ? 1 : 0;
7480
7481 SDValue FloatVal = getValue(V: I.getArgOperand(i: 0));
7482
7483 SDValue SemConst =
7484 DAG.getTargetConstant(Val: static_cast<int>(SemEnum), DL: sdl, VT: MVT::i32);
7485 SDValue RoundConst =
7486 DAG.getTargetConstant(Val: static_cast<int>(*RoundMode), DL: sdl, VT: MVT::i32);
7487 SDValue SatConst = DAG.getTargetConstant(Val: Saturate, DL: sdl, VT: MVT::i32);
7488 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL: sdl, VT: DstVT, N1: FloatVal,
7489 N2: SemConst, N3: RoundConst, N4: SatConst));
7490 return;
7491 }
7492 case Intrinsic::set_rounding:
7493 Res = DAG.getNode(Opcode: ISD::SET_ROUNDING, DL: sdl, VT: MVT::Other,
7494 Ops: {getRoot(), getValue(V: I.getArgOperand(i: 0))});
7495 setValue(V: &I, NewN: Res);
7496 DAG.setRoot(Res.getValue(R: 0));
7497 return;
7498 case Intrinsic::is_fpclass: {
7499 const DataLayout DLayout = DAG.getDataLayout();
7500 EVT DestVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7501 EVT ArgVT = TLI.getValueType(DL: DLayout, Ty: I.getArgOperand(i: 0)->getType());
7502 FPClassTest Test = static_cast<FPClassTest>(
7503 cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue());
7504 MachineFunction &MF = DAG.getMachineFunction();
7505 const Function &F = MF.getFunction();
7506 SDValue Op = getValue(V: I.getArgOperand(i: 0));
7507 SDNodeFlags Flags;
7508 Flags.setNoFPExcept(
7509 !F.getAttributes().hasFnAttr(Kind: llvm::Attribute::StrictFP));
7510 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7511 // expansion can use illegal types. Making expansion early allows
7512 // legalizing these types prior to selection.
7513 if (!TLI.isOperationLegal(Op: ISD::IS_FPCLASS, VT: ArgVT) &&
7514 !TLI.isOperationCustom(Op: ISD::IS_FPCLASS, VT: ArgVT)) {
7515 SDValue Result = TLI.expandIS_FPCLASS(ResultVT: DestVT, Op, Test, Flags, DL: sdl, DAG);
7516 setValue(V: &I, NewN: Result);
7517 return;
7518 }
7519
7520 SDValue Check = DAG.getTargetConstant(Val: Test, DL: sdl, VT: MVT::i32);
7521 SDValue V = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: sdl, VT: DestVT, Ops: {Op, Check}, Flags);
7522 setValue(V: &I, NewN: V);
7523 return;
7524 }
7525 case Intrinsic::get_fpenv: {
7526 const DataLayout DLayout = DAG.getDataLayout();
7527 EVT EnvVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7528 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7529 SDValue Chain = getRoot();
7530 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7531 // and temporary storage in stack.
7532 if (TLI.isOperationLegalOrCustom(Op: ISD::GET_FPENV, VT: EnvVT)) {
7533 Res = DAG.getNode(
7534 Opcode: ISD::GET_FPENV, DL: sdl,
7535 VTList: DAG.getVTList(VT1: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
7536 VT2: MVT::Other),
7537 N: Chain);
7538 } else {
7539 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7540 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7541 auto MPI =
7542 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7543 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7544 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
7545 BaseAlignment: TempAlign);
7546 Chain = DAG.getGetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7547 Res = DAG.getLoad(VT: EnvVT, dl: sdl, Chain, Ptr: Temp, PtrInfo: MPI);
7548 }
7549 setValue(V: &I, NewN: Res);
7550 DAG.setRoot(Res.getValue(R: 1));
7551 return;
7552 }
7553 case Intrinsic::set_fpenv: {
7554 SDValue Env = getValue(V: I.getArgOperand(i: 0));
7555 EVT EnvVT = Env.getValueType();
7556 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7557 SDValue Chain = getRoot();
7558 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7559 // environment from memory.
7560 if (TLI.isOperationLegalOrCustom(Op: ISD::SET_FPENV, VT: EnvVT)) {
7561 Chain = DAG.getNode(Opcode: ISD::SET_FPENV, DL: sdl, VT: MVT::Other, N1: Chain, N2: Env);
7562 } else {
7563 // Allocate space in stack, copy environment bits into it and use this
7564 // memory in SET_FPENV_MEM.
7565 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7566 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7567 auto MPI =
7568 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7569 Chain = DAG.getStore(Chain, dl: sdl, Val: Env, Ptr: Temp, PtrInfo: MPI, Alignment: TempAlign,
7570 MMOFlags: MachineMemOperand::MOStore);
7571 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7572 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
7573 BaseAlignment: TempAlign);
7574 Chain = DAG.getSetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7575 }
7576 DAG.setRoot(Chain);
7577 return;
7578 }
7579 case Intrinsic::reset_fpenv:
7580 DAG.setRoot(DAG.getNode(Opcode: ISD::RESET_FPENV, DL: sdl, VT: MVT::Other, Operand: getRoot()));
7581 return;
7582 case Intrinsic::get_fpmode:
7583 Res = DAG.getNode(
7584 Opcode: ISD::GET_FPMODE, DL: sdl,
7585 VTList: DAG.getVTList(VT1: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
7586 VT2: MVT::Other),
7587 N: DAG.getRoot());
7588 setValue(V: &I, NewN: Res);
7589 DAG.setRoot(Res.getValue(R: 1));
7590 return;
7591 case Intrinsic::set_fpmode:
7592 Res = DAG.getNode(Opcode: ISD::SET_FPMODE, DL: sdl, VT: MVT::Other, N1: {DAG.getRoot()},
7593 N2: getValue(V: I.getArgOperand(i: 0)));
7594 DAG.setRoot(Res);
7595 return;
7596 case Intrinsic::reset_fpmode: {
7597 Res = DAG.getNode(Opcode: ISD::RESET_FPMODE, DL: sdl, VT: MVT::Other, Operand: getRoot());
7598 DAG.setRoot(Res);
7599 return;
7600 }
7601 case Intrinsic::pcmarker: {
7602 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
7603 DAG.setRoot(DAG.getNode(Opcode: ISD::PCMARKER, DL: sdl, VT: MVT::Other, N1: getRoot(), N2: Tmp));
7604 return;
7605 }
7606 case Intrinsic::readcyclecounter: {
7607 SDValue Op = getRoot();
7608 Res = DAG.getNode(Opcode: ISD::READCYCLECOUNTER, DL: sdl,
7609 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N: Op);
7610 setValue(V: &I, NewN: Res);
7611 DAG.setRoot(Res.getValue(R: 1));
7612 return;
7613 }
7614 case Intrinsic::readsteadycounter: {
7615 SDValue Op = getRoot();
7616 Res = DAG.getNode(Opcode: ISD::READSTEADYCOUNTER, DL: sdl,
7617 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N: Op);
7618 setValue(V: &I, NewN: Res);
7619 DAG.setRoot(Res.getValue(R: 1));
7620 return;
7621 }
7622 case Intrinsic::bitreverse:
7623 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITREVERSE, DL: sdl,
7624 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7625 Operand: getValue(V: I.getArgOperand(i: 0))));
7626 return;
7627 case Intrinsic::bswap:
7628 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BSWAP, DL: sdl,
7629 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7630 Operand: getValue(V: I.getArgOperand(i: 0))));
7631 return;
7632 case Intrinsic::cttz: {
7633 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7634 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7635 EVT Ty = Arg.getValueType();
7636 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_POISON,
7637 DL: sdl, VT: Ty, Operand: Arg));
7638 return;
7639 }
7640 case Intrinsic::ctlz: {
7641 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7642 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7643 EVT Ty = Arg.getValueType();
7644 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_POISON,
7645 DL: sdl, VT: Ty, Operand: Arg));
7646 return;
7647 }
7648 case Intrinsic::ctpop: {
7649 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7650 EVT Ty = Arg.getValueType();
7651 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CTPOP, DL: sdl, VT: Ty, Operand: Arg));
7652 return;
7653 }
7654 case Intrinsic::fshl:
7655 case Intrinsic::fshr: {
7656 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7657 SDValue X = getValue(V: I.getArgOperand(i: 0));
7658 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7659 SDValue Z = getValue(V: I.getArgOperand(i: 2));
7660 EVT VT = X.getValueType();
7661
7662 if (X == Y) {
7663 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7664 setValue(V: &I, NewN: DAG.getNode(Opcode: RotateOpcode, DL: sdl, VT, N1: X, N2: Z));
7665 } else {
7666 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7667 setValue(V: &I, NewN: DAG.getNode(Opcode: FunnelOpcode, DL: sdl, VT, N1: X, N2: Y, N3: Z));
7668 }
7669 return;
7670 }
7671 case Intrinsic::clmul: {
7672 SDValue X = getValue(V: I.getArgOperand(i: 0));
7673 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7674 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CLMUL, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7675 return;
7676 }
7677 case Intrinsic::pext: {
7678 SDValue X = getValue(V: I.getArgOperand(i: 0));
7679 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7680 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::PEXT, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7681 return;
7682 }
7683 case Intrinsic::pdep: {
7684 SDValue X = getValue(V: I.getArgOperand(i: 0));
7685 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7686 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::PDEP, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7687 return;
7688 }
7689 case Intrinsic::smulh:
7690 case Intrinsic::umulh: {
7691 auto Opc = Intrinsic == Intrinsic::smulh ? ISD::MULHS : ISD::MULHU;
7692 SDValue X = getValue(V: I.getArgOperand(i: 0));
7693 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7694 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7695 return;
7696 }
7697 case Intrinsic::sadd_sat: {
7698 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7699 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7700 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7701 return;
7702 }
7703 case Intrinsic::uadd_sat: {
7704 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7705 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7706 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7707 return;
7708 }
7709 case Intrinsic::ssub_sat: {
7710 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7711 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7712 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SSUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7713 return;
7714 }
7715 case Intrinsic::usub_sat: {
7716 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7717 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7718 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::USUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7719 return;
7720 }
7721 case Intrinsic::sshl_sat:
7722 case Intrinsic::ushl_sat: {
7723 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7724 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7725
7726 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
7727 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
7728
7729 // Coerce the shift amount to the right type if we can. This exposes the
7730 // truncate or zext to optimization early.
7731 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
7732 assert(ShiftTy.getSizeInBits() >=
7733 Log2_32_Ceil(Op1.getValueSizeInBits()) &&
7734 "Unexpected shift type");
7735 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
7736 }
7737
7738 unsigned Opc =
7739 Intrinsic == Intrinsic::sshl_sat ? ISD::SSHLSAT : ISD::USHLSAT;
7740 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7741 return;
7742 }
7743 case Intrinsic::smul_fix:
7744 case Intrinsic::umul_fix:
7745 case Intrinsic::smul_fix_sat:
7746 case Intrinsic::umul_fix_sat: {
7747 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7748 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7749 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7750 setValue(V: &I, NewN: DAG.getNode(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7751 VT: Op1.getValueType(), N1: Op1, N2: Op2, N3: Op3));
7752 return;
7753 }
7754 case Intrinsic::sdiv_fix:
7755 case Intrinsic::udiv_fix:
7756 case Intrinsic::sdiv_fix_sat:
7757 case Intrinsic::udiv_fix_sat: {
7758 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7759 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7760 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7761 setValue(V: &I, NewN: expandDivFix(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7762 LHS: Op1, RHS: Op2, Scale: Op3, DAG, TLI));
7763 return;
7764 }
7765 case Intrinsic::smax: {
7766 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7767 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7768 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7769 return;
7770 }
7771 case Intrinsic::smin: {
7772 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7773 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7774 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7775 return;
7776 }
7777 case Intrinsic::umax: {
7778 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7779 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7780 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7781 return;
7782 }
7783 case Intrinsic::umin: {
7784 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7785 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7786 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7787 return;
7788 }
7789 case Intrinsic::abs: {
7790 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7791 bool IntMinIsPoison = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->isOne();
7792 unsigned Opc = IntMinIsPoison ? ISD::ABS_MIN_POISON : ISD::ABS;
7793 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: Op1.getValueType(), Operand: Op1));
7794 return;
7795 }
7796 case Intrinsic::scmp: {
7797 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7798 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7799 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7800 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7801 break;
7802 }
7803 case Intrinsic::ucmp: {
7804 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7805 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7806 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7807 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7808 break;
7809 }
7810 case Intrinsic::stackaddress:
7811 case Intrinsic::stacksave: {
7812 unsigned SDOpcode = Intrinsic == Intrinsic::stackaddress ? ISD::STACKADDRESS
7813 : ISD::STACKSAVE;
7814 SDValue Op = getRoot();
7815 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7816 Res = DAG.getNode(Opcode: SDOpcode, DL: sdl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), N: Op);
7817 setValue(V: &I, NewN: Res);
7818 DAG.setRoot(Res.getValue(R: 1));
7819 return;
7820 }
7821 case Intrinsic::stackrestore:
7822 Res = getValue(V: I.getArgOperand(i: 0));
7823 DAG.setRoot(DAG.getNode(Opcode: ISD::STACKRESTORE, DL: sdl, VT: MVT::Other, N1: getRoot(), N2: Res));
7824 return;
7825 case Intrinsic::get_dynamic_area_offset: {
7826 SDValue Op = getRoot();
7827 EVT ResTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7828 Res = DAG.getNode(Opcode: ISD::GET_DYNAMIC_AREA_OFFSET, DL: sdl, VTList: DAG.getVTList(VT: ResTy),
7829 N: Op);
7830 DAG.setRoot(Op);
7831 setValue(V: &I, NewN: Res);
7832 return;
7833 }
7834 case Intrinsic::stackguard: {
7835 MachineFunction &MF = DAG.getMachineFunction();
7836 const Module &M = *MF.getFunction().getParent();
7837 EVT PtrTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7838 SDValue Chain = getRoot();
7839 if (TLI.useLoadStackGuardNode(M)) {
7840 Res = getLoadStackGuard(DAG, DL: sdl, Chain);
7841 Res = DAG.getPtrExtOrTrunc(Op: Res, DL: sdl, VT: PtrTy);
7842 } else {
7843 const Value *Global = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls());
7844 if (!Global) {
7845 LLVMContext &Ctx = *DAG.getContext();
7846 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
7847 setValue(V: &I, NewN: DAG.getPOISON(VT: PtrTy));
7848 return;
7849 }
7850
7851 Align Align = DAG.getDataLayout().getPrefTypeAlign(Ty: Global->getType());
7852 Res = DAG.getLoad(VT: PtrTy, dl: sdl, Chain, Ptr: getValue(V: Global),
7853 PtrInfo: MachinePointerInfo(Global, 0), Alignment: Align,
7854 MMOFlags: MachineMemOperand::MOVolatile);
7855 }
7856 // Mix the cookie with FP if enabled. Skip if using LOAD_STACK_GUARD
7857 // with post-RA mixing (AArch64 MSVCRT), as the mixing will be done during
7858 // post-RA expansion of LOAD_STACK_GUARD.
7859 if (TLI.useStackGuardMixFP() && !TLI.useLoadStackGuardNode(M))
7860 Res = TLI.emitStackGuardMixFP(DAG, Val: Res, DL: sdl);
7861 DAG.setRoot(Chain);
7862 setValue(V: &I, NewN: Res);
7863 return;
7864 }
7865 case Intrinsic::stackprotector: {
7866 // Emit code into the DAG to store the stack guard onto the stack.
7867 MachineFunction &MF = DAG.getMachineFunction();
7868 MachineFrameInfo &MFI = MF.getFrameInfo();
7869 const Module &M = *MF.getFunction().getParent();
7870 SDValue Src, Chain = getRoot();
7871
7872 if (TLI.useLoadStackGuardNode(M))
7873 Src = getLoadStackGuard(DAG, DL: sdl, Chain);
7874 else
7875 Src = getValue(V: I.getArgOperand(i: 0)); // The guard's value.
7876
7877 AllocaInst *Slot = cast<AllocaInst>(Val: I.getArgOperand(i: 1));
7878
7879 int FI = FuncInfo.StaticAllocaMap[Slot];
7880 MFI.setStackProtectorIndex(FI);
7881 EVT PtrTy = TLI.getFrameIndexTy(DL: DAG.getDataLayout());
7882
7883 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrTy);
7884
7885 // Store the stack protector onto the stack.
7886 Res = DAG.getStore(
7887 Chain, dl: sdl, Val: Src, Ptr: FIN,
7888 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI),
7889 Alignment: MaybeAlign(), MMOFlags: MachineMemOperand::MOVolatile);
7890 setValue(V: &I, NewN: Res);
7891 DAG.setRoot(Res);
7892 return;
7893 }
7894 case Intrinsic::objectsize:
7895 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7896
7897 case Intrinsic::is_constant:
7898 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7899
7900 case Intrinsic::annotation:
7901 case Intrinsic::ptr_annotation:
7902 case Intrinsic::launder_invariant_group:
7903 // Drop the intrinsic, but forward the value
7904 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
7905 return;
7906
7907 case Intrinsic::type_test:
7908 case Intrinsic::public_type_test:
7909 case Intrinsic::type_checked_load:
7910 case Intrinsic::type_checked_load_relative: {
7911 // These intrinsics are expected to be lowered by the LowerTypeTests pass
7912 // before code generation. Surviving until here usually indicates a
7913 // misconfiguration, for instance when devirtualization is enabled but LTO
7914 // does not actually run.
7915 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7916 *I.getFunction(),
7917 Intrinsic::getBaseName(id: Intrinsic) +
7918 " intrinsic must be lowered by the LowerTypeTests pass "
7919 "before code generation",
7920 sdl.getDebugLoc()));
7921
7922 // Lower the result to poison so that compilation can continue and collect
7923 // any further diagnostics.
7924 setValueToPoison(V: &I, dl: sdl);
7925 return;
7926 }
7927
7928 case Intrinsic::assume:
7929 case Intrinsic::experimental_noalias_scope_decl:
7930 case Intrinsic::var_annotation:
7931 case Intrinsic::sideeffect:
7932 // Discard annotate attributes, noalias scope declarations, assumptions, and
7933 // artificial side-effects.
7934 return;
7935
7936 case Intrinsic::codeview_annotation: {
7937 // Emit a label associated with this metadata.
7938 MachineFunction &MF = DAG.getMachineFunction();
7939 MCSymbol *Label = MF.getContext().createTempSymbol(Name: "annotation", AlwaysAddSuffix: true);
7940 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
7941 MF.addCodeViewAnnotation(Label, MD: cast<MDNode>(Val: MD));
7942 Res = DAG.getLabelNode(Opcode: ISD::ANNOTATION_LABEL, dl: sdl, Root: getRoot(), Label);
7943 DAG.setRoot(Res);
7944 return;
7945 }
7946
7947 case Intrinsic::init_trampoline: {
7948 const Function *F = cast<Function>(Val: I.getArgOperand(i: 1)->stripPointerCasts());
7949
7950 SDValue Ops[6];
7951 Ops[0] = getRoot();
7952 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7953 Ops[2] = getValue(V: I.getArgOperand(i: 1));
7954 Ops[3] = getValue(V: I.getArgOperand(i: 2));
7955 Ops[4] = DAG.getSrcValue(v: I.getArgOperand(i: 0));
7956 Ops[5] = DAG.getSrcValue(v: F);
7957
7958 Res = DAG.getNode(Opcode: ISD::INIT_TRAMPOLINE, DL: sdl, VT: MVT::Other, Ops);
7959
7960 DAG.setRoot(Res);
7961 return;
7962 }
7963 case Intrinsic::adjust_trampoline:
7964 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ADJUST_TRAMPOLINE, DL: sdl,
7965 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
7966 Operand: getValue(V: I.getArgOperand(i: 0))));
7967 return;
7968 case Intrinsic::gcroot: {
7969 assert(DAG.getMachineFunction().getFunction().hasGC() &&
7970 "only valid in functions with gc specified, enforced by Verifier");
7971 assert(GFI && "implied by previous");
7972 const Value *Alloca = I.getArgOperand(i: 0)->stripPointerCasts();
7973 const Constant *TypeMap = cast<Constant>(Val: I.getArgOperand(i: 1));
7974
7975 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(Val: getValue(V: Alloca).getNode());
7976 GFI->addStackRoot(Num: FI->getIndex(), Metadata: TypeMap);
7977 return;
7978 }
7979 case Intrinsic::gcread:
7980 case Intrinsic::gcwrite:
7981 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7982 case Intrinsic::get_rounding:
7983 Res = DAG.getNode(Opcode: ISD::GET_ROUNDING, DL: sdl, ResultTys: {MVT::i32, MVT::Other}, Ops: getRoot());
7984 setValue(V: &I, NewN: Res);
7985 DAG.setRoot(Res.getValue(R: 1));
7986 return;
7987
7988 case Intrinsic::expect:
7989 case Intrinsic::expect_with_probability:
7990 // Just replace __builtin_expect(exp, c) and
7991 // __builtin_expect_with_probability(exp, c, p) with EXP.
7992 setValue(V: &I, NewN: getValue(V: I.getArgOperand(i: 0)));
7993 return;
7994
7995 case Intrinsic::ubsantrap:
7996 case Intrinsic::debugtrap:
7997 case Intrinsic::trap: {
7998 StringRef TrapFuncName =
7999 I.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
8000 if (TrapFuncName.empty()) {
8001 switch (Intrinsic) {
8002 case Intrinsic::trap:
8003 DAG.setRoot(DAG.getNode(Opcode: ISD::TRAP, DL: sdl, VT: MVT::Other, Operand: getRoot()));
8004 break;
8005 case Intrinsic::debugtrap:
8006 DAG.setRoot(DAG.getNode(Opcode: ISD::DEBUGTRAP, DL: sdl, VT: MVT::Other, Operand: getRoot()));
8007 break;
8008 case Intrinsic::ubsantrap:
8009 DAG.setRoot(DAG.getNode(
8010 Opcode: ISD::UBSANTRAP, DL: sdl, VT: MVT::Other, N1: getRoot(),
8011 N2: DAG.getTargetConstant(
8012 Val: cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue(), DL: sdl,
8013 VT: MVT::i32)));
8014 break;
8015 default: llvm_unreachable("unknown trap intrinsic");
8016 }
8017 DAG.addNoMergeSiteInfo(Node: DAG.getRoot().getNode(),
8018 NoMerge: I.hasFnAttr(Kind: Attribute::NoMerge));
8019 return;
8020 }
8021 TargetLowering::ArgListTy Args;
8022 if (Intrinsic == Intrinsic::ubsantrap) {
8023 Value *Arg = I.getArgOperand(i: 0);
8024 Args.emplace_back(args&: Arg, args: getValue(V: Arg),
8025 args: LibInfo->getExtAttrForI8Param(/*Signed=*/false));
8026 }
8027
8028 TargetLowering::CallLoweringInfo CLI(DAG);
8029 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
8030 CC: CallingConv::C, ResultType: I.getType(),
8031 Target: DAG.getExternalSymbol(Sym: TrapFuncName.data(),
8032 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
8033 ArgsList: std::move(Args));
8034 CLI.NoMerge = I.hasFnAttr(Kind: Attribute::NoMerge);
8035 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8036 DAG.setRoot(Result.second);
8037 return;
8038 }
8039
8040 case Intrinsic::allow_runtime_check:
8041 case Intrinsic::allow_ubsan_check:
8042 setValue(V: &I, NewN: getValue(V: ConstantInt::getTrue(Ty: I.getType())));
8043 return;
8044
8045 case Intrinsic::uadd_with_overflow:
8046 case Intrinsic::sadd_with_overflow:
8047 case Intrinsic::usub_with_overflow:
8048 case Intrinsic::ssub_with_overflow:
8049 case Intrinsic::umul_with_overflow:
8050 case Intrinsic::smul_with_overflow: {
8051 ISD::NodeType Op;
8052 switch (Intrinsic) {
8053 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8054 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
8055 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
8056 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
8057 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
8058 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
8059 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
8060 }
8061 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
8062 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
8063
8064 EVT ResultVT = Op1.getValueType();
8065 EVT OverflowVT = ResultVT.changeElementType(Context&: *Context, EltVT: MVT::i1);
8066
8067 SDVTList VTs = DAG.getVTList(VT1: ResultVT, VT2: OverflowVT);
8068 setValue(V: &I, NewN: DAG.getNode(Opcode: Op, DL: sdl, VTList: VTs, N1: Op1, N2: Op2));
8069 return;
8070 }
8071 case Intrinsic::prefetch: {
8072 SDValue Ops[5];
8073 unsigned rw = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
8074 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
8075 Ops[0] = DAG.getRoot();
8076 Ops[1] = getValue(V: I.getArgOperand(i: 0));
8077 Ops[2] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 1)), DL: sdl,
8078 VT: MVT::i32);
8079 Ops[3] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 2)), DL: sdl,
8080 VT: MVT::i32);
8081 Ops[4] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 3)), DL: sdl,
8082 VT: MVT::i32);
8083 SDValue Result = DAG.getMemIntrinsicNode(
8084 Opcode: ISD::PREFETCH, dl: sdl, VTList: DAG.getVTList(VT: MVT::Other), Ops,
8085 MemVT: EVT::getIntegerVT(Context&: *Context, BitWidth: 8), PtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
8086 /* align */ Alignment: std::nullopt, Flags);
8087
8088 // Chain the prefetch in parallel with any pending loads, to stay out of
8089 // the way of later optimizations.
8090 PendingLoads.push_back(Elt: Result);
8091 Result = getRoot();
8092 DAG.setRoot(Result);
8093 return;
8094 }
8095 case Intrinsic::lifetime_start:
8096 case Intrinsic::lifetime_end: {
8097 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
8098 // Stack coloring is not enabled in O0, discard region information.
8099 if (TM.getOptLevel() == CodeGenOptLevel::None)
8100 return;
8101
8102 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(Val: I.getArgOperand(i: 0));
8103 if (!LifetimeObject)
8104 return;
8105
8106 // First check that the Alloca is static, otherwise it won't have a
8107 // valid frame index.
8108 auto SI = FuncInfo.StaticAllocaMap.find(Val: LifetimeObject);
8109 if (SI == FuncInfo.StaticAllocaMap.end())
8110 return;
8111
8112 const int FrameIndex = SI->second;
8113 Res = DAG.getLifetimeNode(IsStart, dl: sdl, Chain: getRoot(), FrameIndex);
8114 DAG.setRoot(Res);
8115 return;
8116 }
8117 case Intrinsic::pseudoprobe: {
8118 auto Guid = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue();
8119 auto Index = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
8120 auto Attr = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
8121 Res = DAG.getPseudoProbeNode(Dl: sdl, Chain: getRoot(), Guid, Index, Attr);
8122 DAG.setRoot(Res);
8123 return;
8124 }
8125 case Intrinsic::invariant_start:
8126 // Discard region information.
8127 setValue(V: &I,
8128 NewN: DAG.getUNDEF(VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
8129 return;
8130 case Intrinsic::invariant_end:
8131 // Discard region information.
8132 return;
8133 case Intrinsic::clear_cache: {
8134 SDValue InputChain = DAG.getRoot();
8135 SDValue StartVal = getValue(V: I.getArgOperand(i: 0));
8136 SDValue EndVal = getValue(V: I.getArgOperand(i: 1));
8137 Res = DAG.getNode(Opcode: ISD::CLEAR_CACHE, DL: sdl, VTList: DAG.getVTList(VT: MVT::Other),
8138 Ops: {InputChain, StartVal, EndVal});
8139 setValue(V: &I, NewN: Res);
8140 DAG.setRoot(Res);
8141 return;
8142 }
8143 case Intrinsic::donothing:
8144 case Intrinsic::seh_try_begin:
8145 case Intrinsic::seh_scope_begin:
8146 case Intrinsic::seh_try_end:
8147 case Intrinsic::seh_scope_end:
8148 // ignore
8149 return;
8150 case Intrinsic::experimental_stackmap:
8151 visitStackmap(I);
8152 return;
8153 case Intrinsic::experimental_patchpoint_void:
8154 case Intrinsic::experimental_patchpoint:
8155 visitPatchpoint(CB: I);
8156 return;
8157 case Intrinsic::experimental_gc_statepoint:
8158 LowerStatepoint(I: cast<GCStatepointInst>(Val: I));
8159 return;
8160 case Intrinsic::experimental_gc_result:
8161 visitGCResult(I: cast<GCResultInst>(Val: I));
8162 return;
8163 case Intrinsic::experimental_gc_relocate:
8164 visitGCRelocate(Relocate: cast<GCRelocateInst>(Val: I));
8165 return;
8166 case Intrinsic::instrprof_cover:
8167 llvm_unreachable("instrprof failed to lower a cover");
8168 case Intrinsic::instrprof_increment:
8169 llvm_unreachable("instrprof failed to lower an increment");
8170 case Intrinsic::instrprof_timestamp:
8171 llvm_unreachable("instrprof failed to lower a timestamp");
8172 case Intrinsic::instrprof_value_profile:
8173 llvm_unreachable("instrprof failed to lower a value profiling call");
8174 case Intrinsic::instrprof_mcdc_parameters:
8175 llvm_unreachable("instrprof failed to lower mcdc parameters");
8176 case Intrinsic::instrprof_mcdc_tvbitmap_update:
8177 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
8178 case Intrinsic::localescape: {
8179 MachineFunction &MF = DAG.getMachineFunction();
8180 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
8181
8182 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
8183 // is the same on all targets.
8184 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
8185 Value *Arg = I.getArgOperand(i: Idx)->stripPointerCasts();
8186 if (isa<ConstantPointerNull>(Val: Arg))
8187 continue; // Skip null pointers. They represent a hole in index space.
8188 AllocaInst *Slot = cast<AllocaInst>(Val: Arg);
8189 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
8190 "can only escape static allocas");
8191 int FI = FuncInfo.StaticAllocaMap[Slot];
8192 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8193 FuncName: GlobalValue::dropLLVMManglingEscape(Name: MF.getName()), Idx);
8194 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: dl,
8195 MCID: TII->get(Opcode: TargetOpcode::LOCAL_ESCAPE))
8196 .addSym(Sym: FrameAllocSym)
8197 .addFrameIndex(Idx: FI);
8198 }
8199
8200 return;
8201 }
8202
8203 case Intrinsic::localrecover: {
8204 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
8205 MachineFunction &MF = DAG.getMachineFunction();
8206
8207 // Get the symbol that defines the frame offset.
8208 auto *Fn = cast<Function>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
8209 auto *Idx = cast<ConstantInt>(Val: I.getArgOperand(i: 2));
8210 unsigned IdxVal =
8211 unsigned(Idx->getLimitedValue(Limit: std::numeric_limits<int>::max()));
8212 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8213 FuncName: GlobalValue::dropLLVMManglingEscape(Name: Fn->getName()), Idx: IdxVal);
8214
8215 Value *FP = I.getArgOperand(i: 1);
8216 SDValue FPVal = getValue(V: FP);
8217 EVT PtrVT = FPVal.getValueType();
8218
8219 // Create a MCSymbol for the label to avoid any target lowering
8220 // that would make this PC relative.
8221 SDValue OffsetSym = DAG.getMCSymbol(Sym: FrameAllocSym, VT: PtrVT);
8222 SDValue OffsetVal =
8223 DAG.getNode(Opcode: ISD::LOCAL_RECOVER, DL: sdl, VT: PtrVT, Operand: OffsetSym);
8224
8225 // Add the offset to the FP.
8226 SDValue Add = DAG.getMemBasePlusOffset(Base: FPVal, Offset: OffsetVal, DL: sdl);
8227 setValue(V: &I, NewN: Add);
8228
8229 return;
8230 }
8231
8232 case Intrinsic::fake_use: {
8233 Value *V = I.getArgOperand(i: 0);
8234 SDValue Ops[2];
8235 // For Values not declared or previously used in this basic block, the
8236 // NodeMap will not have an entry, and `getValue` will assert if V has no
8237 // valid register value.
8238 auto FakeUseValue = [&]() -> SDValue {
8239 SDValue &N = NodeMap[V];
8240 if (N.getNode())
8241 return N;
8242
8243 // If there's a virtual register allocated and initialized for this
8244 // value, use it.
8245 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
8246 return copyFromReg;
8247 // FIXME: Do we want to preserve constants? It seems pointless.
8248 if (isa<Constant>(Val: V))
8249 return getValue(V);
8250 return SDValue();
8251 }();
8252 if (!FakeUseValue || FakeUseValue.isUndef())
8253 return;
8254 Ops[0] = getRoot();
8255 Ops[1] = FakeUseValue;
8256 // Also, do not translate a fake use with an undef operand, or any other
8257 // empty SDValues.
8258 if (!Ops[1] || Ops[1].isUndef())
8259 return;
8260 DAG.setRoot(DAG.getNode(Opcode: ISD::FAKE_USE, DL: sdl, VT: MVT::Other, Ops));
8261 return;
8262 }
8263
8264 case Intrinsic::reloc_none: {
8265 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
8266 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
8267 SDValue Ops[2] = {
8268 getRoot(),
8269 DAG.getTargetExternalSymbol(
8270 Sym: SymbolName.data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout()))};
8271 DAG.setRoot(DAG.getNode(Opcode: ISD::RELOC_NONE, DL: sdl, VT: MVT::Other, Ops));
8272 return;
8273 }
8274
8275 case Intrinsic::cond_loop: {
8276 SDValue InputChain = DAG.getRoot();
8277 SDValue P = getValue(V: I.getArgOperand(i: 0));
8278 Res = DAG.getNode(Opcode: ISD::COND_LOOP, DL: sdl, VTList: DAG.getVTList(VT: MVT::Other),
8279 Ops: {InputChain, P});
8280 setValue(V: &I, NewN: Res);
8281 DAG.setRoot(Res);
8282 return;
8283 }
8284
8285 case Intrinsic::eh_exceptionpointer:
8286 case Intrinsic::eh_exceptioncode: {
8287 // Get the exception pointer vreg, copy from it, and resize it to fit.
8288 const auto *CPI = cast<CatchPadInst>(Val: I.getArgOperand(i: 0));
8289 MVT PtrVT = TLI.getPointerTy(DL: DAG.getDataLayout());
8290 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(VT: PtrVT);
8291 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, RC: PtrRC);
8292 SDValue N = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: sdl, Reg: VReg, VT: PtrVT);
8293 if (Intrinsic == Intrinsic::eh_exceptioncode)
8294 N = DAG.getZExtOrTrunc(Op: N, DL: sdl, VT: MVT::i32);
8295 setValue(V: &I, NewN: N);
8296 return;
8297 }
8298 case Intrinsic::xray_customevent: {
8299 // Here we want to make sure that the intrinsic behaves as if it has a
8300 // specific calling convention.
8301 const auto &Triple = DAG.getTarget().getTargetTriple();
8302 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64 &&
8303 Triple.getArch() != Triple::hexagon)
8304 return;
8305
8306 SmallVector<SDValue, 8> Ops;
8307
8308 // We want to say that we always want the arguments in registers.
8309 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 0));
8310 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 1));
8311 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
8312 SDValue Chain = getRoot();
8313 Ops.push_back(Elt: LogEntryVal);
8314 Ops.push_back(Elt: StrSizeVal);
8315 Ops.push_back(Elt: Chain);
8316
8317 // We need to enforce the calling convention for the callsite, so that
8318 // argument ordering is enforced correctly, and that register allocation can
8319 // see that some registers may be assumed clobbered and have to preserve
8320 // them across calls to the intrinsic.
8321 MachineSDNode *MN = DAG.getMachineNode(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL,
8322 dl: sdl, VTs: NodeTys, Ops);
8323 SDValue patchableNode = SDValue(MN, 0);
8324 DAG.setRoot(patchableNode);
8325 setValue(V: &I, NewN: patchableNode);
8326 return;
8327 }
8328 case Intrinsic::xray_typedevent: {
8329 // Here we want to make sure that the intrinsic behaves as if it has a
8330 // specific calling convention.
8331 const auto &Triple = DAG.getTarget().getTargetTriple();
8332 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64 &&
8333 Triple.getArch() != Triple::hexagon)
8334 return;
8335
8336 SmallVector<SDValue, 8> Ops;
8337
8338 // We want to say that we always want the arguments in registers.
8339 // It's unclear to me how manipulating the selection DAG here forces callers
8340 // to provide arguments in registers instead of on the stack.
8341 SDValue LogTypeId = getValue(V: I.getArgOperand(i: 0));
8342 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 1));
8343 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 2));
8344 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
8345 SDValue Chain = getRoot();
8346 Ops.push_back(Elt: LogTypeId);
8347 Ops.push_back(Elt: LogEntryVal);
8348 Ops.push_back(Elt: StrSizeVal);
8349 Ops.push_back(Elt: Chain);
8350
8351 // We need to enforce the calling convention for the callsite, so that
8352 // argument ordering is enforced correctly, and that register allocation can
8353 // see that some registers may be assumed clobbered and have to preserve
8354 // them across calls to the intrinsic.
8355 MachineSDNode *MN = DAG.getMachineNode(
8356 Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, dl: sdl, VTs: NodeTys, Ops);
8357 SDValue patchableNode = SDValue(MN, 0);
8358 DAG.setRoot(patchableNode);
8359 setValue(V: &I, NewN: patchableNode);
8360 return;
8361 }
8362 case Intrinsic::experimental_deoptimize:
8363 LowerDeoptimizeCall(CI: &I);
8364 return;
8365 case Intrinsic::stepvector:
8366 visitStepVector(I);
8367 return;
8368 case Intrinsic::vector_reduce_fadd:
8369 case Intrinsic::vector_reduce_fmul:
8370 case Intrinsic::vector_reduce_add:
8371 case Intrinsic::vector_reduce_mul:
8372 case Intrinsic::vector_reduce_and:
8373 case Intrinsic::vector_reduce_or:
8374 case Intrinsic::vector_reduce_xor:
8375 case Intrinsic::vector_reduce_smax:
8376 case Intrinsic::vector_reduce_smin:
8377 case Intrinsic::vector_reduce_umax:
8378 case Intrinsic::vector_reduce_umin:
8379 case Intrinsic::vector_reduce_fmax:
8380 case Intrinsic::vector_reduce_fmin:
8381 case Intrinsic::vector_reduce_fmaximum:
8382 case Intrinsic::vector_reduce_fminimum:
8383 case Intrinsic::vector_reduce_fmaximumnum:
8384 case Intrinsic::vector_reduce_fminimumnum:
8385 visitVectorReduce(I, Intrinsic);
8386 return;
8387
8388 case Intrinsic::icall_branch_funnel: {
8389 SmallVector<SDValue, 16> Ops;
8390 Ops.push_back(Elt: getValue(V: I.getArgOperand(i: 0)));
8391
8392 int64_t Offset;
8393 auto *Base = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
8394 Ptr: I.getArgOperand(i: 1), Offset, DL: DAG.getDataLayout()));
8395 if (!Base)
8396 report_fatal_error(
8397 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
8398 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Base, DL: sdl, VT: MVT::i64, offset: 0));
8399
8400 struct BranchFunnelTarget {
8401 int64_t Offset;
8402 SDValue Target;
8403 };
8404 SmallVector<BranchFunnelTarget, 8> Targets;
8405
8406 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
8407 auto *ElemBase = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
8408 Ptr: I.getArgOperand(i: Op), Offset, DL: DAG.getDataLayout()));
8409 if (ElemBase != Base)
8410 report_fatal_error(reason: "all llvm.icall.branch.funnel operands must refer "
8411 "to the same GlobalValue");
8412
8413 SDValue Val = getValue(V: I.getArgOperand(i: Op + 1));
8414 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
8415 if (!GA)
8416 report_fatal_error(
8417 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
8418 Targets.push_back(Elt: {.Offset: Offset, .Target: DAG.getTargetGlobalAddress(
8419 GV: GA->getGlobal(), DL: sdl, VT: Val.getValueType(),
8420 offset: GA->getOffset())});
8421 }
8422 llvm::sort(C&: Targets,
8423 Comp: [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
8424 return T1.Offset < T2.Offset;
8425 });
8426
8427 for (auto &T : Targets) {
8428 Ops.push_back(Elt: DAG.getTargetConstant(Val: T.Offset, DL: sdl, VT: MVT::i32));
8429 Ops.push_back(Elt: T.Target);
8430 }
8431
8432 Ops.push_back(Elt: DAG.getRoot()); // Chain
8433 SDValue N(DAG.getMachineNode(Opcode: TargetOpcode::ICALL_BRANCH_FUNNEL, dl: sdl,
8434 VT: MVT::Other, Ops),
8435 0);
8436 DAG.setRoot(N);
8437 setValue(V: &I, NewN: N);
8438 HasTailCall = true;
8439 return;
8440 }
8441
8442 case Intrinsic::wasm_landingpad_index:
8443 // Information this intrinsic contained has been transferred to
8444 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
8445 // delete it now.
8446 return;
8447
8448 case Intrinsic::aarch64_settag:
8449 case Intrinsic::aarch64_settag_zero: {
8450 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8451 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
8452 SDValue Val = TSI.EmitTargetCodeForSetTag(
8453 DAG, dl: sdl, Chain: getRoot(), Addr: getValue(V: I.getArgOperand(i: 0)),
8454 Size: getValue(V: I.getArgOperand(i: 1)), DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
8455 ZeroData: ZeroMemory);
8456 DAG.setRoot(Val);
8457 setValue(V: &I, NewN: Val);
8458 return;
8459 }
8460 case Intrinsic::amdgcn_cs_chain: {
8461 // At this point we don't care if it's amdgpu_cs_chain or
8462 // amdgpu_cs_chain_preserve.
8463 CallingConv::ID CC = CallingConv::AMDGPU_CS_Chain;
8464
8465 Type *RetTy = I.getType();
8466 assert(RetTy->isVoidTy() && "Should not return");
8467
8468 SDValue Callee = getValue(V: I.getOperand(i_nocapture: 0));
8469
8470 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
8471 // We'll also tack the value of the EXEC mask at the end.
8472 TargetLowering::ArgListTy Args;
8473 Args.reserve(n: 3);
8474
8475 for (unsigned Idx : {2, 3, 1}) {
8476 TargetLowering::ArgListEntry Arg(getValue(V: I.getOperand(i_nocapture: Idx)),
8477 I.getOperand(i_nocapture: Idx)->getType());
8478 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8479 Args.push_back(x: Arg);
8480 }
8481
8482 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
8483 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
8484 Args[2].IsInReg = true; // EXEC should be inreg
8485
8486 // Forward the flags and any additional arguments.
8487 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
8488 TargetLowering::ArgListEntry Arg(getValue(V: I.getOperand(i_nocapture: Idx)),
8489 I.getOperand(i_nocapture: Idx)->getType());
8490 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8491 Args.push_back(x: Arg);
8492 }
8493
8494 TargetLowering::CallLoweringInfo CLI(DAG);
8495 CLI.setDebugLoc(getCurSDLoc())
8496 .setChain(getRoot())
8497 .setCallee(CC, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
8498 .setNoReturn(true)
8499 .setTailCall(true)
8500 .setConvergent(I.isConvergent());
8501 CLI.CB = &I;
8502 std::pair<SDValue, SDValue> Result =
8503 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
8504 (void)Result;
8505 assert(!Result.first.getNode() && !Result.second.getNode() &&
8506 "Should've lowered as tail call");
8507
8508 HasTailCall = true;
8509 return;
8510 }
8511 case Intrinsic::amdgcn_call_whole_wave: {
8512 TargetLowering::ArgListTy Args;
8513 bool isTailCall = I.isTailCall();
8514
8515 // The first argument is the callee. Skip it when assembling the call args.
8516 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
8517 TargetLowering::ArgListEntry Arg(getValue(V: I.getArgOperand(i: Idx)),
8518 I.getArgOperand(i: Idx)->getType());
8519 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8520
8521 // If we have an explicit sret argument that is an Instruction, (i.e., it
8522 // might point to function-local memory), we can't meaningfully tail-call.
8523 if (Arg.IsSRet && isa<Instruction>(Val: I.getArgOperand(i: Idx)))
8524 isTailCall = false;
8525
8526 Args.push_back(x: Arg);
8527 }
8528
8529 SDValue ConvControlToken;
8530 if (auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
8531 auto *Token = Bundle->Inputs[0].get();
8532 ConvControlToken = getValue(V: Token);
8533 }
8534
8535 TargetLowering::CallLoweringInfo CLI(DAG);
8536 CLI.setDebugLoc(getCurSDLoc())
8537 .setChain(getRoot())
8538 .setCallee(CC: CallingConv::AMDGPU_Gfx_WholeWave, ResultType: I.getType(),
8539 Target: getValue(V: I.getArgOperand(i: 0)), ArgsList: std::move(Args))
8540 .setTailCall(isTailCall && canTailCall(CB: I))
8541 .setIsPreallocated(
8542 I.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
8543 .setConvergent(I.isConvergent())
8544 .setConvergenceControlToken(ConvControlToken);
8545 CLI.CB = &I;
8546
8547 std::pair<SDValue, SDValue> Result =
8548 lowerInvokable(CLI, /*EHPadBB=*/nullptr);
8549
8550 if (Result.first.getNode())
8551 setValue(V: &I, NewN: Result.first);
8552 return;
8553 }
8554 case Intrinsic::ptrmask: {
8555 SDValue Ptr = getValue(V: I.getOperand(i_nocapture: 0));
8556 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
8557
8558 // On arm64_32, pointers are 32 bits when stored in memory, but
8559 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
8560 // match the index type, but the pointer is 64 bits, so the mask must be
8561 // zero-extended up to 64 bits to match the pointer.
8562 EVT PtrVT =
8563 TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
8564 EVT MemVT =
8565 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
8566 assert(PtrVT == Ptr.getValueType());
8567 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
8568 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
8569 // 128-bit, so we have to pad the mask with ones for unused bits.
8570 auto HighOnes = DAG.getNode(
8571 Opcode: ISD::SHL, DL: sdl, VT: PtrVT, N1: DAG.getAllOnesConstant(DL: sdl, VT: PtrVT),
8572 N2: DAG.getShiftAmountConstant(Val: Mask.getValueType().getFixedSizeInBits(),
8573 VT: PtrVT, DL: sdl));
8574 Mask = DAG.getNode(Opcode: ISD::OR, DL: sdl, VT: PtrVT,
8575 N1: DAG.getZExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT), N2: HighOnes);
8576 } else if (Mask.getValueType() != PtrVT)
8577 Mask = DAG.getPtrExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT);
8578
8579 assert(Mask.getValueType() == PtrVT);
8580 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::AND, DL: sdl, VT: PtrVT, N1: Ptr, N2: Mask));
8581 return;
8582 }
8583 case Intrinsic::threadlocal_address: {
8584 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
8585 return;
8586 }
8587 case Intrinsic::get_active_lane_mask: {
8588 EVT CCVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8589 SDValue Index = getValue(V: I.getOperand(i_nocapture: 0));
8590 SDValue TripCount = getValue(V: I.getOperand(i_nocapture: 1));
8591 EVT ElementVT = Index.getValueType();
8592
8593 if (!TLI.shouldExpandGetActiveLaneMask(VT: CCVT, OpVT: ElementVT)) {
8594 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: sdl, VT: CCVT, N1: Index,
8595 N2: TripCount));
8596 return;
8597 }
8598
8599 EVT VecTy = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElementVT,
8600 EC: CCVT.getVectorElementCount());
8601
8602 SDValue VectorIndex = DAG.getSplat(VT: VecTy, DL: sdl, Op: Index);
8603 SDValue VectorTripCount = DAG.getSplat(VT: VecTy, DL: sdl, Op: TripCount);
8604 SDValue VectorStep = DAG.getStepVector(DL: sdl, ResVT: VecTy);
8605 SDValue VectorInduction = DAG.getNode(
8606 Opcode: ISD::UADDSAT, DL: sdl, VT: VecTy, N1: VectorIndex, N2: VectorStep);
8607 SDValue SetCC = DAG.getSetCC(DL: sdl, VT: CCVT, LHS: VectorInduction,
8608 RHS: VectorTripCount, Cond: ISD::CondCode::SETULT);
8609 setValue(V: &I, NewN: SetCC);
8610 return;
8611 }
8612 case Intrinsic::mask_beforefirst: {
8613 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
8614 setValue(V: &I,
8615 NewN: DAG.getNode(Opcode: ISD::MASK_BEFOREFIRST, DL: sdl, VT: Op.getValueType(), Operand: Op));
8616 return;
8617 }
8618 case Intrinsic::experimental_get_vector_length: {
8619 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8620 "Expected positive VF");
8621 unsigned VF = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 1))->getZExtValue();
8622 bool IsScalable = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->isOne();
8623
8624 SDValue Count = getValue(V: I.getOperand(i_nocapture: 0));
8625 EVT CountVT = Count.getValueType();
8626
8627 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8628 visitTargetIntrinsic(I, Intrinsic);
8629 return;
8630 }
8631
8632 // Expand to a umin between the trip count and the maximum elements the type
8633 // can hold.
8634 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8635
8636 // Extend the trip count to at least the result VT.
8637 if (CountVT.bitsLT(VT)) {
8638 Count = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: sdl, VT, Operand: Count);
8639 CountVT = VT;
8640 }
8641
8642 SDValue MaxEVL = DAG.getElementCount(DL: sdl, VT: CountVT,
8643 EC: ElementCount::get(MinVal: VF, Scalable: IsScalable));
8644
8645 SDValue UMin = DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: CountVT, N1: Count, N2: MaxEVL);
8646 // Clip to the result type if needed.
8647 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: sdl, VT, Operand: UMin);
8648
8649 setValue(V: &I, NewN: Trunc);
8650 return;
8651 }
8652 case Intrinsic::vector_partial_reduce_add: {
8653 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8654 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8655 setValue(V: &I,
8656 NewN: DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_UMLA, DL: sdl, VT: Acc.getValueType(), N1: Acc,
8657 N2: Input, N3: DAG.getConstant(Val: 1, DL: sdl, VT: Input.getValueType())));
8658 return;
8659 }
8660 case Intrinsic::vector_partial_reduce_fadd: {
8661 SDNodeFlags Flags;
8662 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
8663 Flags.copyFMF(FPMO: *FPOp);
8664 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8665 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8666 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_FMLA, DL: sdl, VT: Acc.getValueType(),
8667 N1: Acc, N2: Input,
8668 N3: DAG.getConstantFP(Val: 1.0, DL: sdl, VT: Input.getValueType()),
8669 Flags));
8670 return;
8671 }
8672 case Intrinsic::experimental_cttz_elts: {
8673 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
8674 EVT OpVT = Op.getValueType();
8675 EVT RetTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8676 bool ZeroIsPoison =
8677 !cast<ConstantSDNode>(Val: getValue(V: I.getOperand(i_nocapture: 1)))->isZero();
8678 if (OpVT.getVectorElementType() != MVT::i1) {
8679 // Compare the input vector elements to zero & use to count trailing
8680 // zeros.
8681 SDValue AllZero = DAG.getConstant(Val: 0, DL: sdl, VT: OpVT);
8682 EVT I1OpVT = OpVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1);
8683 Op = DAG.getSetCC(DL: sdl, VT: I1OpVT, LHS: Op, RHS: AllZero, Cond: ISD::SETNE);
8684 }
8685 setValue(V: &I, NewN: DAG.getNode(Opcode: ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
8686 : ISD::CTTZ_ELTS,
8687 DL: sdl, VT: RetTy, Operand: Op));
8688 return;
8689 }
8690 case Intrinsic::vector_insert: {
8691 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8692 SDValue SubVec = getValue(V: I.getOperand(i_nocapture: 1));
8693 SDValue Index = getValue(V: I.getOperand(i_nocapture: 2));
8694
8695 // The intrinsic's index type is i64, but the SDNode requires an index type
8696 // suitable for the target. Convert the index as required.
8697 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8698 if (Index.getValueType() != VectorIdxTy)
8699 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8700
8701 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8702 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: SubVec,
8703 N3: Index));
8704 return;
8705 }
8706 case Intrinsic::vector_extract: {
8707 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8708 SDValue Index = getValue(V: I.getOperand(i_nocapture: 1));
8709 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8710
8711 // The intrinsic's index type is i64, but the SDNode requires an index type
8712 // suitable for the target. Convert the index as required.
8713 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8714 if (Index.getValueType() != VectorIdxTy)
8715 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8716
8717 setValue(V: &I,
8718 NewN: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: Index));
8719 return;
8720 }
8721 case Intrinsic::experimental_vector_match: {
8722 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
8723 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
8724 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
8725 EVT ResVT = Mask.getValueType();
8726 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL: sdl, VT: ResVT, N1: Op1, N2: Op2, N3: Mask));
8727 return;
8728 }
8729 case Intrinsic::vector_reverse:
8730 visitVectorReverse(I);
8731 return;
8732 case Intrinsic::vector_splice_left:
8733 case Intrinsic::vector_splice_right:
8734 visitVectorSplice(I);
8735 return;
8736 case Intrinsic::callbr_landingpad:
8737 visitCallBrLandingPad(I);
8738 return;
8739 case Intrinsic::vector_interleave2:
8740 visitVectorInterleave(I, Factor: 2);
8741 return;
8742 case Intrinsic::vector_interleave3:
8743 visitVectorInterleave(I, Factor: 3);
8744 return;
8745 case Intrinsic::vector_interleave4:
8746 visitVectorInterleave(I, Factor: 4);
8747 return;
8748 case Intrinsic::vector_interleave5:
8749 visitVectorInterleave(I, Factor: 5);
8750 return;
8751 case Intrinsic::vector_interleave6:
8752 visitVectorInterleave(I, Factor: 6);
8753 return;
8754 case Intrinsic::vector_interleave7:
8755 visitVectorInterleave(I, Factor: 7);
8756 return;
8757 case Intrinsic::vector_interleave8:
8758 visitVectorInterleave(I, Factor: 8);
8759 return;
8760 case Intrinsic::vector_deinterleave2:
8761 visitVectorDeinterleave(I, Factor: 2);
8762 return;
8763 case Intrinsic::vector_deinterleave3:
8764 visitVectorDeinterleave(I, Factor: 3);
8765 return;
8766 case Intrinsic::vector_deinterleave4:
8767 visitVectorDeinterleave(I, Factor: 4);
8768 return;
8769 case Intrinsic::vector_deinterleave5:
8770 visitVectorDeinterleave(I, Factor: 5);
8771 return;
8772 case Intrinsic::vector_deinterleave6:
8773 visitVectorDeinterleave(I, Factor: 6);
8774 return;
8775 case Intrinsic::vector_deinterleave7:
8776 visitVectorDeinterleave(I, Factor: 7);
8777 return;
8778 case Intrinsic::vector_deinterleave8:
8779 visitVectorDeinterleave(I, Factor: 8);
8780 return;
8781 case Intrinsic::vector_repeat: {
8782 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8783 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8784 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL: sdl, VT: ResultVT, Operand: Vec));
8785 return;
8786 }
8787 case Intrinsic::experimental_vector_compress:
8788 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: sdl,
8789 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
8790 N1: getValue(V: I.getArgOperand(i: 0)),
8791 N2: getValue(V: I.getArgOperand(i: 1)),
8792 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
8793 return;
8794 case Intrinsic::vector_shuffle:
8795 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_SHUFFLE_VAR, DL: sdl,
8796 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
8797 N1: getValue(V: I.getArgOperand(i: 0)),
8798 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
8799 return;
8800 case Intrinsic::experimental_convergence_anchor:
8801 case Intrinsic::experimental_convergence_entry:
8802 case Intrinsic::experimental_convergence_loop:
8803 visitConvergenceControl(I, Intrinsic);
8804 return;
8805 case Intrinsic::experimental_vector_histogram_add: {
8806 visitVectorHistogram(I, IntrinsicID: Intrinsic);
8807 return;
8808 }
8809 case Intrinsic::experimental_vector_extract_last_active: {
8810 visitVectorExtractLastActive(I, Intrinsic);
8811 return;
8812 }
8813 case Intrinsic::loop_dependence_war_mask:
8814 setValue(V: &I,
8815 NewN: DAG.getNode(Opcode: ISD::LOOP_DEPENDENCE_WAR_MASK, DL: sdl,
8816 VT: EVT::getEVT(Ty: I.getType()), N1: getValue(V: I.getOperand(i_nocapture: 0)),
8817 N2: getValue(V: I.getOperand(i_nocapture: 1)), N3: getValue(V: I.getOperand(i_nocapture: 2)),
8818 N4: DAG.getConstant(Val: 0, DL: sdl, VT: MVT::i64)));
8819 return;
8820 case Intrinsic::loop_dependence_raw_mask:
8821 setValue(V: &I,
8822 NewN: DAG.getNode(Opcode: ISD::LOOP_DEPENDENCE_RAW_MASK, DL: sdl,
8823 VT: EVT::getEVT(Ty: I.getType()), N1: getValue(V: I.getOperand(i_nocapture: 0)),
8824 N2: getValue(V: I.getOperand(i_nocapture: 1)), N3: getValue(V: I.getOperand(i_nocapture: 2)),
8825 N4: DAG.getConstant(Val: 0, DL: sdl, VT: MVT::i64)));
8826 return;
8827 case Intrinsic::masked_udiv:
8828 setValue(V: &I,
8829 NewN: DAG.getNode(Opcode: ISD::MASKED_UDIV, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8830 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8831 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8832 return;
8833 case Intrinsic::masked_sdiv:
8834 setValue(V: &I,
8835 NewN: DAG.getNode(Opcode: ISD::MASKED_SDIV, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8836 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8837 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8838 return;
8839 case Intrinsic::masked_urem:
8840 setValue(V: &I,
8841 NewN: DAG.getNode(Opcode: ISD::MASKED_UREM, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8842 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8843 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8844 return;
8845 case Intrinsic::masked_srem:
8846 setValue(V: &I,
8847 NewN: DAG.getNode(Opcode: ISD::MASKED_SREM, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8848 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8849 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8850 return;
8851 }
8852}
8853
8854void SelectionDAGBuilder::pushFPOpOutChain(SDValue Result,
8855 fp::ExceptionBehavior EB) {
8856 assert(Result.getNode()->getNumValues() == 2);
8857 SDValue OutChain = Result.getValue(R: 1);
8858 assert(OutChain.getValueType() == MVT::Other);
8859
8860 // Instead of updating the root immediately, push the produced chain to the
8861 // appropriate list, deferring the update until the root is requested. In this
8862 // case, the nodes from the lists are chained using TokenFactor, indicating
8863 // that the operations are independent.
8864 //
8865 // In particular, the root is updated before any call that might access the
8866 // floating-point environment, except for constrained intrinsics.
8867 switch (EB) {
8868 case fp::ExceptionBehavior::ebMayTrap:
8869 case fp::ExceptionBehavior::ebIgnore:
8870 PendingConstrainedFP.push_back(Elt: OutChain);
8871 break;
8872 case fp::ExceptionBehavior::ebStrict:
8873 PendingConstrainedFPStrict.push_back(Elt: OutChain);
8874 break;
8875 }
8876}
8877
8878void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8879 const ConstrainedFPIntrinsic &FPI) {
8880 SDLoc sdl = getCurSDLoc();
8881
8882 // We do not need to serialize constrained FP intrinsics against
8883 // each other or against (nonvolatile) loads, so they can be
8884 // chained like loads.
8885 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
8886 SDValue Chain = getFPOperationRoot(EB);
8887 SmallVector<SDValue, 4> Opers;
8888 Opers.push_back(Elt: Chain);
8889 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8890 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: I)));
8891
8892 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8893 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: FPI.getType());
8894 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
8895
8896 SDNodeFlags Flags;
8897 if (EB == fp::ExceptionBehavior::ebIgnore)
8898 Flags.setNoFPExcept(true);
8899
8900 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &FPI))
8901 Flags.copyFMF(FPMO: *FPOp);
8902
8903 unsigned Opcode;
8904 switch (FPI.getIntrinsicID()) {
8905 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8906#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8907 case Intrinsic::INTRINSIC: \
8908 Opcode = ISD::STRICT_##DAGN; \
8909 break;
8910#include "llvm/IR/ConstrainedOps.def"
8911 case Intrinsic::experimental_constrained_fmuladd: {
8912 Opcode = ISD::STRICT_FMA;
8913 // Break fmuladd into fmul and fadd.
8914 if (!TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
8915 Opers.pop_back();
8916 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8917 pushFPOpOutChain(Result: Mul, EB);
8918 Opcode = ISD::STRICT_FADD;
8919 Opers.clear();
8920 Opers.push_back(Elt: Mul.getValue(R: 1));
8921 Opers.push_back(Elt: Mul.getValue(R: 0));
8922 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: 2)));
8923 }
8924 break;
8925 }
8926 }
8927
8928 // A few strict DAG nodes carry additional operands that are not
8929 // set up by the default code above.
8930 switch (Opcode) {
8931 default: break;
8932 case ISD::STRICT_FP_ROUND:
8933 Opers.push_back(
8934 Elt: DAG.getTargetConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
8935 break;
8936 case ISD::STRICT_FSETCC:
8937 case ISD::STRICT_FSETCCS: {
8938 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
8939 ISD::CondCode Condition = getFCmpCondCode(Pred: FPCmp->getPredicate());
8940 if (DAG.isKnownNeverNaN(Op: Opers[1]) && DAG.isKnownNeverNaN(Op: Opers[2]))
8941 Condition = getFCmpCodeWithoutNaN(CC: Condition);
8942 Opers.push_back(Elt: DAG.getCondCode(Cond: Condition));
8943 break;
8944 }
8945 }
8946
8947 SDValue Result = DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8948 pushFPOpOutChain(Result, EB);
8949
8950 SDValue FPResult = Result.getValue(R: 0);
8951 setValue(V: &FPI, NewN: FPResult);
8952}
8953
8954static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8955 std::optional<unsigned> ResOPC;
8956 switch (VPIntrin.getIntrinsicID()) {
8957 case Intrinsic::vp_cttz_elts: {
8958 bool IsZeroPoison = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8959 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_POISON : ISD::VP_CTTZ_ELTS;
8960 break;
8961 }
8962#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8963 case Intrinsic::VPID: \
8964 ResOPC = ISD::VPSD; \
8965 break;
8966#include "llvm/IR/VPIntrinsics.def"
8967 }
8968
8969 if (!ResOPC)
8970 llvm_unreachable(
8971 "Inconsistency: no SDNode available for this VPIntrinsic!");
8972
8973 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8974 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8975 if (VPIntrin.getFastMathFlags().allowReassoc())
8976 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8977 : ISD::VP_REDUCE_FMUL;
8978 }
8979
8980 return *ResOPC;
8981}
8982
8983void SelectionDAGBuilder::visitVPLoad(
8984 const VPIntrinsic &VPIntrin, EVT VT,
8985 const SmallVectorImpl<SDValue> &OpValues) {
8986 SDLoc DL = getCurSDLoc();
8987 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8988 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8989 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8990 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8991 SDValue LD;
8992 // Do not serialize variable-length loads of constant memory with
8993 // anything.
8994 if (!Alignment)
8995 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8996 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8997 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8998 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8999 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9000 MachineMemOperand::Flags MMOFlags =
9001 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9002 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9003 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
9004 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment,
9005 Metadata: MMOMetadata(AAInfo, Ranges));
9006 LD = DAG.getLoadVP(VT, dl: DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
9007 MMO, IsExpanding: false /*IsExpanding */);
9008 if (AddToChain)
9009 PendingLoads.push_back(Elt: LD.getValue(R: 1));
9010 setValue(V: &VPIntrin, NewN: LD);
9011}
9012
9013void SelectionDAGBuilder::visitVPLoadFF(
9014 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,
9015 const SmallVectorImpl<SDValue> &OpValues) {
9016 assert(OpValues.size() == 3 && "Unexpected number of operands");
9017 SDLoc DL = getCurSDLoc();
9018 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
9019 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9020 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9021 const MDNode *Ranges = VPIntrin.getMetadata(KindID: LLVMContext::MD_range);
9022 SDValue LD;
9023 // Do not serialize variable-length loads of constant memory with
9024 // anything.
9025 if (!Alignment)
9026 Alignment = DAG.getEVTAlign(MemoryVT: VT);
9027 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
9028 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
9029 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9030 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9031 PtrInfo: MachinePointerInfo(PtrOperand), F: MachineMemOperand::MOLoad,
9032 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment,
9033 Metadata: MMOMetadata(AAInfo, Ranges));
9034 LD = DAG.getLoadFFVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
9035 MMO);
9036 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EVLVT, Operand: LD.getValue(R: 1));
9037 if (AddToChain)
9038 PendingLoads.push_back(Elt: LD.getValue(R: 2));
9039 setValue(V: &VPIntrin, NewN: DAG.getMergeValues(Ops: {LD.getValue(R: 0), Trunc}, dl: DL));
9040}
9041
9042void SelectionDAGBuilder::visitVPGather(
9043 const VPIntrinsic &VPIntrin, EVT VT,
9044 const SmallVectorImpl<SDValue> &OpValues) {
9045 SDLoc DL = getCurSDLoc();
9046 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9047 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
9048 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9049 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9050 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
9051 SDValue LD;
9052 if (!Alignment)
9053 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9054 unsigned AS =
9055 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9056 MachineMemOperand::Flags MMOFlags =
9057 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9058 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9059 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9060 BaseAlignment: *Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
9061 SDValue Base, Index, Scale;
9062 bool UniformBase =
9063 getUniformBase(Ptr: PtrOperand, Base, Index, Scale, SDB: this, CurBB: VPIntrin.getParent(),
9064 ElemSize: VT.getScalarStoreSize());
9065 if (!UniformBase) {
9066 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9067 Index = getValue(V: PtrOperand);
9068 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9069 }
9070 EVT IdxVT = Index.getValueType();
9071 EVT EltTy = IdxVT.getVectorElementType();
9072 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
9073 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
9074 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
9075 }
9076 LD = DAG.getGatherVP(
9077 VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), VT, dl: DL,
9078 Ops: {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
9079 IndexType: ISD::SIGNED_SCALED);
9080 PendingLoads.push_back(Elt: LD.getValue(R: 1));
9081 setValue(V: &VPIntrin, NewN: LD);
9082}
9083
9084void SelectionDAGBuilder::visitVPStore(
9085 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9086 SDLoc DL = getCurSDLoc();
9087 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
9088 EVT VT = OpValues[0].getValueType();
9089 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9090 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9091 SDValue ST;
9092 if (!Alignment)
9093 Alignment = DAG.getEVTAlign(MemoryVT: VT);
9094 SDValue Ptr = OpValues[1];
9095 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
9096 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9097 MachineMemOperand::Flags MMOFlags =
9098 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9099 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9100 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
9101 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, Metadata: AAInfo);
9102 ST = DAG.getStoreVP(Chain: getMemoryRoot(), dl: DL, Val: OpValues[0], Ptr, Offset,
9103 Mask: OpValues[2], EVL: OpValues[3], MemVT: VT, MMO, AM: ISD::UNINDEXED,
9104 /* IsTruncating */ false, /*IsCompressing*/ false);
9105 DAG.setRoot(ST);
9106 setValue(V: &VPIntrin, NewN: ST);
9107}
9108
9109void SelectionDAGBuilder::visitVPScatter(
9110 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9111 SDLoc DL = getCurSDLoc();
9112 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9113 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
9114 EVT VT = OpValues[0].getValueType();
9115 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9116 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9117 SDValue ST;
9118 if (!Alignment)
9119 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9120 unsigned AS =
9121 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9122 MachineMemOperand::Flags MMOFlags =
9123 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9124 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9125 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9126 BaseAlignment: *Alignment, Metadata: AAInfo);
9127 SDValue Base, Index, Scale;
9128 bool UniformBase =
9129 getUniformBase(Ptr: PtrOperand, Base, Index, Scale, SDB: this, CurBB: VPIntrin.getParent(),
9130 ElemSize: VT.getScalarStoreSize());
9131 if (!UniformBase) {
9132 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9133 Index = getValue(V: PtrOperand);
9134 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9135 }
9136 EVT IdxVT = Index.getValueType();
9137 EVT EltTy = IdxVT.getVectorElementType();
9138 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
9139 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
9140 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
9141 }
9142 ST = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT, dl: DL,
9143 Ops: {getMemoryRoot(), OpValues[0], Base, Index, Scale,
9144 OpValues[2], OpValues[3]},
9145 MMO, IndexType: ISD::SIGNED_SCALED);
9146 DAG.setRoot(ST);
9147 setValue(V: &VPIntrin, NewN: ST);
9148}
9149
9150void SelectionDAGBuilder::visitVPStridedLoad(
9151 const VPIntrinsic &VPIntrin, EVT VT,
9152 const SmallVectorImpl<SDValue> &OpValues) {
9153 SDLoc DL = getCurSDLoc();
9154 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
9155 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9156 if (!Alignment)
9157 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9158 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9159 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
9160 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
9161 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
9162 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9163 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9164 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9165 MachineMemOperand::Flags MMOFlags =
9166 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9167 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9168 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9169 BaseAlignment: *Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
9170
9171 SDValue LD = DAG.getStridedLoadVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Stride: OpValues[1],
9172 Mask: OpValues[2], EVL: OpValues[3], MMO,
9173 IsExpanding: false /*IsExpanding*/);
9174
9175 if (AddToChain)
9176 PendingLoads.push_back(Elt: LD.getValue(R: 1));
9177 setValue(V: &VPIntrin, NewN: LD);
9178}
9179
9180void SelectionDAGBuilder::visitVPStridedStore(
9181 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9182 SDLoc DL = getCurSDLoc();
9183 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
9184 EVT VT = OpValues[0].getValueType();
9185 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9186 if (!Alignment)
9187 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9188 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9189 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9190 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9191 MachineMemOperand::Flags MMOFlags =
9192 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9193 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9194 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9195 BaseAlignment: *Alignment, Metadata: AAInfo);
9196
9197 SDValue ST = DAG.getStridedStoreVP(
9198 Chain: getMemoryRoot(), DL, Val: OpValues[0], Ptr: OpValues[1],
9199 Offset: DAG.getPOISON(VT: OpValues[1].getValueType()), Stride: OpValues[2], Mask: OpValues[3],
9200 EVL: OpValues[4], MemVT: VT, MMO, AM: ISD::UNINDEXED, /*IsTruncating*/ false,
9201 /*IsCompressing*/ false);
9202
9203 DAG.setRoot(ST);
9204 setValue(V: &VPIntrin, NewN: ST);
9205}
9206
9207void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
9208 const VPIntrinsic &VPIntrin) {
9209 SDLoc DL = getCurSDLoc();
9210 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
9211
9212 auto IID = VPIntrin.getIntrinsicID();
9213
9214 SmallVector<EVT, 4> ValueVTs;
9215 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9216 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: VPIntrin.getType(), ValueVTs);
9217 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
9218
9219 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IntrinsicID: IID);
9220
9221 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
9222 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
9223 "Unexpected target EVL type");
9224
9225 // Request operands.
9226 SmallVector<SDValue, 7> OpValues;
9227 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
9228 auto Op = getValue(V: VPIntrin.getArgOperand(i: I));
9229 if (I == EVLParamPos)
9230 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EVLParamVT, Operand: Op);
9231 OpValues.push_back(Elt: Op);
9232 }
9233
9234 switch (Opcode) {
9235 default: {
9236 SDNodeFlags SDFlags;
9237 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &VPIntrin))
9238 SDFlags.copyFMF(FPMO: *FPMO);
9239 SDValue Result = DAG.getNode(Opcode, DL, VTList: VTs, Ops: OpValues, Flags: SDFlags);
9240 setValue(V: &VPIntrin, NewN: Result);
9241 break;
9242 }
9243 case ISD::VP_LOAD:
9244 visitVPLoad(VPIntrin, VT: ValueVTs[0], OpValues);
9245 break;
9246 case ISD::VP_LOAD_FF:
9247 visitVPLoadFF(VPIntrin, VT: ValueVTs[0], EVLVT: ValueVTs[1], OpValues);
9248 break;
9249 case ISD::VP_GATHER:
9250 visitVPGather(VPIntrin, VT: ValueVTs[0], OpValues);
9251 break;
9252 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9253 visitVPStridedLoad(VPIntrin, VT: ValueVTs[0], OpValues);
9254 break;
9255 case ISD::VP_STORE:
9256 visitVPStore(VPIntrin, OpValues);
9257 break;
9258 case ISD::VP_SCATTER:
9259 visitVPScatter(VPIntrin, OpValues);
9260 break;
9261 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9262 visitVPStridedStore(VPIntrin, OpValues);
9263 break;
9264 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
9265 case ISD::VP_CTTZ_ELTS: {
9266 SDValue Result =
9267 DAG.getNode(Opcode, DL, VTList: VTs, Ops: {OpValues[0], OpValues[2], OpValues[3]});
9268 setValue(V: &VPIntrin, NewN: Result);
9269 break;
9270 }
9271 }
9272}
9273
9274SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
9275 const BasicBlock *EHPadBB,
9276 MCSymbol *&BeginLabel) {
9277 MachineFunction &MF = DAG.getMachineFunction();
9278
9279 // Skip emitting EH_LABEL on targets whose exception tables don't reference
9280 // them (32-bit x86 SEH, Wasm).
9281 if (!MF.getContext().getAsmInfo().usesPerInvokeEHLabels(
9282 EH: FuncInfo.ExceptionModel)) {
9283 BeginLabel = nullptr;
9284 return Chain;
9285 }
9286
9287 // Insert a label before the invoke call to mark the try range. This can be
9288 // used to detect deletion of the invoke via the MachineModuleInfo.
9289 BeginLabel = MF.getContext().createTempSymbol();
9290
9291 // For SjLj, keep track of which landing pads go with which invokes
9292 // so as to maintain the ordering of pads in the LSDA.
9293 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
9294 if (CallSiteIndex) {
9295 MF.setCallSiteBeginLabel(BeginLabel, Site: CallSiteIndex);
9296 LPadToCallSiteMap[FuncInfo.getMBB(BB: EHPadBB)].push_back(Elt: CallSiteIndex);
9297
9298 // Now that the call site is handled, stop tracking it.
9299 FuncInfo.setCurrentCallSite(0);
9300 }
9301
9302 return DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: BeginLabel);
9303}
9304
9305SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
9306 const BasicBlock *EHPadBB,
9307 MCSymbol *BeginLabel) {
9308 // No labels were emitted.
9309 if (!BeginLabel)
9310 return Chain;
9311
9312 MachineFunction &MF = DAG.getMachineFunction();
9313
9314 // Insert a label at the end of the invoke call to mark the try range. This
9315 // can be used to detect deletion of the invoke via the MachineModuleInfo.
9316 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
9317 Chain = DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: EndLabel);
9318
9319 // Inform MachineModuleInfo of range.
9320 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
9321 // There is a platform (e.g. wasm) that uses funclet style IR but does not
9322 // actually use outlined funclets and their LSDA info style.
9323 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
9324 assert(II && "II should've been set");
9325 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
9326 EHInfo->addIPToStateRange(II, InvokeBegin: BeginLabel, InvokeEnd: EndLabel);
9327 } else if (!isScopedEHPersonality(Pers)) {
9328 assert(EHPadBB);
9329 MF.addInvoke(LandingPad: FuncInfo.getMBB(BB: EHPadBB), BeginLabel, EndLabel);
9330 }
9331
9332 return Chain;
9333}
9334
9335std::pair<SDValue, SDValue>
9336SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
9337 const BasicBlock *EHPadBB) {
9338 MCSymbol *BeginLabel = nullptr;
9339
9340 if (EHPadBB) {
9341 // Both PendingLoads and PendingExports must be flushed here;
9342 // this call might not return.
9343 (void)getRoot();
9344 DAG.setRoot(lowerStartEH(Chain: getControlRoot(), EHPadBB, BeginLabel));
9345 CLI.setChain(getRoot());
9346 }
9347
9348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9349 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
9350
9351 assert((CLI.IsTailCall || Result.second.getNode()) &&
9352 "Non-null chain expected with non-tail call!");
9353 assert((Result.second.getNode() || !Result.first.getNode()) &&
9354 "Null value expected with tail call!");
9355
9356 if (!Result.second.getNode()) {
9357 // As a special case, a null chain means that a tail call has been emitted
9358 // and the DAG root is already updated.
9359 HasTailCall = true;
9360
9361 // Since there's no actual continuation from this block, nothing can be
9362 // relying on us setting vregs for them.
9363 PendingExports.clear();
9364 } else {
9365 DAG.setRoot(Result.second);
9366 }
9367
9368 if (EHPadBB) {
9369 DAG.setRoot(lowerEndEH(Chain: getRoot(), II: cast_or_null<InvokeInst>(Val: CLI.CB), EHPadBB,
9370 BeginLabel));
9371 Result.second = getRoot();
9372 }
9373
9374 return Result;
9375}
9376
9377bool SelectionDAGBuilder::canTailCall(const CallBase &CB) const {
9378 bool isMustTailCall = CB.isMustTailCall();
9379
9380 // Avoid emitting tail calls in functions with the disable-tail-calls
9381 // attribute.
9382 const Function *Caller = CB.getParent()->getParent();
9383 if (!isMustTailCall &&
9384 Caller->getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
9385 return false;
9386
9387 // We can't tail call inside a function with a swifterror argument. Lowering
9388 // does not support this yet. It would have to move into the swifterror
9389 // register before the call.
9390 if (DAG.hasSwiftErrorArg())
9391 return false;
9392
9393 // Check if target-independent constraints permit a tail call here.
9394 // Target-dependent constraints are checked within TLI->LowerCallTo.
9395 return isInTailCallPosition(Call: CB, TM: DAG.getTarget());
9396}
9397
9398void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
9399 bool isTailCall, bool isMustTailCall,
9400 const BasicBlock *EHPadBB,
9401 const TargetLowering::PtrAuthInfo *PAI) {
9402 auto &DL = DAG.getDataLayout();
9403 FunctionType *FTy = CB.getFunctionType();
9404 Type *RetTy = CB.getType();
9405
9406 TargetLowering::ArgListTy Args;
9407 Args.reserve(n: CB.arg_size());
9408
9409 const Value *SwiftErrorVal = nullptr;
9410 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9411
9412 if (isTailCall)
9413 isTailCall = canTailCall(CB);
9414
9415 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
9416 const Value *V = *I;
9417
9418 // Skip empty types
9419 if (V->getType()->isEmptyTy())
9420 continue;
9421
9422 SDValue ArgNode = getValue(V);
9423 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
9424 Entry.setAttributes(Call: &CB, ArgIdx: I - CB.arg_begin());
9425
9426 // Use swifterror virtual register as input to the call.
9427 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
9428 SwiftErrorVal = V;
9429 // We find the virtual register for the actual swifterror argument.
9430 // Instead of using the Value, we use the virtual register instead.
9431 Entry.Node =
9432 DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
9433 VT: EVT(TLI.getPointerTy(DL)));
9434 }
9435
9436 Args.push_back(x: Entry);
9437
9438 // If we have an explicit sret argument that is an Instruction, (i.e., it
9439 // might point to function-local memory), we can't meaningfully tail-call.
9440 if (Entry.IsSRet && isa<Instruction>(Val: V))
9441 isTailCall = false;
9442 }
9443
9444 // If call site has a cfguardtarget operand bundle, create and add an
9445 // additional ArgListEntry.
9446 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_cfguardtarget)) {
9447 Value *V = Bundle->Inputs[0];
9448 TargetLowering::ArgListEntry Entry(V, getValue(V));
9449 Entry.IsCFGuardTarget = true;
9450 Args.push_back(x: Entry);
9451 }
9452
9453 // Disable tail calls if there is an swifterror argument. Targets have not
9454 // been updated to support tail calls.
9455 if (TLI.supportSwiftError() && SwiftErrorVal)
9456 isTailCall = false;
9457
9458 ConstantInt *CFIType = nullptr;
9459 if (CB.isIndirectCall()) {
9460 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi)) {
9461 if (!TLI.supportKCFIBundles())
9462 report_fatal_error(
9463 reason: "Target doesn't support calls with kcfi operand bundles.");
9464 CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
9465 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
9466 }
9467 }
9468
9469 SDValue ConvControlToken;
9470 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
9471 auto *Token = Bundle->Inputs[0].get();
9472 ConvControlToken = getValue(V: Token);
9473 }
9474
9475 GlobalValue *DeactivationSymbol = nullptr;
9476 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
9477 DeactivationSymbol = cast<GlobalValue>(Val: Bundle->Inputs[0].get());
9478 }
9479
9480 TargetLowering::CallLoweringInfo CLI(DAG);
9481 CLI.setDebugLoc(getCurSDLoc())
9482 .setChain(getRoot())
9483 .setCallee(ResultType: RetTy, FTy, Target: Callee, ArgsList: std::move(Args), Call: CB)
9484 .setTailCall(isTailCall)
9485 .setConvergent(CB.isConvergent())
9486 .setIsPreallocated(
9487 CB.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
9488 .setCFIType(CFIType)
9489 .setConvergenceControlToken(ConvControlToken)
9490 .setDeactivationSymbol(DeactivationSymbol);
9491
9492 // Set the pointer authentication info if we have it.
9493 if (PAI) {
9494 if (!TLI.supportPtrAuthBundles())
9495 report_fatal_error(
9496 reason: "This target doesn't support calls with ptrauth operand bundles.");
9497 CLI.setPtrAuth(*PAI);
9498 }
9499
9500 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9501
9502 if (Result.first.getNode()) {
9503 Result.first = lowerRangeToAssertZExt(DAG, I: CB, Op: Result.first);
9504 Result.first = lowerNoFPClassToAssertNoFPClass(DAG, I: CB, Op: Result.first);
9505 setValue(V: &CB, NewN: Result.first);
9506 }
9507
9508 // The last element of CLI.InVals has the SDValue for swifterror return.
9509 // Here we copy it to a virtual register and update SwiftErrorMap for
9510 // book-keeping.
9511 if (SwiftErrorVal && TLI.supportSwiftError()) {
9512 // Get the last element of InVals.
9513 SDValue Src = CLI.InVals.back();
9514 Register VReg =
9515 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
9516 SDValue CopyNode = CLI.DAG.getCopyToReg(Chain: Result.second, dl: CLI.DL, Reg: VReg, N: Src);
9517 DAG.setRoot(CopyNode);
9518 }
9519}
9520
9521static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
9522 SelectionDAGBuilder &Builder) {
9523 // Check to see if this load can be trivially constant folded, e.g. if the
9524 // input is from a string literal.
9525 if (const Constant *LoadInput = dyn_cast<Constant>(Val: PtrVal)) {
9526 // Cast pointer to the type we really want to load.
9527 Type *LoadTy =
9528 Type::getIntNTy(C&: PtrVal->getContext(), N: LoadVT.getScalarSizeInBits());
9529 if (LoadVT.isVector())
9530 LoadTy = FixedVectorType::get(ElementType: LoadTy, NumElts: LoadVT.getVectorNumElements());
9531 if (const Constant *LoadCst =
9532 ConstantFoldLoadFromConstPtr(C: const_cast<Constant *>(LoadInput),
9533 Ty: LoadTy, DL: Builder.DAG.getDataLayout()))
9534 return Builder.getValue(V: LoadCst);
9535 }
9536
9537 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9538 // still constant memory, the input chain can be the entry node.
9539 SDValue Root;
9540 bool ConstantMemory = false;
9541
9542 // Do not serialize (non-volatile) loads of constant memory with anything.
9543 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(P: PtrVal)) {
9544 Root = Builder.DAG.getEntryNode();
9545 ConstantMemory = true;
9546 } else {
9547 // Do not serialize non-volatile loads against each other.
9548 Root = Builder.DAG.getRoot();
9549 }
9550
9551 SDValue Ptr = Builder.getValue(V: PtrVal);
9552 SDValue LoadVal =
9553 Builder.DAG.getLoad(VT: LoadVT, dl: Builder.getCurSDLoc(), Chain: Root, Ptr,
9554 PtrInfo: MachinePointerInfo(PtrVal), Alignment: Align(1));
9555
9556 if (!ConstantMemory)
9557 Builder.PendingLoads.push_back(Elt: LoadVal.getValue(R: 1));
9558 return LoadVal;
9559}
9560
9561/// Record the value for an instruction that produces an integer result,
9562/// converting the type where necessary.
9563void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9564 SDValue Value,
9565 bool IsSigned) {
9566 EVT VT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9567 Ty: I.getType(), AllowUnknown: true);
9568 Value = DAG.getExtOrTrunc(IsSigned, Op: Value, DL: getCurSDLoc(), VT);
9569 setValue(V: &I, NewN: Value);
9570}
9571
9572/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9573/// true and lower it. Otherwise return false, and it will be lowered like a
9574/// normal call.
9575/// The caller already checked that \p I calls the appropriate LibFunc with a
9576/// correct prototype.
9577bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9578 const Value *LHS = I.getArgOperand(i: 0), *RHS = I.getArgOperand(i: 1);
9579 const Value *Size = I.getArgOperand(i: 2);
9580 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(Val: getValue(V: Size));
9581 if (CSize && CSize->getZExtValue() == 0) {
9582 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9583 Ty: I.getType(), AllowUnknown: true);
9584 setValue(V: &I, NewN: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: CallVT));
9585 return true;
9586 }
9587
9588 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9589 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9590 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: LHS), Op2: getValue(V: RHS),
9591 Op3: getValue(V: Size), CI: &I);
9592 if (Res.first.getNode()) {
9593 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9594 PendingLoads.push_back(Elt: Res.second);
9595 return true;
9596 }
9597
9598 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9599 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9600 if (!CSize || !isOnlyUsedInZeroEqualityComparison(CtxI: &I))
9601 return false;
9602
9603 // If the target has a fast compare for the given size, it will return a
9604 // preferred load type for that size. Require that the load VT is legal and
9605 // that the target supports unaligned loads of that type. Otherwise, return
9606 // INVALID.
9607 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9608 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9609 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9610 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9611 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9612 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9613 // TODO: Check alignment of src and dest ptrs.
9614 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9615 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9616 if (!TLI.isTypeLegal(VT: LVT) ||
9617 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: SrcAS) ||
9618 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: DstAS))
9619 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
9620 }
9621
9622 return LVT;
9623 };
9624
9625 // This turns into unaligned loads. We only do this if the target natively
9626 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9627 // we'll only produce a small number of byte loads.
9628 MVT LoadVT;
9629 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9630 switch (NumBitsToCompare) {
9631 default:
9632 return false;
9633 case 16:
9634 LoadVT = MVT::i16;
9635 break;
9636 case 32:
9637 LoadVT = MVT::i32;
9638 break;
9639 case 64:
9640 case 128:
9641 case 256:
9642 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9643 break;
9644 }
9645
9646 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9647 return false;
9648
9649 SDValue LoadL = getMemCmpLoad(PtrVal: LHS, LoadVT, Builder&: *this);
9650 SDValue LoadR = getMemCmpLoad(PtrVal: RHS, LoadVT, Builder&: *this);
9651
9652 // Bitcast to a wide integer type if the loads are vectors.
9653 if (LoadVT.isVector()) {
9654 EVT CmpVT = EVT::getIntegerVT(Context&: LHS->getContext(), BitWidth: LoadVT.getSizeInBits());
9655 LoadL = DAG.getBitcast(VT: CmpVT, V: LoadL);
9656 LoadR = DAG.getBitcast(VT: CmpVT, V: LoadR);
9657 }
9658
9659 SDValue Cmp = DAG.getSetCC(DL: getCurSDLoc(), VT: MVT::i1, LHS: LoadL, RHS: LoadR, Cond: ISD::SETNE);
9660 processIntegerCallValue(I, Value: Cmp, IsSigned: false);
9661 return true;
9662}
9663
9664/// See if we can lower a memchr call into an optimized form. If so, return
9665/// true and lower it. Otherwise return false, and it will be lowered like a
9666/// normal call.
9667/// The caller already checked that \p I calls the appropriate LibFunc with a
9668/// correct prototype.
9669bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9670 const Value *Src = I.getArgOperand(i: 0);
9671 const Value *Char = I.getArgOperand(i: 1);
9672 const Value *Length = I.getArgOperand(i: 2);
9673
9674 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9675 std::pair<SDValue, SDValue> Res =
9676 TSI.EmitTargetCodeForMemchr(DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(),
9677 Src: getValue(V: Src), Char: getValue(V: Char), Length: getValue(V: Length),
9678 SrcPtrInfo: MachinePointerInfo(Src));
9679 if (Res.first.getNode()) {
9680 setValue(V: &I, NewN: Res.first);
9681 PendingLoads.push_back(Elt: Res.second);
9682 return true;
9683 }
9684
9685 return false;
9686}
9687
9688/// See if we can lower a memccpy call into an optimized form. If so, return
9689/// true and lower it, otherwise return false and it will be lowered like a
9690/// normal call.
9691/// The caller already checked that \p I calls the appropriate LibFunc with a
9692/// correct prototype.
9693bool SelectionDAGBuilder::visitMemCCpyCall(const CallInst &I) {
9694 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9695 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemccpy(
9696 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Dst: getValue(V: I.getArgOperand(i: 0)),
9697 Src: getValue(V: I.getArgOperand(i: 1)), C: getValue(V: I.getArgOperand(i: 2)),
9698 Size: getValue(V: I.getArgOperand(i: 3)), CI: &I);
9699
9700 if (Res.first) {
9701 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9702 PendingLoads.push_back(Elt: Res.second);
9703 return true;
9704 }
9705 return false;
9706}
9707
9708/// See if we can lower a mempcpy call into an optimized form. If so, return
9709/// true and lower it. Otherwise return false, and it will be lowered like a
9710/// normal call.
9711/// The caller already checked that \p I calls the appropriate LibFunc with a
9712/// correct prototype.
9713bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9714 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
9715 SDValue Src = getValue(V: I.getArgOperand(i: 1));
9716 SDValue Size = getValue(V: I.getArgOperand(i: 2));
9717
9718 Align DstAlign = DAG.InferPtrAlign(Ptr: Dst).valueOrOne();
9719 Align SrcAlign = DAG.InferPtrAlign(Ptr: Src).valueOrOne();
9720
9721 SDLoc sdl = getCurSDLoc();
9722
9723 // In the mempcpy context we need to pass in a false value for isTailCall
9724 // because the return pointer needs to be adjusted by the size of
9725 // the copied memory.
9726 SDValue Root = getMemoryRoot();
9727 SDValue MC = DAG.getMemcpy(
9728 Chain: Root, dl: sdl, Dst, Src, Size, DstAlign, SrcAlign, isVol: false, AlwaysInline: false,
9729 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
9730 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)), AAInfo: I.getAAMetadata());
9731 assert(MC.getNode() != nullptr &&
9732 "** memcpy should not be lowered as TailCall in mempcpy context **");
9733 DAG.setRoot(MC);
9734
9735 // Check if Size needs to be truncated or extended.
9736 Size = DAG.getSExtOrTrunc(Op: Size, DL: sdl, VT: Dst.getValueType());
9737
9738 // Adjust return pointer to point just past the last dst byte.
9739 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Base: Dst, Offset: Size, DL: sdl);
9740 setValue(V: &I, NewN: DstPlusSize);
9741 return true;
9742}
9743
9744/// See if we can lower a strcpy call into an optimized form. If so, return
9745/// true and lower it, otherwise return false and it will be lowered like a
9746/// normal call.
9747/// The caller already checked that \p I calls the appropriate LibFunc with a
9748/// correct prototype.
9749bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9750 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9751
9752 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9753 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcpy(
9754 DAG, DL: getCurSDLoc(), Chain: getRoot(), Dest: getValue(V: Arg0), Src: getValue(V: Arg1),
9755 DestPtrInfo: MachinePointerInfo(Arg0), SrcPtrInfo: MachinePointerInfo(Arg1), isStpcpy, CI: &I);
9756 if (Res.first.getNode()) {
9757 setValue(V: &I, NewN: Res.first);
9758 DAG.setRoot(Res.second);
9759 return true;
9760 }
9761
9762 return false;
9763}
9764
9765/// See if we can lower a strcmp call into an optimized form. If so, return
9766/// true and lower it, otherwise return false and it will be lowered like a
9767/// normal call.
9768/// The caller already checked that \p I calls the appropriate LibFunc with a
9769/// correct prototype.
9770bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9771 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9772
9773 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9774 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcmp(
9775 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: Arg0), Op2: getValue(V: Arg1),
9776 Op1PtrInfo: MachinePointerInfo(Arg0), Op2PtrInfo: MachinePointerInfo(Arg1), CI: &I);
9777 if (Res.first.getNode()) {
9778 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9779 PendingLoads.push_back(Elt: Res.second);
9780 return true;
9781 }
9782
9783 return false;
9784}
9785
9786/// See if we can lower a strlen call into an optimized form. If so, return
9787/// true and lower it, otherwise return false and it will be lowered like a
9788/// normal call.
9789/// The caller already checked that \p I calls the appropriate LibFunc with a
9790/// correct prototype.
9791bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9792 const Value *Arg0 = I.getArgOperand(i: 0);
9793
9794 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9795 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(
9796 DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(), Src: getValue(V: Arg0), CI: &I);
9797 if (Res.first.getNode()) {
9798 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9799 PendingLoads.push_back(Elt: Res.second);
9800 return true;
9801 }
9802
9803 return false;
9804}
9805
9806/// See if we can lower a strnlen call into an optimized form. If so, return
9807/// true and lower it, otherwise return false and it will be lowered like a
9808/// normal call.
9809/// The caller already checked that \p I calls the appropriate LibFunc with a
9810/// correct prototype.
9811bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9812 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9813
9814 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9815 std::pair<SDValue, SDValue> Res =
9816 TSI.EmitTargetCodeForStrnlen(DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(),
9817 Src: getValue(V: Arg0), MaxLength: getValue(V: Arg1),
9818 SrcPtrInfo: MachinePointerInfo(Arg0));
9819 if (Res.first.getNode()) {
9820 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9821 PendingLoads.push_back(Elt: Res.second);
9822 return true;
9823 }
9824
9825 return false;
9826}
9827
9828/// See if we can lower a Strstr call into an optimized form. If so, return
9829/// true and lower it, otherwise return false and it will be lowered like a
9830/// normal call.
9831/// The caller already checked that \p I calls the appropriate LibFunc with a
9832/// correct prototype.
9833bool SelectionDAGBuilder::visitStrstrCall(const CallInst &I) {
9834 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9835 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9836 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrstr(
9837 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: Arg0), Op2: getValue(V: Arg1), CI: &I);
9838 if (Res.first) {
9839 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9840 PendingLoads.push_back(Elt: Res.second);
9841 return true;
9842 }
9843 return false;
9844}
9845
9846/// See if we can lower a unary floating-point operation into an SDNode with
9847/// the specified Opcode. If so, return true and lower it, otherwise return
9848/// false and it will be lowered like a normal call.
9849/// The caller already checked that \p I calls the appropriate LibFunc with a
9850/// correct prototype.
9851bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9852 unsigned Opcode) {
9853 // We already checked this call's prototype; verify it doesn't modify errno.
9854 // Do not perform optimizations for call sites that require strict
9855 // floating-point semantics.
9856 if (!I.onlyReadsMemory() || I.isStrictFP())
9857 return false;
9858
9859 SDNodeFlags Flags;
9860 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9861
9862 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
9863 setValue(V: &I,
9864 NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Tmp.getValueType(), Operand: Tmp, Flags));
9865 return true;
9866}
9867
9868/// See if we can lower a binary floating-point operation into an SDNode with
9869/// the specified Opcode. If so, return true and lower it. Otherwise return
9870/// false, and it will be lowered like a normal call.
9871/// The caller already checked that \p I calls the appropriate LibFunc with a
9872/// correct prototype.
9873bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9874 unsigned Opcode) {
9875 // We already checked this call's prototype; verify it doesn't modify errno.
9876 // Do not perform optimizations for call sites that require strict
9877 // floating-point semantics.
9878 if (!I.onlyReadsMemory() || I.isStrictFP())
9879 return false;
9880
9881 SDNodeFlags Flags;
9882 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9883
9884 SDValue Tmp0 = getValue(V: I.getArgOperand(i: 0));
9885 SDValue Tmp1 = getValue(V: I.getArgOperand(i: 1));
9886 EVT VT = Tmp0.getValueType();
9887 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT, N1: Tmp0, N2: Tmp1, Flags));
9888 return true;
9889}
9890
9891void SelectionDAGBuilder::visitCall(const CallInst &I) {
9892 // Handle inline assembly differently.
9893 if (I.isInlineAsm()) {
9894 visitInlineAsm(Call: I);
9895 return;
9896 }
9897
9898 diagnoseDontCall(CI: I);
9899
9900 if (Function *F = I.getCalledFunction()) {
9901 if (F->isDeclaration()) {
9902 // Is this an LLVM intrinsic?
9903 if (unsigned IID = F->getIntrinsicID()) {
9904 visitIntrinsicCall(I, Intrinsic: IID);
9905 return;
9906 }
9907 }
9908
9909 // Check for well-known libc/libm calls. If the function is internal, it
9910 // can't be a library call. Don't do the check if marked as nobuiltin for
9911 // some reason.
9912 // This code should not handle libcalls that are already canonicalized to
9913 // intrinsics by the middle-end.
9914 LibFunc Func = !I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName()
9915 ? LibInfo->getLibFunc(FDecl: *F)
9916 : NotLibFunc;
9917 if (LibInfo->hasOptimizedCodeGen(F: Func)) {
9918 switch (Func) {
9919 default: break;
9920 case LibFunc_bcmp:
9921 if (visitMemCmpBCmpCall(I))
9922 return;
9923 break;
9924 case LibFunc_sin:
9925 case LibFunc_sinf:
9926 case LibFunc_sinl:
9927 if (visitUnaryFloatCall(I, Opcode: ISD::FSIN))
9928 return;
9929 break;
9930 case LibFunc_cos:
9931 case LibFunc_cosf:
9932 case LibFunc_cosl:
9933 if (visitUnaryFloatCall(I, Opcode: ISD::FCOS))
9934 return;
9935 break;
9936 case LibFunc_tan:
9937 case LibFunc_tanf:
9938 case LibFunc_tanl:
9939 if (visitUnaryFloatCall(I, Opcode: ISD::FTAN))
9940 return;
9941 break;
9942 case LibFunc_asin:
9943 case LibFunc_asinf:
9944 case LibFunc_asinl:
9945 if (visitUnaryFloatCall(I, Opcode: ISD::FASIN))
9946 return;
9947 break;
9948 case LibFunc_acos:
9949 case LibFunc_acosf:
9950 case LibFunc_acosl:
9951 if (visitUnaryFloatCall(I, Opcode: ISD::FACOS))
9952 return;
9953 break;
9954 case LibFunc_atan:
9955 case LibFunc_atanf:
9956 case LibFunc_atanl:
9957 if (visitUnaryFloatCall(I, Opcode: ISD::FATAN))
9958 return;
9959 break;
9960 case LibFunc_atan2:
9961 case LibFunc_atan2f:
9962 case LibFunc_atan2l:
9963 if (visitBinaryFloatCall(I, Opcode: ISD::FATAN2))
9964 return;
9965 break;
9966 case LibFunc_sinh:
9967 case LibFunc_sinhf:
9968 case LibFunc_sinhl:
9969 if (visitUnaryFloatCall(I, Opcode: ISD::FSINH))
9970 return;
9971 break;
9972 case LibFunc_cosh:
9973 case LibFunc_coshf:
9974 case LibFunc_coshl:
9975 if (visitUnaryFloatCall(I, Opcode: ISD::FCOSH))
9976 return;
9977 break;
9978 case LibFunc_tanh:
9979 case LibFunc_tanhf:
9980 case LibFunc_tanhl:
9981 if (visitUnaryFloatCall(I, Opcode: ISD::FTANH))
9982 return;
9983 break;
9984 case LibFunc_sqrt:
9985 case LibFunc_sqrtf:
9986 case LibFunc_sqrtl:
9987 case LibFunc_sqrt_finite:
9988 case LibFunc_sqrtf_finite:
9989 case LibFunc_sqrtl_finite:
9990 if (visitUnaryFloatCall(I, Opcode: ISD::FSQRT))
9991 return;
9992 break;
9993 case LibFunc_log2:
9994 case LibFunc_log2f:
9995 case LibFunc_log2l:
9996 if (visitUnaryFloatCall(I, Opcode: ISD::FLOG2))
9997 return;
9998 break;
9999 case LibFunc_exp2:
10000 case LibFunc_exp2f:
10001 case LibFunc_exp2l:
10002 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP2))
10003 return;
10004 break;
10005 case LibFunc_exp10:
10006 case LibFunc_exp10f:
10007 case LibFunc_exp10l:
10008 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP10))
10009 return;
10010 break;
10011 case LibFunc_strstr:
10012 if (visitStrstrCall(I))
10013 return;
10014 break;
10015 case LibFunc_memcmp:
10016 if (visitMemCmpBCmpCall(I))
10017 return;
10018 break;
10019 case LibFunc_memccpy:
10020 if (visitMemCCpyCall(I))
10021 return;
10022 break;
10023 case LibFunc_mempcpy:
10024 if (visitMemPCpyCall(I))
10025 return;
10026 break;
10027 case LibFunc_memchr:
10028 if (visitMemChrCall(I))
10029 return;
10030 break;
10031 case LibFunc_strcpy:
10032 if (visitStrCpyCall(I, isStpcpy: false))
10033 return;
10034 break;
10035 case LibFunc_stpcpy:
10036 if (visitStrCpyCall(I, isStpcpy: true))
10037 return;
10038 break;
10039 case LibFunc_strcmp:
10040 if (visitStrCmpCall(I))
10041 return;
10042 break;
10043 case LibFunc_strlen:
10044 if (visitStrLenCall(I))
10045 return;
10046 break;
10047 case LibFunc_strnlen:
10048 if (visitStrNLenCall(I))
10049 return;
10050 break;
10051 }
10052 }
10053 }
10054
10055 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
10056 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), /*EHPadBB=*/nullptr);
10057 return;
10058 }
10059
10060 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
10061 // have to do anything here to lower funclet bundles.
10062 // CFGuardTarget bundles are lowered in LowerCallTo.
10063 failForInvalidBundles(
10064 I, Name: "calls",
10065 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
10066 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
10067 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
10068 LLVMContext::OB_convergencectrl, LLVMContext::OB_deactivation_symbol});
10069
10070 SDValue Callee = getValue(V: I.getCalledOperand());
10071
10072 if (I.hasDeoptState())
10073 LowerCallSiteWithDeoptBundle(Call: &I, Callee, EHPadBB: nullptr);
10074 else
10075 // Check if we can potentially perform a tail call. More detailed checking
10076 // is be done within LowerCallTo, after more information about the call is
10077 // known.
10078 LowerCallTo(CB: I, Callee, isTailCall: I.isTailCall(), isMustTailCall: I.isMustTailCall());
10079}
10080
10081void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(
10082 const CallBase &CB, const BasicBlock *EHPadBB) {
10083 auto PAB = CB.getOperandBundle(Name: "ptrauth");
10084 const Value *CalleeV = CB.getCalledOperand();
10085
10086 // Gather the call ptrauth data from the operand bundle:
10087 // [ i32 <key>, i64 <discriminator> ]
10088 const auto *Key = cast<ConstantInt>(Val: PAB->Inputs[0]);
10089 const Value *Discriminator = PAB->Inputs[1];
10090
10091 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
10092 assert(Discriminator->getType()->isIntegerTy(64) &&
10093 "Invalid ptrauth discriminator");
10094
10095 // Look through ptrauth constants to find the raw callee.
10096 // Do a direct unauthenticated call if we found it and everything matches.
10097 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CalleeV))
10098 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
10099 DL: DAG.getDataLayout()))
10100 return LowerCallTo(CB, Callee: getValue(V: CalleeCPA->getPointer()), isTailCall: CB.isTailCall(),
10101 isMustTailCall: CB.isMustTailCall(), EHPadBB);
10102
10103 // Functions should never be ptrauth-called directly.
10104 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
10105
10106 // Otherwise, do an authenticated indirect call.
10107 TargetLowering::PtrAuthInfo PAI = {.Key: Key->getZExtValue(),
10108 .Discriminator: getValue(V: Discriminator)};
10109
10110 LowerCallTo(CB, Callee: getValue(V: CalleeV), isTailCall: CB.isTailCall(), isMustTailCall: CB.isMustTailCall(),
10111 EHPadBB, PAI: &PAI);
10112}
10113
10114namespace {
10115
10116/// AsmOperandInfo - This contains information for each constraint that we are
10117/// lowering.
10118class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
10119public:
10120 /// CallOperand - If this is the result output operand or a clobber
10121 /// this is null, otherwise it is the incoming operand to the CallInst.
10122 /// This gets modified as the asm is processed.
10123 SDValue CallOperand;
10124
10125 /// AssignedRegs - If this is a register or register class operand, this
10126 /// contains the set of register corresponding to the operand.
10127 RegsForValue AssignedRegs;
10128
10129 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
10130 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
10131 }
10132
10133 /// Whether or not this operand accesses memory
10134 bool hasMemory(const TargetLowering &TLI) const {
10135 // Indirect operand accesses access memory.
10136 if (isIndirect)
10137 return true;
10138
10139 for (const auto &Code : Codes)
10140 if (TLI.getConstraintType(Constraint: Code) == TargetLowering::C_Memory)
10141 return true;
10142
10143 return false;
10144 }
10145};
10146
10147
10148} // end anonymous namespace
10149
10150/// Make sure that the output operand \p OpInfo and its corresponding input
10151/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
10152/// out).
10153static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
10154 SDISelAsmOperandInfo &MatchingOpInfo,
10155 SelectionDAG &DAG) {
10156 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
10157 return;
10158
10159 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
10160 const auto &TLI = DAG.getTargetLoweringInfo();
10161
10162 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
10163 TLI.getRegForInlineAsmConstraint(TRI, Constraint: OpInfo.ConstraintCode,
10164 VT: OpInfo.ConstraintVT);
10165 std::pair<unsigned, const TargetRegisterClass *> InputRC =
10166 TLI.getRegForInlineAsmConstraint(TRI, Constraint: MatchingOpInfo.ConstraintCode,
10167 VT: MatchingOpInfo.ConstraintVT);
10168 const bool OutOpIsIntOrFP =
10169 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
10170 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
10171 MatchingOpInfo.ConstraintVT.isFloatingPoint();
10172 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
10173 // FIXME: error out in a more elegant fashion
10174 report_fatal_error(reason: "Unsupported asm: input constraint"
10175 " with a matching output constraint of"
10176 " incompatible type!");
10177 }
10178 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
10179}
10180
10181/// Get a direct memory input to behave well as an indirect operand.
10182/// This may introduce stores, hence the need for a \p Chain.
10183/// \return The (possibly updated) chain.
10184static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
10185 SDISelAsmOperandInfo &OpInfo,
10186 SelectionDAG &DAG) {
10187 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10188
10189 // If we don't have an indirect input, put it in the constpool if we can,
10190 // otherwise spill it to a stack slot.
10191 // TODO: This isn't quite right. We need to handle these according to
10192 // the addressing mode that the constraint wants. Also, this may take
10193 // an additional register for the computation and we don't want that
10194 // either.
10195
10196 // If the operand is a float, integer, or vector constant, spill to a
10197 // constant pool entry to get its address.
10198 const Value *OpVal = OpInfo.CallOperandVal;
10199 if (isa<ConstantFP>(Val: OpVal) || isa<ConstantInt>(Val: OpVal) ||
10200 isa<ConstantVector>(Val: OpVal) || isa<ConstantDataVector>(Val: OpVal)) {
10201 OpInfo.CallOperand = DAG.getConstantPool(
10202 C: cast<Constant>(Val: OpVal), VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
10203 return Chain;
10204 }
10205
10206 // Otherwise, create a stack slot and emit a store to it before the asm.
10207 Type *Ty = OpVal->getType();
10208 auto &DL = DAG.getDataLayout();
10209 TypeSize TySize = DL.getTypeAllocSize(Ty);
10210 MachineFunction &MF = DAG.getMachineFunction();
10211 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
10212 int StackID = 0;
10213 if (TySize.isScalable())
10214 StackID = TFI->getStackIDForScalableVectors();
10215 int SSFI = MF.getFrameInfo().CreateStackObject(Size: TySize.getKnownMinValue(),
10216 Alignment: DL.getPrefTypeAlign(Ty), isSpillSlot: false,
10217 Alloca: nullptr, ID: StackID);
10218 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: TLI.getFrameIndexTy(DL));
10219 Chain = DAG.getTruncStore(Chain, dl: Location, Val: OpInfo.CallOperand, Ptr: StackSlot,
10220 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: SSFI),
10221 SVT: TLI.getMemValueType(DL, Ty));
10222 OpInfo.CallOperand = StackSlot;
10223
10224 return Chain;
10225}
10226
10227/// GetRegistersForValue - Assign registers (virtual or physical) for the
10228/// specified operand. We prefer to assign virtual registers, to allow the
10229/// register allocator to handle the assignment process. However, if the asm
10230/// uses features that we can't model on machineinstrs, we have SDISel do the
10231/// allocation. This produces generally horrible, but correct, code.
10232///
10233/// OpInfo describes the operand
10234/// RefOpInfo describes the matching operand if any, the operand otherwise
10235static std::optional<unsigned>
10236getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
10237 SDISelAsmOperandInfo &OpInfo,
10238 SDISelAsmOperandInfo &RefOpInfo) {
10239 LLVMContext &Context = *DAG.getContext();
10240 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10241
10242 MachineFunction &MF = DAG.getMachineFunction();
10243 SmallVector<Register, 4> Regs;
10244 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10245
10246 // No work to do for memory/address operands.
10247 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10248 OpInfo.ConstraintType == TargetLowering::C_Address)
10249 return std::nullopt;
10250
10251 // If this is a constraint for a single physreg, or a constraint for a
10252 // register class, find it.
10253 unsigned AssignedReg;
10254 const TargetRegisterClass *RC;
10255 std::tie(args&: AssignedReg, args&: RC) = TLI.getRegForInlineAsmConstraint(
10256 TRI: &TRI, Constraint: RefOpInfo.ConstraintCode, VT: RefOpInfo.ConstraintVT);
10257 // RC is unset only on failure. Return immediately.
10258 if (!RC)
10259 return std::nullopt;
10260
10261 // Get the actual register value type. This is important, because the user
10262 // may have asked for (e.g.) the AX register in i32 type. We need to
10263 // remember that AX is actually i16 to get the right extension.
10264 const MVT RegVT = *TRI.legalclasstypes_begin(RC: *RC);
10265
10266 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10267 // If this is an FP operand in an integer register (or visa versa), or more
10268 // generally if the operand value disagrees with the register class we plan
10269 // to stick it in, fix the operand type.
10270 //
10271 // If this is an input value, the bitcast to the new type is done now.
10272 // Bitcast for output value is done at the end of visitInlineAsm().
10273 if ((OpInfo.Type == InlineAsm::isOutput ||
10274 OpInfo.Type == InlineAsm::isInput) &&
10275 !TRI.isTypeLegalForClass(RC: *RC, T: OpInfo.ConstraintVT)) {
10276 // Try to convert to the first EVT that the reg class contains. If the
10277 // types are identical size, use a bitcast to convert (e.g. two differing
10278 // vector types). Note: output bitcast is done at the end of
10279 // visitInlineAsm().
10280 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10281 // Exclude indirect inputs while they are unsupported because the code
10282 // to perform the load is missing and thus OpInfo.CallOperand still
10283 // refers to the input address rather than the pointed-to value.
10284 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
10285 OpInfo.CallOperand =
10286 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: RegVT, Operand: OpInfo.CallOperand);
10287 OpInfo.ConstraintVT = RegVT;
10288 // If the operand is an FP value and we want it in integer registers,
10289 // use the corresponding integer type. This turns an f64 value into
10290 // i64, which can be passed with two i32 values on a 32-bit machine.
10291 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10292 MVT VT = MVT::getIntegerVT(BitWidth: OpInfo.ConstraintVT.getSizeInBits());
10293 // No integer type of that size to bitcast to (e.g. f80).
10294 if (!VT.isValid())
10295 return std::nullopt;
10296 if (OpInfo.Type == InlineAsm::isInput)
10297 OpInfo.CallOperand =
10298 DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: OpInfo.CallOperand);
10299 OpInfo.ConstraintVT = VT;
10300 }
10301 }
10302 }
10303
10304 // No need to allocate a matching input constraint since the constraint it's
10305 // matching to has already been allocated.
10306 if (OpInfo.isMatchingInputConstraint())
10307 return std::nullopt;
10308
10309 EVT ValueVT = OpInfo.ConstraintVT;
10310 if (OpInfo.ConstraintVT == MVT::Other)
10311 ValueVT = RegVT;
10312
10313 // Initialize NumRegs.
10314 unsigned NumRegs = 1;
10315 if (OpInfo.ConstraintVT != MVT::Other)
10316 NumRegs = TLI.getNumRegisters(Context, VT: OpInfo.ConstraintVT, RegisterVT: RegVT);
10317
10318 // If this is a constraint for a specific physical register, like {r17},
10319 // assign it now.
10320
10321 // If this associated to a specific register, initialize iterator to correct
10322 // place. If virtual, make sure we have enough registers
10323
10324 // Initialize iterator if necessary
10325 TargetRegisterClass::iterator I = RC->begin();
10326 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10327
10328 // Do not check for single registers.
10329 if (AssignedReg) {
10330 I = std::find(first: I, last: RC->end(), val: AssignedReg);
10331 if (I == RC->end()) {
10332 // RC does not contain the selected register, which indicates a
10333 // mismatch between the register and the required type/bitwidth.
10334 return {AssignedReg};
10335 }
10336 }
10337
10338 for (; NumRegs; --NumRegs, ++I) {
10339 assert(I != RC->end() && "Ran out of registers to allocate!");
10340 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RegClass: RC);
10341 Regs.push_back(Elt: R);
10342 }
10343
10344 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
10345 return std::nullopt;
10346}
10347
10348static unsigned
10349findMatchingInlineAsmOperand(unsigned OperandNo,
10350 const std::vector<SDValue> &AsmNodeOperands) {
10351 // Scan until we find the definition we already emitted of this operand.
10352 unsigned CurOp = InlineAsm::Op_FirstOperand;
10353 for (; OperandNo; --OperandNo) {
10354 // Advance to the next operand.
10355 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10356 const InlineAsm::Flag F(OpFlag);
10357 assert(
10358 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
10359 "Skipped past definitions?");
10360 CurOp += F.getNumOperandRegisters() + 1;
10361 }
10362 return CurOp;
10363}
10364
10365namespace {
10366
10367class ExtraFlags {
10368 unsigned Flags = 0;
10369
10370public:
10371 explicit ExtraFlags(const CallBase &Call) {
10372 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10373 if (IA->hasSideEffects())
10374 Flags |= InlineAsm::Extra_HasSideEffects;
10375 if (IA->isAlignStack())
10376 Flags |= InlineAsm::Extra_IsAlignStack;
10377 if (IA->canThrow())
10378 Flags |= InlineAsm::Extra_MayUnwind;
10379 if (Call.isConvergent())
10380 Flags |= InlineAsm::Extra_IsConvergent;
10381 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
10382 }
10383
10384 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
10385 // Ideally, we would only check against memory constraints. However, the
10386 // meaning of an Other constraint can be target-specific and we can't easily
10387 // reason about it. Therefore, be conservative and set MayLoad/MayStore
10388 // for Other constraints as well.
10389 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10390 OpInfo.ConstraintType == TargetLowering::C_Other) {
10391 if (OpInfo.Type == InlineAsm::isInput)
10392 Flags |= InlineAsm::Extra_MayLoad;
10393 else if (OpInfo.Type == InlineAsm::isOutput)
10394 Flags |= InlineAsm::Extra_MayStore;
10395 else if (OpInfo.Type == InlineAsm::isClobber)
10396 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
10397 }
10398 }
10399
10400 unsigned get() const { return Flags; }
10401};
10402
10403} // end anonymous namespace
10404
10405static bool isFunction(SDValue Op) {
10406 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
10407 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
10408 auto Fn = dyn_cast_or_null<Function>(Val: GA->getGlobal());
10409
10410 // In normal "call dllimport func" instruction (non-inlineasm) it force
10411 // indirect access by specifing call opcode. And usually specially print
10412 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
10413 // not do in this way now. (In fact, this is similar with "Data Access"
10414 // action). So here we ignore dllimport function.
10415 if (Fn && !Fn->hasDLLImportStorageClass())
10416 return true;
10417 }
10418 }
10419 return false;
10420}
10421
10422namespace {
10423
10424struct ConstraintDecisionInfo {
10425 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
10426 std::vector<SDValue> AsmNodeOperands;
10427 SDValue Glue, Chain;
10428 bool HasSideEffect = false;
10429 MCSymbol *BeginLabel = nullptr;
10430
10431 SmallVector<char> Buffer;
10432 raw_svector_ostream ErrorMsg;
10433
10434 ConstraintDecisionInfo() : ErrorMsg(Buffer) {}
10435};
10436
10437} // end anonymous namespace
10438
10439/// Construct operand info objects.
10440static bool
10441constructOperandInfo(ConstraintDecisionInfo &Info,
10442 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10443 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10444 ExtraFlags &ExtraInfo) {
10445 for (auto &T : TargetConstraints) {
10446 Info.ConstraintOperands.push_back(Elt: SDISelAsmOperandInfo(T));
10447 SDISelAsmOperandInfo &OpInfo = Info.ConstraintOperands.back();
10448
10449 if (OpInfo.CallOperandVal)
10450 OpInfo.CallOperand = Builder.getValue(V: OpInfo.CallOperandVal);
10451
10452 if (!Info.HasSideEffect)
10453 Info.HasSideEffect = OpInfo.hasMemory(TLI);
10454
10455 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
10456 // FIXME: Could we compute this on OpInfo rather than T?
10457
10458 // Compute the constraint code and ConstraintType to use.
10459 TLI.ComputeConstraintToUse(OpInfo&: T, Op: SDValue());
10460
10461 if (T.ConstraintType == TargetLowering::C_Immediate && OpInfo.CallOperand &&
10462 !isa<ConstantSDNode>(Val: OpInfo.CallOperand)) {
10463 // We've delayed emitting a diagnostic like the "n" constraint because
10464 // inlining could cause an integer showing up.
10465 Info.ErrorMsg << "constraint '" << T.ConstraintCode
10466 << "' expects an integer constant expression";
10467 return true;
10468 }
10469
10470 ExtraInfo.update(OpInfo: T);
10471 }
10472
10473 return false;
10474}
10475
10476/// Compute which constraint option to use for each operand.
10477static void
10478computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
10479 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10480 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10481 const TargetMachine &TM, SelectionDAG &DAG) {
10482 const auto *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10483 SmallVector<StringRef, 4> AsmStrs;
10484 IA->collectAsmStrs(AsmStrs);
10485
10486 int OpNo = -1;
10487 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10488 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10489 OpNo++;
10490
10491 // If this is an output operand with a matching input operand, look up the
10492 // matching input. If their types mismatch, e.g. one is an integer, the
10493 // other is floating point, or their sizes are different, flag it as an
10494 // error.
10495 if (OpInfo.hasMatchingInput()) {
10496 SDISelAsmOperandInfo &Input =
10497 Info.ConstraintOperands[OpInfo.MatchingInput];
10498 patchMatchingInput(OpInfo, MatchingOpInfo&: Input, DAG);
10499 }
10500
10501 // Compute the constraint code and ConstraintType to use.
10502 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
10503
10504 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10505 OpInfo.Type == InlineAsm::isClobber) ||
10506 OpInfo.ConstraintType == TargetLowering::C_Address)
10507 continue;
10508
10509 // In Linux PIC model, there are 4 cases about value/label addressing:
10510 //
10511 // 1: Function call or Label jmp inside the module.
10512 // 2: Data access (such as global variable, static variable) inside module.
10513 // 3: Function call or Label jmp outside the module.
10514 // 4: Data access (such as global variable) outside the module.
10515 //
10516 // Due to current llvm inline asm architecture designed to not "recognize"
10517 // the asm code, there are quite troubles for us to treat mem addressing
10518 // differently for same value/adress used in different instructions.
10519 // For example, in pic model, call a func may in plt way or direclty
10520 // pc-related, but lea/mov a function adress may use got.
10521 //
10522 // Here we try to "recognize" function call for the case 1 and case 3 in
10523 // inline asm. And try to adjust the constraint for them.
10524 //
10525 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10526 // label, so here we don't handle jmp function label now, but we need to
10527 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10528 if (OpInfo.isIndirect && isFunction(Op: OpInfo.CallOperand) &&
10529 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10530 TM.getCodeModel() != CodeModel::Large) {
10531 OpInfo.isIndirect = false;
10532 OpInfo.ConstraintType = TargetLowering::C_Address;
10533 }
10534
10535 // If this is a memory input, and if the operand is not indirect, do what we
10536 // need to provide an address for the memory input.
10537 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10538 !OpInfo.isIndirect) {
10539 assert((OpInfo.isMultipleAlternative ||
10540 (OpInfo.Type == InlineAsm::isInput)) &&
10541 "Can only indirectify direct input operands!");
10542
10543 // Memory operands really want the address of the value.
10544 Info.Chain = getAddressForMemoryInput(Chain: Info.Chain, Location: Builder.getCurSDLoc(),
10545 OpInfo, DAG);
10546
10547 // There is no longer a Value* corresponding to this operand.
10548 OpInfo.CallOperandVal = nullptr;
10549
10550 // It is now an indirect operand.
10551 OpInfo.isIndirect = true;
10552 }
10553 }
10554}
10555
10556/// Prepare DAG-level operands. As part of this, assign virtual and physical
10557/// registers for inputs and output.
10558static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info,
10559 const CallBase &Call,
10560 SelectionDAGBuilder &Builder,
10561 const TargetLowering &TLI,
10562 SelectionDAG &DAG) {
10563 SDLoc DL = Builder.getCurSDLoc();
10564 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10565 // Assign Registers.
10566 SDISelAsmOperandInfo &RefOpInfo =
10567 OpInfo.isMatchingInputConstraint()
10568 ? Info.ConstraintOperands[OpInfo.getMatchedOperand()]
10569 : OpInfo;
10570 const auto RegError = getRegistersForValue(DAG, DL, OpInfo, RefOpInfo);
10571 if (RegError) {
10572 const MachineFunction &MF = DAG.getMachineFunction();
10573 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10574 const char *RegName = TRI.getName(RegNo: *RegError);
10575 Info.ErrorMsg << "register '" << RegName << "' allocated for constraint '"
10576 << OpInfo.ConstraintCode
10577 << "' does not match required type";
10578 return true;
10579 }
10580
10581 auto DetectWriteToReservedRegister = [&]() {
10582 const MachineFunction &MF = DAG.getMachineFunction();
10583 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10584
10585 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10586 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, PhysReg: Reg)) {
10587 Info.ErrorMsg << "write to reserved register '"
10588 << TRI.getRegAsmName(Reg) << "'";
10589 return true;
10590 }
10591 }
10592
10593 return false;
10594 };
10595 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10596 (OpInfo.Type == InlineAsm::isInput &&
10597 !OpInfo.isMatchingInputConstraint())) &&
10598 "Only address as input operand is allowed.");
10599
10600 switch (OpInfo.Type) {
10601 case InlineAsm::isOutput:
10602 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10603 const InlineAsm::ConstraintCode ConstraintID =
10604 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10605 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10606 "Failed to convert memory constraint code to constraint id.");
10607
10608 // Add information to the INLINEASM node to know about this output.
10609 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
10610 OpFlags.setMemConstraint(ConstraintID);
10611 Info.AsmNodeOperands.push_back(
10612 x: DAG.getTargetConstant(Val: OpFlags, DL, VT: MVT::i32));
10613 Info.AsmNodeOperands.push_back(x: OpInfo.CallOperand);
10614 } else {
10615 // Otherwise, this outputs to a register (directly for C_Register /
10616 // C_RegisterClass, and a target-defined fashion for
10617 // C_Immediate/C_Other). Find a register that we can use.
10618 if (OpInfo.AssignedRegs.Regs.empty()) {
10619 Info.ErrorMsg << "could not allocate output register for "
10620 << "constraint '" << OpInfo.ConstraintCode << "'";
10621 return true;
10622 }
10623
10624 if (DetectWriteToReservedRegister())
10625 return true;
10626
10627 // Add information to the INLINEASM node to know that this register is
10628 // set.
10629 OpInfo.AssignedRegs.AddInlineAsmOperands(
10630 Code: OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10631 : InlineAsm::Kind::RegDef,
10632 HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10633 }
10634 break;
10635
10636 case InlineAsm::isInput:
10637 case InlineAsm::isLabel: {
10638 SDValue InOperandVal = OpInfo.CallOperand;
10639
10640 if (OpInfo.isMatchingInputConstraint()) {
10641 // If this is required to match an output register we have already set,
10642 // just use its register.
10643 auto CurOp = findMatchingInlineAsmOperand(OperandNo: OpInfo.getMatchedOperand(),
10644 AsmNodeOperands: Info.AsmNodeOperands);
10645 InlineAsm::Flag Flag(Info.AsmNodeOperands[CurOp]->getAsZExtVal());
10646 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10647 if (OpInfo.isIndirect) {
10648 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10649 Info.ErrorMsg << "inline asm not supported yet: cannot handle "
10650 << "tied indirect register inputs";
10651 return true;
10652 }
10653
10654 SmallVector<Register, 4> Regs;
10655 MachineFunction &MF = DAG.getMachineFunction();
10656 MachineRegisterInfo &MRI = MF.getRegInfo();
10657 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10658 auto *R = cast<RegisterSDNode>(Val&: Info.AsmNodeOperands[CurOp + 1]);
10659 Register TiedReg = R->getReg();
10660 MVT RegVT = R->getSimpleValueType(ResNo: 0);
10661 const TargetRegisterClass *RC;
10662 if (TiedReg.isVirtual()) {
10663 RC = MRI.getRegClass(Reg: TiedReg);
10664 } else if (RegVT == MVT::Untyped) {
10665 RC = TRI.getMinimalPhysRegClass(Reg: TiedReg);
10666 } else {
10667 RC = TLI.getRegClassFor(VT: RegVT);
10668 if (!RC->contains(Reg: TiedReg)) {
10669 const TargetRegisterClass *PhysRC =
10670 TRI.getMinimalPhysRegClass(Reg: TiedReg);
10671 if (PhysRC->isAllocatable())
10672 RC = PhysRC;
10673 }
10674 }
10675
10676 for (unsigned I = 0, E = Flag.getNumOperandRegisters(); I != E; ++I)
10677 Regs.push_back(Elt: MRI.createVirtualRegister(RegClass: RC));
10678
10679 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10680
10681 // Use the produced MatchedRegs object to
10682 MatchedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl: DL, Chain&: Info.Chain,
10683 Glue: &Info.Glue, V: &Call);
10684 MatchedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::RegUse, HasMatching: true,
10685 MatchingIdx: OpInfo.getMatchedOperand(), dl: DL, DAG,
10686 Ops&: Info.AsmNodeOperands);
10687 break;
10688 }
10689
10690 assert(Flag.isMemKind() && "Unknown matching constraint!");
10691 assert(Flag.getNumOperandRegisters() == 1 &&
10692 "Unexpected number of operands");
10693
10694 // Add information to the INLINEASM node to know about this input.
10695 // See InlineAsm.h isUseOperandTiedToDef.
10696 Flag.clearMemConstraint();
10697 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10698 Info.AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10699 Val: Flag, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10700 Info.AsmNodeOperands.push_back(x: Info.AsmNodeOperands[CurOp + 1]);
10701 break;
10702 }
10703
10704 // Treat indirect 'X' constraint as memory.
10705 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10706 OpInfo.isIndirect)
10707 OpInfo.ConstraintType = TargetLowering::C_Memory;
10708
10709 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10710 OpInfo.ConstraintType == TargetLowering::C_Other) {
10711 std::vector<SDValue> Ops;
10712 TLI.LowerAsmOperandForConstraint(Op: InOperandVal, Constraint: OpInfo.ConstraintCode,
10713 Ops, DAG);
10714 if (Ops.empty()) {
10715 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10716 if (isa<ConstantSDNode>(Val: InOperandVal)) {
10717 Info.ErrorMsg << "value out of range for constraint '"
10718 << OpInfo.ConstraintCode << "'";
10719 return true;
10720 }
10721
10722 Info.ErrorMsg << "invalid operand for inline asm constraint '"
10723 << OpInfo.ConstraintCode << "'";
10724 return true;
10725 }
10726
10727 // Add information to the INLINEASM node to know about this input.
10728 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10729 Info.AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10730 Val: ResOpType, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10731 llvm::append_range(C&: Info.AsmNodeOperands, R&: Ops);
10732 break;
10733 }
10734
10735 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10736 assert((OpInfo.isIndirect ||
10737 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10738 "Operand must be indirect to be a mem!");
10739
10740 assert(InOperandVal.getValueType() ==
10741 TLI.getPointerTy(
10742 DAG.getDataLayout(),
10743 OpInfo.CallOperandVal
10744 ? cast<PointerType>(OpInfo.CallOperandVal->getType())
10745 ->getAddressSpace()
10746 : 0) &&
10747 "Memory operands expect pointer values");
10748
10749 const InlineAsm::ConstraintCode ConstraintID =
10750 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10751 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10752 "Failed to convert memory constraint code to constraint id.");
10753
10754 // Add information to the INLINEASM node to know about this input.
10755 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10756 ResOpType.setMemConstraint(ConstraintID);
10757 Info.AsmNodeOperands.push_back(
10758 x: DAG.getTargetConstant(Val: ResOpType, DL, VT: MVT::i32));
10759 Info.AsmNodeOperands.push_back(x: InOperandVal);
10760 break;
10761 }
10762
10763 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10764 const InlineAsm::ConstraintCode ConstraintID =
10765 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10766 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10767 "Failed to convert memory constraint code to constraint id.");
10768
10769 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10770
10771 SDValue AsmOp = InOperandVal;
10772 if (TLI.supportsInlineAsmFuncOperands() && isFunction(Op: InOperandVal)) {
10773 auto *GA = cast<GlobalAddressSDNode>(Val&: InOperandVal);
10774 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10775 AsmOp = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL,
10776 VT: InOperandVal.getValueType(),
10777 offset: GA->getOffset());
10778 }
10779
10780 // Add information to the INLINEASM node to know about this input.
10781 ResOpType.setMemConstraint(ConstraintID);
10782
10783 Info.AsmNodeOperands.push_back(
10784 x: DAG.getTargetConstant(Val: ResOpType, DL, VT: MVT::i32));
10785 Info.AsmNodeOperands.push_back(x: AsmOp);
10786 break;
10787 }
10788
10789 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10790 OpInfo.ConstraintType != TargetLowering::C_Register) {
10791 Info.ErrorMsg << "unknown asm constraint '" << OpInfo.ConstraintCode
10792 << "'";
10793 return true;
10794 }
10795
10796 // TODO: Support this.
10797 if (OpInfo.isIndirect) {
10798 Info.ErrorMsg << "cannot handle indirect register inputs yet for "
10799 << "constraint '" << OpInfo.ConstraintCode << "'";
10800 return true;
10801 }
10802
10803 // Copy the input into the appropriate registers.
10804 if (OpInfo.AssignedRegs.Regs.empty()) {
10805 Info.ErrorMsg << "could not allocate input reg for constraint '"
10806 << OpInfo.ConstraintCode << "'";
10807 return true;
10808 }
10809
10810 if (DetectWriteToReservedRegister())
10811 return true;
10812
10813 OpInfo.AssignedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl: DL, Chain&: Info.Chain,
10814 Glue: &Info.Glue, V: &Call);
10815 OpInfo.AssignedRegs.AddInlineAsmOperands(
10816 Code: InlineAsm::Kind::RegUse, HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10817 break;
10818 }
10819
10820 case InlineAsm::isClobber:
10821 // Add the clobbered value to the operand list, so that the register
10822 // allocator is aware that the physreg got clobbered.
10823 if (!OpInfo.AssignedRegs.Regs.empty())
10824 OpInfo.AssignedRegs.AddInlineAsmOperands(
10825 Code: InlineAsm::Kind::Clobber, HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10826 break;
10827 }
10828 }
10829
10830 return false;
10831}
10832
10833/// DetermineConstraints - Find the constraints to use for inline asm operands.
10834static bool
10835determineConstraints(ConstraintDecisionInfo &Info,
10836 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10837 const CallBase &Call, SelectionDAGBuilder &Builder,
10838 const TargetLowering &TLI, const TargetMachine &TM,
10839 SelectionDAG &DAG, const BasicBlock *EHPadBB) {
10840 const auto *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10841 ExtraFlags ExtraInfo(Call);
10842
10843 // First pass: Construct operand info objects.
10844 Info.HasSideEffect = IA->hasSideEffects();
10845 if (constructOperandInfo(Info, TargetConstraints, Builder, TLI, ExtraInfo))
10846 return true;
10847
10848 // We won't need to flush pending loads if this asm doesn't touch
10849 // memory and is nonvolatile.
10850 Info.Chain = Info.HasSideEffect ? Builder.getRoot() : DAG.getRoot();
10851
10852 bool IsCallBr = isa<CallBrInst>(Val: Call);
10853 bool EmitEHLabels = isa<InvokeInst>(Val: Call);
10854 if (IsCallBr || EmitEHLabels)
10855 // If this is a callbr or invoke we need to flush pending exports since
10856 // inlineasm_br and invoke are terminators.
10857 // We need to do this before nodes are glued to the inlineasm_br node.
10858 Info.Chain = Builder.getControlRoot();
10859
10860 if (EmitEHLabels)
10861 Info.Chain = Builder.lowerStartEH(Chain: Info.Chain, EHPadBB, BeginLabel&: Info.BeginLabel);
10862
10863 // Second pass: Compute which constraint option to use.
10864 computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM, DAG);
10865
10866 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10867 Info.AsmNodeOperands.push_back(x: SDValue()); // reserve space for input chain
10868 Info.AsmNodeOperands.push_back(x: DAG.getTargetExternalSymbol(
10869 Sym: IA->getAsmString().data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout())));
10870
10871 // If we have a !srcloc metadata node associated with it, we want to attach
10872 // this to the ultimately generated inline asm machineinstr. To do this, we
10873 // pass in the third operand as this (potentially null) inline asm MDNode.
10874 const MDNode *SrcLoc = Call.getMetadata(Kind: "srcloc");
10875 Info.AsmNodeOperands.push_back(x: DAG.getMDNode(MD: SrcLoc));
10876
10877 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10878 // bits as operand 3.
10879 Info.AsmNodeOperands.push_back(
10880 x: DAG.getTargetConstant(Val: ExtraInfo.get(), DL: Builder.getCurSDLoc(),
10881 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10882
10883 // Third pass: Prepare DAG-level operands
10884 return prepareDAGLevelOperands(Info, Call, Builder, TLI, DAG);
10885}
10886
10887/// visitInlineAsm - Handle a call to an InlineAsm object.
10888void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
10889 const BasicBlock *EHPadBB) {
10890 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10891 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
10892 DL: DAG.getDataLayout(), TRI: DAG.getSubtarget().getRegisterInfo(), Call);
10893
10894 assert((!isa<InvokeInst>(Call) || EHPadBB) &&
10895 "InvokeInst must have an EHPadBB");
10896
10897 ConstraintDecisionInfo Info;
10898 if (determineConstraints(Info, TargetConstraints, Call, Builder&: *this, TLI, TM, DAG,
10899 EHPadBB))
10900 return emitInlineAsmError(Call, Message: Info.ErrorMsg.str());
10901
10902 SDValue Glue = Info.Glue;
10903 SDValue Chain = Info.Chain;
10904
10905 // Finish up input operands. Set the input chain and add the flag last.
10906 Info.AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10907 if (Glue.getNode())
10908 Info.AsmNodeOperands.push_back(x: Glue);
10909
10910 bool IsCallBr = isa<CallBrInst>(Val: Call);
10911 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10912 Chain =
10913 DAG.getNode(Opcode: ISDOpc, DL: getCurSDLoc(), VTList: DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue),
10914 Ops: Info.AsmNodeOperands);
10915 Glue = Chain.getValue(R: 1);
10916
10917 // Do additional work to generate outputs.
10918
10919 SmallVector<EVT, 1> ResultVTs;
10920 SmallVector<SDValue, 1> ResultValues;
10921 SmallVector<SDValue, 8> OutChains;
10922
10923 llvm::Type *CallResultType = Call.getType();
10924 ArrayRef<Type *> ResultTypes;
10925 if (StructType *StructResult = dyn_cast<StructType>(Val: CallResultType))
10926 ResultTypes = StructResult->elements();
10927 else if (!CallResultType->isVoidTy())
10928 ResultTypes = ArrayRef(CallResultType);
10929
10930 auto CurResultType = ResultTypes.begin();
10931 auto handleRegAssign = [&](SDValue V) {
10932 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10933 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10934 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: *CurResultType);
10935 ++CurResultType;
10936 // If the type of the inline asm call site return value is different but has
10937 // same size as the type of the asm output bitcast it. One example of this
10938 // is for vectors with different width / number of elements. This can
10939 // happen for register classes that can contain multiple different value
10940 // types. The preg or vreg allocated may not have the same VT as was
10941 // expected.
10942 //
10943 // This can also happen for a return value that disagrees with the register
10944 // class it is put in, eg. a double in a general-purpose register on a
10945 // 32-bit machine.
10946 if (ResultVT != V.getValueType() &&
10947 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10948 V = DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10949 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10950 V.getValueType().isInteger()) {
10951 // If a result value was tied to an input value, the computed result
10952 // may have a wider width than the expected result. Extract the
10953 // relevant portion.
10954 V = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10955 }
10956 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10957 ResultVTs.push_back(Elt: ResultVT);
10958 ResultValues.push_back(Elt: V);
10959 };
10960
10961 // Deal with output operands.
10962 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10963 if (OpInfo.Type == InlineAsm::isOutput) {
10964 SDValue Val;
10965 // Skip trivial output operands.
10966 if (OpInfo.AssignedRegs.Regs.empty())
10967 continue;
10968
10969 switch (OpInfo.ConstraintType) {
10970 case TargetLowering::C_Register:
10971 case TargetLowering::C_RegisterClass:
10972 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(),
10973 Chain, Glue: &Glue, V: &Call);
10974 break;
10975 case TargetLowering::C_Immediate:
10976 case TargetLowering::C_Other:
10977 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, DL: getCurSDLoc(),
10978 OpInfo, DAG);
10979 break;
10980 case TargetLowering::C_Memory:
10981 break; // Already handled.
10982 case TargetLowering::C_Address:
10983 break; // Silence warning.
10984 case TargetLowering::C_Unknown:
10985 assert(false && "Unexpected unknown constraint");
10986 }
10987
10988 // Indirect output manifest as stores. Record output chains.
10989 if (OpInfo.isIndirect) {
10990 const Value *Ptr = OpInfo.CallOperandVal;
10991 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10992 SDValue Store = DAG.getStore(Chain, dl: getCurSDLoc(), Val, Ptr: getValue(V: Ptr),
10993 PtrInfo: MachinePointerInfo(Ptr));
10994 OutChains.push_back(Elt: Store);
10995 } else {
10996 // generate CopyFromRegs to associated registers.
10997 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10998 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10999 for (const SDValue &V : Val->op_values())
11000 handleRegAssign(V);
11001 } else
11002 handleRegAssign(Val);
11003 }
11004 }
11005 }
11006
11007 // Set results.
11008 if (!ResultValues.empty()) {
11009 assert(CurResultType == ResultTypes.end() &&
11010 "Mismatch in number of ResultTypes");
11011 assert(ResultValues.size() == ResultTypes.size() &&
11012 "Mismatch in number of output operands in asm result");
11013
11014 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
11015 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
11016 setValue(V: &Call, NewN: V);
11017 }
11018
11019 // Collect store chains.
11020 if (!OutChains.empty())
11021 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: getCurSDLoc(), VT: MVT::Other, Ops: OutChains);
11022
11023 if (const auto *II = dyn_cast<InvokeInst>(Val: &Call))
11024 Chain = lowerEndEH(Chain, II, EHPadBB, BeginLabel: Info.BeginLabel);
11025
11026 // Only Update Root if inline assembly has a memory effect.
11027 if (ResultValues.empty() || Info.HasSideEffect || !OutChains.empty() ||
11028 IsCallBr || isa<InvokeInst>(Val: Call))
11029 DAG.setRoot(Chain);
11030}
11031
11032void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
11033 const Twine &Message) {
11034 LLVMContext &Ctx = *DAG.getContext();
11035 Ctx.diagnose(DI: DiagnosticInfoInlineAsm(Call, Message));
11036
11037 // Make sure we leave the DAG in a valid state
11038 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11039 SmallVector<EVT, 1> ValueVTs;
11040 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: Call.getType(), ValueVTs);
11041
11042 if (ValueVTs.empty())
11043 return;
11044
11045 SmallVector<SDValue, 1> Ops;
11046 for (const EVT &VT : ValueVTs)
11047 Ops.push_back(Elt: DAG.getUNDEF(VT));
11048
11049 setValue(V: &Call, NewN: DAG.getMergeValues(Ops, dl: getCurSDLoc()));
11050}
11051
11052void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
11053 DAG.setRoot(DAG.getNode(Opcode: ISD::VASTART, DL: getCurSDLoc(),
11054 VT: MVT::Other, N1: getRoot(),
11055 N2: getValue(V: I.getArgOperand(i: 0)),
11056 N3: DAG.getSrcValue(v: I.getArgOperand(i: 0))));
11057}
11058
11059void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
11060 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11061 const DataLayout &DL = DAG.getDataLayout();
11062 SDValue V = DAG.getVAArg(
11063 VT: TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType()), dl: getCurSDLoc(),
11064 Chain: getRoot(), Ptr: getValue(V: I.getOperand(i_nocapture: 0)), SV: DAG.getSrcValue(v: I.getOperand(i_nocapture: 0)),
11065 Align: DL.getABITypeAlign(Ty: I.getType()).value());
11066 DAG.setRoot(V.getValue(R: 1));
11067
11068 if (I.getType()->isPointerTy())
11069 V = DAG.getPtrExtOrTrunc(
11070 Op: V, DL: getCurSDLoc(), VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()));
11071 setValue(V: &I, NewN: V);
11072}
11073
11074void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
11075 DAG.setRoot(DAG.getNode(Opcode: ISD::VAEND, DL: getCurSDLoc(),
11076 VT: MVT::Other, N1: getRoot(),
11077 N2: getValue(V: I.getArgOperand(i: 0)),
11078 N3: DAG.getSrcValue(v: I.getArgOperand(i: 0))));
11079}
11080
11081void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
11082 DAG.setRoot(DAG.getNode(Opcode: ISD::VACOPY, DL: getCurSDLoc(),
11083 VT: MVT::Other, N1: getRoot(),
11084 N2: getValue(V: I.getArgOperand(i: 0)),
11085 N3: getValue(V: I.getArgOperand(i: 1)),
11086 N4: DAG.getSrcValue(v: I.getArgOperand(i: 0)),
11087 N5: DAG.getSrcValue(v: I.getArgOperand(i: 1))));
11088}
11089
11090SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
11091 const Instruction &I,
11092 SDValue Op) {
11093 std::optional<ConstantRange> CR = getRange(I);
11094
11095 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
11096 return Op;
11097
11098 APInt Hi = CR->getUnsignedMax();
11099 unsigned Bits = std::max(a: Hi.getActiveBits(),
11100 b: static_cast<unsigned>(IntegerType::MIN_INT_BITS));
11101
11102 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
11103
11104 SDLoc SL = getCurSDLoc();
11105
11106 SDValue ZExt = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: Op.getValueType(), N1: Op,
11107 N2: DAG.getValueType(SmallVT));
11108 unsigned NumVals = Op.getNode()->getNumValues();
11109 if (NumVals == 1)
11110 return ZExt;
11111
11112 SmallVector<SDValue, 4> Ops;
11113
11114 Ops.push_back(Elt: ZExt);
11115 for (unsigned I = 1; I != NumVals; ++I)
11116 Ops.push_back(Elt: Op.getValue(R: I));
11117
11118 return DAG.getMergeValues(Ops, dl: SL);
11119}
11120
11121SDValue SelectionDAGBuilder::lowerNoFPClassToAssertNoFPClass(
11122 SelectionDAG &DAG, const Instruction &I, SDValue Op) {
11123 FPClassTest Classes = getNoFPClass(I);
11124 if (Classes == fcNone)
11125 return Op;
11126
11127 SDLoc SL = getCurSDLoc();
11128 SDValue TestConst = DAG.getTargetConstant(Val: Classes, DL: SDLoc(), VT: MVT::i32);
11129
11130 if (Op.getOpcode() != ISD::MERGE_VALUES) {
11131 return DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: SL, VT: Op.getValueType(), N1: Op,
11132 N2: TestConst);
11133 }
11134
11135 SmallVector<SDValue, 8> Ops(Op.getNumOperands());
11136 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
11137 SDValue MergeOp = Op.getOperand(i: I);
11138 Ops[I] = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: SL, VT: MergeOp.getValueType(),
11139 N1: MergeOp, N2: TestConst);
11140 }
11141
11142 return DAG.getMergeValues(Ops, dl: SL);
11143}
11144
11145/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
11146/// the call being lowered.
11147///
11148/// This is a helper for lowering intrinsics that follow a target calling
11149/// convention or require stack pointer adjustment. Only a subset of the
11150/// intrinsic's operands need to participate in the calling convention.
11151void SelectionDAGBuilder::populateCallLoweringInfo(
11152 TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
11153 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
11154 AttributeSet RetAttrs, bool IsPatchPoint) {
11155 TargetLowering::ArgListTy Args;
11156 Args.reserve(n: NumArgs);
11157
11158 // Populate the argument list.
11159 // Attributes for args start at offset 1, after the return attribute.
11160 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
11161 ArgI != ArgE; ++ArgI) {
11162 const Value *V = Call->getOperand(i_nocapture: ArgI);
11163
11164 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
11165
11166 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
11167 Entry.setAttributes(Call, ArgIdx: ArgI);
11168 Args.push_back(x: Entry);
11169 }
11170
11171 CLI.setDebugLoc(getCurSDLoc())
11172 .setChain(getRoot())
11173 .setCallee(CC: Call->getCallingConv(), ResultType: ReturnTy, Target: Callee, ArgsList: std::move(Args),
11174 ResultAttrs: RetAttrs)
11175 .setDiscardResult(Call->use_empty())
11176 .setIsPatchPoint(IsPatchPoint)
11177 .setIsPreallocated(
11178 Call->countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0);
11179}
11180
11181/// Add a stack map intrinsic call's live variable operands to a stackmap
11182/// or patchpoint target node's operand list.
11183///
11184/// Constants are converted to TargetConstants purely as an optimization to
11185/// avoid constant materialization and register allocation.
11186///
11187/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
11188/// generate addess computation nodes, and so FinalizeISel can convert the
11189/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
11190/// address materialization and register allocation, but may also be required
11191/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
11192/// alloca in the entry block, then the runtime may assume that the alloca's
11193/// StackMap location can be read immediately after compilation and that the
11194/// location is valid at any point during execution (this is similar to the
11195/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
11196/// only available in a register, then the runtime would need to trap when
11197/// execution reaches the StackMap in order to read the alloca's location.
11198static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
11199 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
11200 SelectionDAGBuilder &Builder) {
11201 SelectionDAG &DAG = Builder.DAG;
11202 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
11203 SDValue Op = Builder.getValue(V: Call.getArgOperand(i: I));
11204
11205 // Things on the stack are pointer-typed, meaning that they are already
11206 // legal and can be emitted directly to target nodes.
11207 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op)) {
11208 Ops.push_back(Elt: DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: Op.getValueType()));
11209 } else {
11210 // Otherwise emit a target independent node to be legalised.
11211 Ops.push_back(Elt: Builder.getValue(V: Call.getArgOperand(i: I)));
11212 }
11213 }
11214}
11215
11216/// Lower llvm.experimental.stackmap.
11217void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
11218 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
11219 // [live variables...])
11220
11221 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
11222
11223 SDValue Chain, InGlue, Callee;
11224 SmallVector<SDValue, 32> Ops;
11225
11226 SDLoc DL = getCurSDLoc();
11227 Callee = getValue(V: CI.getCalledOperand());
11228
11229 // The stackmap intrinsic only records the live variables (the arguments
11230 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
11231 // intrinsic, this won't be lowered to a function call. This means we don't
11232 // have to worry about calling conventions and target specific lowering code.
11233 // Instead we perform the call lowering right here.
11234 //
11235 // chain, flag = CALLSEQ_START(chain, 0, 0)
11236 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
11237 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
11238 //
11239 Chain = DAG.getCALLSEQ_START(Chain: getRoot(), InSize: 0, OutSize: 0, DL);
11240 InGlue = Chain.getValue(R: 1);
11241
11242 // Add the STACKMAP operands, starting with DAG house-keeping.
11243 Ops.push_back(Elt: Chain);
11244 Ops.push_back(Elt: InGlue);
11245
11246 // Add the <id>, <numShadowBytes> operands.
11247 //
11248 // These do not require legalisation, and can be emitted directly to target
11249 // constant nodes.
11250 SDValue ID = getValue(V: CI.getArgOperand(i: 0));
11251 assert(ID.getValueType() == MVT::i64);
11252 SDValue IDConst =
11253 DAG.getTargetConstant(Val: ID->getAsZExtVal(), DL, VT: ID.getValueType());
11254 Ops.push_back(Elt: IDConst);
11255
11256 SDValue Shad = getValue(V: CI.getArgOperand(i: 1));
11257 assert(Shad.getValueType() == MVT::i32);
11258 SDValue ShadConst =
11259 DAG.getTargetConstant(Val: Shad->getAsZExtVal(), DL, VT: Shad.getValueType());
11260 Ops.push_back(Elt: ShadConst);
11261
11262 // Add the live variables.
11263 addStackMapLiveVars(Call: CI, StartIdx: 2, DL, Ops, Builder&: *this);
11264
11265 // Create the STACKMAP node.
11266 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
11267 Chain = DAG.getNode(Opcode: ISD::STACKMAP, DL, VTList: NodeTys, Ops);
11268 InGlue = Chain.getValue(R: 1);
11269
11270 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: InGlue, DL);
11271
11272 // Stackmaps don't generate values, so nothing goes into the NodeMap.
11273
11274 // Set the root to the target-lowered call chain.
11275 DAG.setRoot(Chain);
11276
11277 // Inform the Frame Information that we have a stackmap in this function.
11278 FuncInfo.MF->getFrameInfo().setHasStackMap();
11279}
11280
11281/// Lower llvm.experimental.patchpoint directly to its target opcode.
11282void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
11283 const BasicBlock *EHPadBB) {
11284 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
11285 // i32 <numBytes>,
11286 // i8* <target>,
11287 // i32 <numArgs>,
11288 // [Args...],
11289 // [live variables...])
11290
11291 CallingConv::ID CC = CB.getCallingConv();
11292 bool IsAnyRegCC = CC == CallingConv::AnyReg;
11293 bool HasDef = !CB.getType()->isVoidTy();
11294 SDLoc dl = getCurSDLoc();
11295 SDValue Callee = getValue(V: CB.getArgOperand(i: PatchPointOpers::TargetPos));
11296
11297 // Handle immediate and symbolic callees.
11298 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Val&: Callee))
11299 Callee = DAG.getIntPtrConstant(Val: ConstCallee->getZExtValue(), DL: dl,
11300 /*isTarget=*/true);
11301 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11302 Callee = DAG.getTargetGlobalAddress(GV: SymbolicCallee->getGlobal(),
11303 DL: SDLoc(SymbolicCallee),
11304 VT: SymbolicCallee->getValueType(ResNo: 0));
11305
11306 // Get the real number of arguments participating in the call <numArgs>
11307 SDValue NArgVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NArgPos));
11308 unsigned NumArgs = NArgVal->getAsZExtVal();
11309
11310 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
11311 // Intrinsics include all meta-operands up to but not including CC.
11312 unsigned NumMetaOpers = PatchPointOpers::CCPos;
11313 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
11314 "Not enough arguments provided to the patchpoint intrinsic");
11315
11316 // For AnyRegCC the arguments are lowered later on manually.
11317 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11318 Type *ReturnTy =
11319 IsAnyRegCC ? Type::getVoidTy(C&: *DAG.getContext()) : CB.getType();
11320
11321 TargetLowering::CallLoweringInfo CLI(DAG);
11322 populateCallLoweringInfo(CLI, Call: &CB, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee,
11323 ReturnTy, RetAttrs: CB.getAttributes().getRetAttrs(), IsPatchPoint: true);
11324 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
11325
11326 SDNode *CallEnd = Result.second.getNode();
11327 if (CallEnd->getOpcode() == ISD::EH_LABEL)
11328 CallEnd = CallEnd->getOperand(Num: 0).getNode();
11329 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
11330 CallEnd = CallEnd->getOperand(Num: 0).getNode();
11331
11332 /// Get a call instruction from the call sequence chain.
11333 /// Tail calls are not allowed.
11334 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
11335 "Expected a callseq node.");
11336 SDNode *Call = CallEnd->getOperand(Num: 0).getNode();
11337 bool HasGlue = Call->getGluedNode();
11338
11339 // Replace the target specific call node with the patchable intrinsic.
11340 SmallVector<SDValue, 8> Ops;
11341
11342 // Push the chain.
11343 Ops.push_back(Elt: *(Call->op_begin()));
11344
11345 // Optionally, push the glue (if any).
11346 if (HasGlue)
11347 Ops.push_back(Elt: *(Call->op_end() - 1));
11348
11349 // Push the register mask info.
11350 if (HasGlue)
11351 Ops.push_back(Elt: *(Call->op_end() - 2));
11352 else
11353 Ops.push_back(Elt: *(Call->op_end() - 1));
11354
11355 // Add the <id> and <numBytes> constants.
11356 SDValue IDVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::IDPos));
11357 Ops.push_back(Elt: DAG.getTargetConstant(Val: IDVal->getAsZExtVal(), DL: dl, VT: MVT::i64));
11358 SDValue NBytesVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NBytesPos));
11359 Ops.push_back(Elt: DAG.getTargetConstant(Val: NBytesVal->getAsZExtVal(), DL: dl, VT: MVT::i32));
11360
11361 // Add the callee.
11362 Ops.push_back(Elt: Callee);
11363
11364 // Adjust <numArgs> to account for any arguments that have been passed on the
11365 // stack instead.
11366 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
11367 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
11368 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11369 Ops.push_back(Elt: DAG.getTargetConstant(Val: NumCallRegArgs, DL: dl, VT: MVT::i32));
11370
11371 // Add the calling convention
11372 Ops.push_back(Elt: DAG.getTargetConstant(Val: (unsigned)CC, DL: dl, VT: MVT::i32));
11373
11374 // Add the arguments we omitted previously. The register allocator should
11375 // place these in any free register.
11376 if (IsAnyRegCC)
11377 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
11378 Ops.push_back(Elt: getValue(V: CB.getArgOperand(i)));
11379
11380 // Push the arguments from the call instruction.
11381 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
11382 Ops.append(in_start: Call->op_begin() + 2, in_end: e);
11383
11384 // Push live variables for the stack map.
11385 addStackMapLiveVars(Call: CB, StartIdx: NumMetaOpers + NumArgs, DL: dl, Ops, Builder&: *this);
11386
11387 SDVTList NodeTys;
11388 if (IsAnyRegCC && HasDef) {
11389 // Create the return types based on the intrinsic definition
11390 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11391 SmallVector<EVT, 3> ValueVTs;
11392 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: CB.getType(), ValueVTs);
11393 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
11394
11395 // There is always a chain and a glue type at the end
11396 ValueVTs.push_back(Elt: MVT::Other);
11397 ValueVTs.push_back(Elt: MVT::Glue);
11398 NodeTys = DAG.getVTList(VTs: ValueVTs);
11399 } else
11400 NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
11401
11402 // Replace the target specific call node with a PATCHPOINT node.
11403 SDValue PPV = DAG.getNode(Opcode: ISD::PATCHPOINT, DL: dl, VTList: NodeTys, Ops);
11404
11405 // Update the NodeMap.
11406 if (HasDef) {
11407 if (IsAnyRegCC)
11408 setValue(V: &CB, NewN: SDValue(PPV.getNode(), 0));
11409 else
11410 setValue(V: &CB, NewN: Result.first);
11411 }
11412
11413 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
11414 // call sequence. Furthermore the location of the chain and glue can change
11415 // when the AnyReg calling convention is used and the intrinsic returns a
11416 // value.
11417 if (IsAnyRegCC && HasDef) {
11418 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
11419 SDValue To[] = {PPV.getValue(R: 1), PPV.getValue(R: 2)};
11420 DAG.ReplaceAllUsesOfValuesWith(From, To, Num: 2);
11421 } else
11422 DAG.ReplaceAllUsesWith(From: Call, To: PPV.getNode());
11423 DAG.DeleteNode(N: Call);
11424
11425 // Inform the Frame Information that we have a patchpoint in this function.
11426 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11427}
11428
11429void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
11430 unsigned Intrinsic) {
11431 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11432 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
11433 SDValue Op2;
11434 if (I.arg_size() > 1)
11435 Op2 = getValue(V: I.getArgOperand(i: 1));
11436 SDLoc dl = getCurSDLoc();
11437 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
11438 SDValue Res;
11439 SDNodeFlags SDFlags;
11440 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
11441 SDFlags.copyFMF(FPMO: *FPMO);
11442
11443 switch (Intrinsic) {
11444 case Intrinsic::vector_reduce_fadd:
11445 if (SDFlags.hasAllowReassociation())
11446 Res = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Op1,
11447 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FADD, DL: dl, VT, Operand: Op2, Flags: SDFlags),
11448 Flags: SDFlags);
11449 else
11450 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FADD, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
11451 break;
11452 case Intrinsic::vector_reduce_fmul:
11453 if (SDFlags.hasAllowReassociation())
11454 Res = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Op1,
11455 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FMUL, DL: dl, VT, Operand: Op2, Flags: SDFlags),
11456 Flags: SDFlags);
11457 else
11458 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FMUL, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
11459 break;
11460 case Intrinsic::vector_reduce_add:
11461 Res = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT, Operand: Op1);
11462 break;
11463 case Intrinsic::vector_reduce_mul:
11464 Res = DAG.getNode(Opcode: ISD::VECREDUCE_MUL, DL: dl, VT, Operand: Op1);
11465 break;
11466 case Intrinsic::vector_reduce_and:
11467 Res = DAG.getNode(Opcode: ISD::VECREDUCE_AND, DL: dl, VT, Operand: Op1);
11468 break;
11469 case Intrinsic::vector_reduce_or:
11470 Res = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: dl, VT, Operand: Op1);
11471 break;
11472 case Intrinsic::vector_reduce_xor:
11473 Res = DAG.getNode(Opcode: ISD::VECREDUCE_XOR, DL: dl, VT, Operand: Op1);
11474 break;
11475 case Intrinsic::vector_reduce_smax:
11476 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMAX, DL: dl, VT, Operand: Op1);
11477 break;
11478 case Intrinsic::vector_reduce_smin:
11479 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMIN, DL: dl, VT, Operand: Op1);
11480 break;
11481 case Intrinsic::vector_reduce_umax:
11482 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL: dl, VT, Operand: Op1);
11483 break;
11484 case Intrinsic::vector_reduce_umin:
11485 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMIN, DL: dl, VT, Operand: Op1);
11486 break;
11487 case Intrinsic::vector_reduce_fmax:
11488 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAX, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11489 break;
11490 case Intrinsic::vector_reduce_fmin:
11491 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMIN, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11492 break;
11493 case Intrinsic::vector_reduce_fmaximum:
11494 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAXIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11495 break;
11496 case Intrinsic::vector_reduce_fminimum:
11497 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMINIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11498 break;
11499 case Intrinsic::vector_reduce_fmaximumnum:
11500 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAXIMUMNUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11501 break;
11502 case Intrinsic::vector_reduce_fminimumnum:
11503 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMINIMUMNUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11504 break;
11505 default:
11506 llvm_unreachable("Unhandled vector reduce intrinsic");
11507 }
11508 setValue(V: &I, NewN: Res);
11509}
11510
11511/// Returns an AttributeList representing the attributes applied to the return
11512/// value of the given call.
11513static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
11514 SmallVector<Attribute::AttrKind, 2> Attrs;
11515 if (CLI.RetSExt)
11516 Attrs.push_back(Elt: Attribute::SExt);
11517 if (CLI.RetZExt)
11518 Attrs.push_back(Elt: Attribute::ZExt);
11519 if (CLI.IsInReg)
11520 Attrs.push_back(Elt: Attribute::InReg);
11521
11522 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
11523 Kinds: Attrs);
11524}
11525
11526/// TargetLowering::LowerCallTo - This is the default LowerCallTo
11527/// implementation, which just calls LowerCall.
11528/// FIXME: When all targets are
11529/// migrated to using LowerCall, this hook should be integrated into SDISel.
11530std::pair<SDValue, SDValue>
11531TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
11532 LLVMContext &Context = CLI.RetTy->getContext();
11533
11534 // Handle the incoming return values from the call.
11535 CLI.Ins.clear();
11536 SmallVector<Type *, 4> RetOrigTys;
11537 SmallVector<TypeSize, 4> Offsets;
11538 auto &DL = CLI.DAG.getDataLayout();
11539 ComputeValueTypes(DL, Ty: CLI.OrigRetTy, Types&: RetOrigTys, Offsets: &Offsets);
11540
11541 SmallVector<EVT, 4> RetVTs;
11542 if (CLI.RetTy != CLI.OrigRetTy) {
11543 assert(RetOrigTys.size() == 1 &&
11544 "Only supported for non-aggregate returns");
11545 RetVTs.push_back(Elt: getValueType(DL, Ty: CLI.RetTy));
11546 } else {
11547 for (Type *Ty : RetOrigTys)
11548 RetVTs.push_back(Elt: getValueType(DL, Ty));
11549 }
11550
11551 if (CLI.IsPostTypeLegalization) {
11552 // If we are lowering a libcall after legalization, split the return type.
11553 SmallVector<Type *, 4> OldRetOrigTys;
11554 SmallVector<EVT, 4> OldRetVTs;
11555 SmallVector<TypeSize, 4> OldOffsets;
11556 RetOrigTys.swap(RHS&: OldRetOrigTys);
11557 RetVTs.swap(RHS&: OldRetVTs);
11558 Offsets.swap(RHS&: OldOffsets);
11559
11560 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
11561 EVT RetVT = OldRetVTs[i];
11562 uint64_t Offset = OldOffsets[i];
11563 MVT RegisterVT = getRegisterType(Context, VT: RetVT);
11564 unsigned NumRegs = getNumRegisters(Context, VT: RetVT);
11565 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
11566 RetOrigTys.append(NumInputs: NumRegs, Elt: OldRetOrigTys[i]);
11567 RetVTs.append(NumInputs: NumRegs, Elt: RegisterVT);
11568 for (unsigned j = 0; j != NumRegs; ++j)
11569 Offsets.push_back(Elt: TypeSize::getFixed(ExactSize: Offset + j * RegisterVTByteSZ));
11570 }
11571 }
11572
11573 SmallVector<ISD::OutputArg, 4> Outs;
11574 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI: *this, DL);
11575
11576 bool CanLowerReturn =
11577 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
11578 CLI.IsVarArg, Outs, Context, RetTy: CLI.RetTy);
11579
11580 SDValue DemoteStackSlot;
11581 int DemoteStackIdx = -100;
11582 if (!CanLowerReturn) {
11583 // FIXME: equivalent assert?
11584 // assert(!CS.hasInAllocaArgument() &&
11585 // "sret demotion is incompatible with inalloca");
11586 uint64_t TySize = DL.getTypeAllocSize(Ty: CLI.RetTy);
11587 Align Alignment = DL.getPrefTypeAlign(Ty: CLI.RetTy);
11588 MachineFunction &MF = CLI.DAG.getMachineFunction();
11589 DemoteStackIdx =
11590 MF.getFrameInfo().CreateStackObject(Size: TySize, Alignment, isSpillSlot: false);
11591 Type *StackSlotPtrType = PointerType::get(C&: Context, AddressSpace: DL.getAllocaAddrSpace());
11592
11593 DemoteStackSlot = CLI.DAG.getFrameIndex(FI: DemoteStackIdx, VT: getFrameIndexTy(DL));
11594 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11595 Entry.IsSRet = true;
11596 Entry.Alignment = Alignment;
11597 CLI.getArgs().insert(position: CLI.getArgs().begin(), x: Entry);
11598 CLI.NumFixedArgs += 1;
11599 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11600 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(C&: Context);
11601
11602 // sret demotion isn't compatible with tail-calls, since the sret argument
11603 // points into the callers stack frame.
11604 CLI.IsTailCall = false;
11605 } else {
11606 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11607 Ty: CLI.RetTy, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11608 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
11609 ISD::ArgFlagsTy Flags;
11610 if (NeedsRegBlock) {
11611 Flags.setInConsecutiveRegs();
11612 if (I == RetVTs.size() - 1)
11613 Flags.setInConsecutiveRegsLast();
11614 }
11615 EVT VT = RetVTs[I];
11616 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11617 unsigned NumRegs =
11618 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11619 for (unsigned i = 0; i != NumRegs; ++i) {
11620 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
11621 CLI.IsReturnValueUsed, ISD::InputArg::NoArgIndex, 0);
11622 if (CLI.RetTy->isPointerTy()) {
11623 Ret.Flags.setPointer();
11624 Ret.Flags.setPointerAddrSpace(
11625 cast<PointerType>(Val: CLI.RetTy)->getAddressSpace());
11626 }
11627 if (CLI.RetSExt)
11628 Ret.Flags.setSExt();
11629 if (CLI.RetZExt)
11630 Ret.Flags.setZExt();
11631 if (CLI.IsInReg)
11632 Ret.Flags.setInReg();
11633 CLI.Ins.push_back(Elt: Ret);
11634 }
11635 }
11636 }
11637
11638 // We push in swifterror return as the last element of CLI.Ins.
11639 ArgListTy &Args = CLI.getArgs();
11640 if (supportSwiftError()) {
11641 for (const ArgListEntry &Arg : Args) {
11642 if (Arg.IsSwiftError) {
11643 ISD::ArgFlagsTy Flags;
11644 Flags.setSwiftError();
11645 ISD::InputArg Ret(Flags, getPointerTy(DL), EVT(getPointerTy(DL)),
11646 PointerType::getUnqual(C&: Context),
11647 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
11648 CLI.Ins.push_back(Elt: Ret);
11649 }
11650 }
11651 }
11652
11653 // Handle all of the outgoing arguments.
11654 CLI.Outs.clear();
11655 CLI.OutVals.clear();
11656 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11657 SmallVector<Type *, 4> OrigArgTys;
11658 ComputeValueTypes(DL, Ty: Args[i].OrigTy, Types&: OrigArgTys);
11659 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11660 Type *FinalType = Args[i].Ty;
11661 if (Args[i].IsByVal)
11662 FinalType = Args[i].IndirectType;
11663 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11664 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11665 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
11666 ++Value) {
11667 Type *OrigArgTy = OrigArgTys[Value];
11668 Type *ArgTy = OrigArgTy;
11669 if (Args[i].Ty != Args[i].OrigTy) {
11670 assert(Value == 0 && "Only supported for non-aggregate arguments");
11671 ArgTy = Args[i].Ty;
11672 }
11673
11674 EVT VT = getValueType(DL, Ty: ArgTy);
11675 SDValue Op = SDValue(Args[i].Node.getNode(),
11676 Args[i].Node.getResNo() + Value);
11677 ISD::ArgFlagsTy Flags;
11678
11679 // Certain targets (such as MIPS), may have a different ABI alignment
11680 // for a type depending on the context. Give the target a chance to
11681 // specify the alignment it wants.
11682 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11683 Flags.setOrigAlign(OriginalAlignment);
11684
11685 if (i >= CLI.NumFixedArgs)
11686 Flags.setVarArg();
11687 if (ArgTy->isPointerTy()) {
11688 Flags.setPointer();
11689 Flags.setPointerAddrSpace(cast<PointerType>(Val: ArgTy)->getAddressSpace());
11690 }
11691 if (Args[i].IsZExt)
11692 Flags.setZExt();
11693 if (Args[i].IsSExt)
11694 Flags.setSExt();
11695 if (Args[i].IsNoExt)
11696 Flags.setNoExt();
11697 if (Args[i].IsInReg) {
11698 // If we are using vectorcall calling convention, a structure that is
11699 // passed InReg - is surely an HVA
11700 if (CLI.CallConv == CallingConv::X86_VectorCall &&
11701 isa<StructType>(Val: FinalType)) {
11702 // The first value of a structure is marked
11703 if (0 == Value)
11704 Flags.setHvaStart();
11705 Flags.setHva();
11706 }
11707 // Set InReg Flag
11708 Flags.setInReg();
11709 }
11710 if (Args[i].IsSRet)
11711 Flags.setSRet();
11712 if (Args[i].IsSwiftSelf)
11713 Flags.setSwiftSelf();
11714 if (Args[i].IsSwiftAsync)
11715 Flags.setSwiftAsync();
11716 if (Args[i].IsSwiftError)
11717 Flags.setSwiftError();
11718 if (Args[i].IsCFGuardTarget)
11719 Flags.setCFGuardTarget();
11720 if (Args[i].IsByVal)
11721 Flags.setByVal();
11722 if (Args[i].IsByRef)
11723 Flags.setByRef();
11724 if (Args[i].IsPreallocated) {
11725 Flags.setPreallocated();
11726 // Set the byval flag for CCAssignFn callbacks that don't know about
11727 // preallocated. This way we can know how many bytes we should've
11728 // allocated and how many bytes a callee cleanup function will pop. If
11729 // we port preallocated to more targets, we'll have to add custom
11730 // preallocated handling in the various CC lowering callbacks.
11731 Flags.setByVal();
11732 }
11733 if (Args[i].IsInAlloca) {
11734 Flags.setInAlloca();
11735 // Set the byval flag for CCAssignFn callbacks that don't know about
11736 // inalloca. This way we can know how many bytes we should've allocated
11737 // and how many bytes a callee cleanup function will pop. If we port
11738 // inalloca to more targets, we'll have to add custom inalloca handling
11739 // in the various CC lowering callbacks.
11740 Flags.setByVal();
11741 }
11742 Align MemAlign;
11743 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11744 unsigned FrameSize = DL.getTypeAllocSize(Ty: Args[i].IndirectType);
11745 Flags.setByValSize(FrameSize);
11746
11747 // info is not there but there are cases it cannot get right.
11748 if (auto MA = Args[i].Alignment)
11749 MemAlign = *MA;
11750 else
11751 MemAlign = getByValTypeAlignment(Ty: Args[i].IndirectType, DL);
11752 } else if (auto MA = Args[i].Alignment) {
11753 MemAlign = *MA;
11754 } else {
11755 MemAlign = OriginalAlignment;
11756 }
11757 Flags.setMemAlign(MemAlign);
11758 if (Args[i].IsNest)
11759 Flags.setNest();
11760 if (NeedsRegBlock)
11761 Flags.setInConsecutiveRegs();
11762
11763 MVT PartVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11764 unsigned NumParts =
11765 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11766 SmallVector<SDValue, 4> Parts(NumParts);
11767 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
11768
11769 if (Args[i].IsSExt)
11770 ExtendKind = ISD::SIGN_EXTEND;
11771 else if (Args[i].IsZExt)
11772 ExtendKind = ISD::ZERO_EXTEND;
11773
11774 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11775 // for now.
11776 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11777 CanLowerReturn) {
11778 assert((CLI.RetTy == Args[i].Ty ||
11779 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11780 CLI.RetTy->getPointerAddressSpace() ==
11781 Args[i].Ty->getPointerAddressSpace())) &&
11782 RetVTs.size() == NumValues && "unexpected use of 'returned'");
11783 // Before passing 'returned' to the target lowering code, ensure that
11784 // either the register MVT and the actual EVT are the same size or that
11785 // the return value and argument are extended in the same way; in these
11786 // cases it's safe to pass the argument register value unchanged as the
11787 // return register value (although it's at the target's option whether
11788 // to do so)
11789 // TODO: allow code generation to take advantage of partially preserved
11790 // registers rather than clobbering the entire register when the
11791 // parameter extension method is not compatible with the return
11792 // extension method
11793 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11794 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11795 CLI.RetZExt == Args[i].IsZExt))
11796 Flags.setReturned();
11797 }
11798
11799 getCopyToParts(DAG&: CLI.DAG, DL: CLI.DL, Val: Op, Parts: &Parts[0], NumParts, PartVT, V: CLI.CB,
11800 CallConv: CLI.CallConv, ExtendKind);
11801
11802 for (unsigned j = 0; j != NumParts; ++j) {
11803 // if it isn't first piece, alignment must be 1
11804 // For scalable vectors the scalable part is currently handled
11805 // by individual targets, so we just use the known minimum size here.
11806 ISD::OutputArg MyFlags(
11807 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
11808 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11809 if (NumParts > 1 && j == 0)
11810 MyFlags.Flags.setSplit();
11811 else if (j != 0) {
11812 MyFlags.Flags.setOrigAlign(Align(1));
11813 if (j == NumParts - 1)
11814 MyFlags.Flags.setSplitEnd();
11815 }
11816
11817 CLI.Outs.push_back(Elt: MyFlags);
11818 CLI.OutVals.push_back(Elt: Parts[j]);
11819 }
11820
11821 if (NeedsRegBlock && Value == NumValues - 1)
11822 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11823 }
11824 }
11825
11826 SmallVector<SDValue, 4> InVals;
11827 CLI.Chain = LowerCall(CLI, InVals);
11828
11829 // Update CLI.InVals to use outside of this function.
11830 CLI.InVals = InVals;
11831
11832 // Verify that the target's LowerCall behaved as expected.
11833 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11834 "LowerCall didn't return a valid chain!");
11835 assert((!CLI.IsTailCall || InVals.empty()) &&
11836 "LowerCall emitted a return value for a tail call!");
11837 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11838 "LowerCall didn't emit the correct number of values!");
11839
11840 // For a tail call, the return value is merely live-out and there aren't
11841 // any nodes in the DAG representing it. Return a special value to
11842 // indicate that a tail call has been emitted and no more Instructions
11843 // should be processed in the current block.
11844 if (CLI.IsTailCall) {
11845 CLI.DAG.setRoot(CLI.Chain);
11846 return std::make_pair(x: SDValue(), y: SDValue());
11847 }
11848
11849#ifndef NDEBUG
11850 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11851 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11852 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11853 "LowerCall emitted a value with the wrong type!");
11854 }
11855#endif
11856
11857 SmallVector<SDValue, 4> ReturnValues;
11858 if (!CanLowerReturn) {
11859 // The instruction result is the result of loading from the
11860 // hidden sret parameter.
11861 MVT PtrVT = getPointerTy(DL, AS: DL.getAllocaAddrSpace());
11862
11863 unsigned NumValues = RetVTs.size();
11864 ReturnValues.resize(N: NumValues);
11865 SmallVector<SDValue, 4> Chains(NumValues);
11866
11867 // An aggregate return value cannot wrap around the address space, so
11868 // offsets to its parts don't wrap either.
11869 MachineFunction &MF = CLI.DAG.getMachineFunction();
11870 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(ObjectIdx: DemoteStackIdx);
11871 for (unsigned i = 0; i < NumValues; ++i) {
11872 SDValue Add = CLI.DAG.getMemBasePlusOffset(
11873 Base: DemoteStackSlot, Offset: CLI.DAG.getConstant(Val: Offsets[i], DL: CLI.DL, VT: PtrVT),
11874 DL: CLI.DL, Flags: SDNodeFlags::NoUnsignedWrap);
11875 SDValue L = CLI.DAG.getLoad(
11876 VT: RetVTs[i], dl: CLI.DL, Chain: CLI.Chain, Ptr: Add,
11877 PtrInfo: MachinePointerInfo::getFixedStack(MF&: CLI.DAG.getMachineFunction(),
11878 FI: DemoteStackIdx, Offset: Offsets[i]),
11879 Alignment: HiddenSRetAlign);
11880 ReturnValues[i] = L;
11881 Chains[i] = L.getValue(R: 1);
11882 }
11883
11884 CLI.Chain = CLI.DAG.getNode(Opcode: ISD::TokenFactor, DL: CLI.DL, VT: MVT::Other, Ops: Chains);
11885 } else {
11886 // Collect the legal value parts into potentially illegal values
11887 // that correspond to the original function's return values.
11888 std::optional<ISD::NodeType> AssertOp;
11889 if (CLI.RetSExt)
11890 AssertOp = ISD::AssertSext;
11891 else if (CLI.RetZExt)
11892 AssertOp = ISD::AssertZext;
11893 unsigned CurReg = 0;
11894 for (EVT VT : RetVTs) {
11895 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11896 unsigned NumRegs =
11897 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11898
11899 ReturnValues.push_back(Elt: getCopyFromParts(
11900 DAG&: CLI.DAG, DL: CLI.DL, Parts: &InVals[CurReg], NumParts: NumRegs, PartVT: RegisterVT, ValueVT: VT, V: nullptr,
11901 InChain: CLI.Chain, CC: CLI.CallConv, AssertOp));
11902 CurReg += NumRegs;
11903 }
11904
11905 // For a function returning void, there is no return value. We can't create
11906 // such a node, so we just return a null return value in that case. In
11907 // that case, nothing will actually look at the value.
11908 if (ReturnValues.empty())
11909 return std::make_pair(x: SDValue(), y&: CLI.Chain);
11910 }
11911
11912 SDValue Res = CLI.DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: CLI.DL,
11913 VTList: CLI.DAG.getVTList(VTs: RetVTs), Ops: ReturnValues);
11914 return std::make_pair(x&: Res, y&: CLI.Chain);
11915}
11916
11917/// Places new result values for the node in Results (their number
11918/// and types must exactly match those of the original return values of
11919/// the node), or leaves Results empty, which indicates that the node is not
11920/// to be custom lowered after all.
11921void TargetLowering::LowerOperationWrapper(SDNode *N,
11922 SmallVectorImpl<SDValue> &Results,
11923 SelectionDAG &DAG) const {
11924 SDValue Res = LowerOperation(Op: SDValue(N, 0), DAG);
11925
11926 if (!Res.getNode())
11927 return;
11928
11929 // If the original node has one result, take the return value from
11930 // LowerOperation as is. It might not be result number 0.
11931 if (N->getNumValues() == 1) {
11932 Results.push_back(Elt: Res);
11933 return;
11934 }
11935
11936 // If the original node has multiple results, then the return node should
11937 // have the same number of results.
11938 assert((N->getNumValues() == Res->getNumValues()) &&
11939 "Lowering returned the wrong number of results!");
11940
11941 // Places new result values base on N result number.
11942 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11943 Results.push_back(Elt: Res.getValue(R: I));
11944}
11945
11946SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
11947 llvm_unreachable("LowerOperation not implemented for this target!");
11948}
11949
11950void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
11951 Register Reg,
11952 ISD::NodeType ExtendType) {
11953 SDValue Op = getNonRegisterValue(V);
11954 assert((Op.getOpcode() != ISD::CopyFromReg ||
11955 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11956 "Copy from a reg to the same reg!");
11957 assert(!Reg.isPhysical() && "Is a physreg");
11958
11959 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11960 // If this is an InlineAsm we have to match the registers required, not the
11961 // notional registers required by the type.
11962
11963 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11964 std::nullopt); // This is not an ABI copy.
11965 SDValue Chain = DAG.getEntryNode();
11966
11967 if (ExtendType == ISD::ANY_EXTEND) {
11968 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(Val: V);
11969 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11970 ExtendType = PreferredExtendIt->second;
11971 }
11972 RFV.getCopyToRegs(Val: Op, DAG, dl: getCurSDLoc(), Chain, Glue: nullptr, V, PreferredExtendType: ExtendType);
11973 PendingExports.push_back(Elt: Chain);
11974}
11975
11976#include "llvm/CodeGen/SelectionDAGISel.h"
11977
11978/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11979/// entry block, return true. This includes arguments used by switches, since
11980/// the switch may expand into multiple basic blocks.
11981static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11982 // With FastISel active, we may be splitting blocks, so force creation
11983 // of virtual registers for all non-dead arguments.
11984 if (FastISel)
11985 return A->use_empty();
11986
11987 const BasicBlock &Entry = A->getParent()->front();
11988 for (const User *U : A->users())
11989 if (cast<Instruction>(Val: U)->getParent() != &Entry || isa<SwitchInst>(Val: U))
11990 return false; // Use not in entry block.
11991
11992 return true;
11993}
11994
11995using ArgCopyElisionMapTy =
11996 DenseMap<const Argument *,
11997 std::pair<const AllocaInst *, const StoreInst *>>;
11998
11999/// Scan the entry block of the function in FuncInfo for arguments that look
12000/// like copies into a local alloca. Record any copied arguments in
12001/// ArgCopyElisionCandidates.
12002static void
12003findArgumentCopyElisionCandidates(const DataLayout &DL,
12004 FunctionLoweringInfo *FuncInfo,
12005 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
12006 // Record the state of every static alloca used in the entry block. Argument
12007 // allocas are all used in the entry block, so we need approximately as many
12008 // entries as we have arguments.
12009 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
12010 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
12011 unsigned NumArgs = FuncInfo->Fn->arg_size();
12012 StaticAllocas.reserve(NumEntries: NumArgs * 2);
12013
12014 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
12015 if (!V)
12016 return nullptr;
12017 V = V->stripPointerCasts();
12018 const auto *AI = dyn_cast<AllocaInst>(Val: V);
12019 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(Val: AI))
12020 return nullptr;
12021 auto Iter = StaticAllocas.insert(KV: {AI, Unknown});
12022 return &Iter.first->second;
12023 };
12024
12025 // Look for stores of arguments to static allocas. Look through bitcasts and
12026 // GEPs to handle type coercions, as long as the alloca is fully initialized
12027 // by the store. Any non-store use of an alloca escapes it and any subsequent
12028 // unanalyzed store might write it.
12029 // FIXME: Handle structs initialized with multiple stores.
12030 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
12031 // Look for stores, and handle non-store uses conservatively.
12032 const auto *SI = dyn_cast<StoreInst>(Val: &I);
12033 if (!SI) {
12034 // We will look through cast uses, so ignore them completely.
12035 if (I.isCast())
12036 continue;
12037 // Ignore pseudo op intrinsics, they don't escape or store to allocas.
12038 if (isa<PseudoProbeInst>(Val: I))
12039 continue;
12040 // This is an unknown instruction. Assume it escapes or writes to all
12041 // static alloca operands.
12042 for (const Use &U : I.operands()) {
12043 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
12044 *Info = StaticAllocaInfo::Clobbered;
12045 }
12046 continue;
12047 }
12048
12049 // If the stored value is a static alloca, mark it as escaped.
12050 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
12051 *Info = StaticAllocaInfo::Clobbered;
12052
12053 // Check if the destination is a static alloca.
12054 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
12055 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
12056 if (!Info)
12057 continue;
12058 const AllocaInst *AI = cast<AllocaInst>(Val: Dst);
12059
12060 // Skip allocas that have been initialized or clobbered.
12061 if (*Info != StaticAllocaInfo::Unknown)
12062 continue;
12063
12064 // Check if the stored value is an argument, and that this store fully
12065 // initializes the alloca.
12066 // If the argument type has padding bits we can't directly forward a pointer
12067 // as the upper bits may contain garbage.
12068 // Don't elide copies from the same argument twice.
12069 const Value *Val = SI->getValueOperand()->stripPointerCasts();
12070 const auto *Arg = dyn_cast<Argument>(Val);
12071 std::optional<TypeSize> AllocaSize = AI->getAllocationSize(DL);
12072 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
12073 Arg->getType()->isEmptyTy() || !AllocaSize ||
12074 DL.getTypeStoreSize(Ty: Arg->getType()) != *AllocaSize ||
12075 !DL.typeSizeEqualsStoreSize(Ty: Arg->getType()) ||
12076 ArgCopyElisionCandidates.count(Val: Arg)) {
12077 *Info = StaticAllocaInfo::Clobbered;
12078 continue;
12079 }
12080
12081 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
12082 << '\n');
12083
12084 // Mark this alloca and store for argument copy elision.
12085 *Info = StaticAllocaInfo::Elidable;
12086 ArgCopyElisionCandidates.insert(KV: {Arg, {AI, SI}});
12087
12088 // Stop scanning if we've seen all arguments. This will happen early in -O0
12089 // builds, which is useful, because -O0 builds have large entry blocks and
12090 // many allocas.
12091 if (ArgCopyElisionCandidates.size() == NumArgs)
12092 break;
12093 }
12094}
12095
12096/// Try to elide argument copies from memory into a local alloca. Succeeds if
12097/// ArgVal is a load from a suitable fixed stack object.
12098static void tryToElideArgumentCopy(
12099 FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
12100 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
12101 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
12102 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
12103 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
12104 // Check if this is a load from a fixed stack object.
12105 auto *LNode = dyn_cast<LoadSDNode>(Val: ArgVals[0]);
12106 if (!LNode)
12107 return;
12108 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode());
12109 if (!FINode)
12110 return;
12111
12112 // Check that the fixed stack object is the right size and alignment.
12113 // Look at the alignment that the user wrote on the alloca instead of looking
12114 // at the stack object.
12115 auto ArgCopyIter = ArgCopyElisionCandidates.find(Val: &Arg);
12116 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
12117 const AllocaInst *AI = ArgCopyIter->second.first;
12118 int FixedIndex = FINode->getIndex();
12119 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
12120 int OldIndex = AllocaIndex;
12121 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
12122 if (MFI.getObjectSize(ObjectIdx: FixedIndex) != MFI.getObjectSize(ObjectIdx: OldIndex)) {
12123 LLVM_DEBUG(
12124 dbgs() << " argument copy elision failed due to bad fixed stack "
12125 "object size\n");
12126 return;
12127 }
12128 Align RequiredAlignment = AI->getAlign();
12129 if (MFI.getObjectAlign(ObjectIdx: FixedIndex) < RequiredAlignment) {
12130 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
12131 "greater than stack argument alignment ("
12132 << DebugStr(RequiredAlignment) << " vs "
12133 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
12134 return;
12135 }
12136
12137 // Perform the elision. Delete the old stack object and replace its only use
12138 // in the variable info map. Mark the stack object as mutable and aliased.
12139 LLVM_DEBUG({
12140 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
12141 << " Replacing frame index " << OldIndex << " with " << FixedIndex
12142 << '\n';
12143 });
12144 MFI.RemoveStackObject(ObjectIdx: OldIndex);
12145 MFI.setIsImmutableObjectIndex(ObjectIdx: FixedIndex, IsImmutable: false);
12146 MFI.setIsAliasedObjectIndex(ObjectIdx: FixedIndex, IsAliased: true);
12147 AllocaIndex = FixedIndex;
12148 ArgCopyElisionFrameIndexMap.insert(KV: {OldIndex, FixedIndex});
12149 for (SDValue ArgVal : ArgVals)
12150 Chains.push_back(Elt: ArgVal.getValue(R: 1));
12151
12152 // Avoid emitting code for the store implementing the copy.
12153 const StoreInst *SI = ArgCopyIter->second.second;
12154 ElidedArgCopyInstrs.insert(Ptr: SI);
12155
12156 // Check for uses of the argument again so that we can avoid exporting ArgVal
12157 // if it is't used by anything other than the store.
12158 for (const Value *U : Arg.users()) {
12159 if (U != SI) {
12160 ArgHasUses = true;
12161 break;
12162 }
12163 }
12164}
12165
12166void SelectionDAGISel::LowerArguments(const Function &F, bool FastISel) {
12167 SelectionDAG &DAG = SDB->DAG;
12168 SDLoc dl = SDB->getCurSDLoc();
12169 const DataLayout &DL = DAG.getDataLayout();
12170 SmallVector<ISD::InputArg, 16> Ins;
12171
12172 // In Naked functions we aren't going to save any registers.
12173 if (F.hasFnAttribute(Kind: Attribute::Naked))
12174 return;
12175
12176 if (!FuncInfo->CanLowerReturn) {
12177 // Put in an sret pointer parameter before all the other parameters.
12178 MVT ValueVT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
12179
12180 ISD::ArgFlagsTy Flags;
12181 Flags.setSRet();
12182 MVT RegisterVT = TLI->getRegisterType(Context&: *DAG.getContext(), VT: ValueVT);
12183 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
12184 ISD::InputArg::NoArgIndex, 0);
12185 Ins.push_back(Elt: RetArg);
12186 }
12187
12188 // Look for stores of arguments to static allocas. Mark such arguments with a
12189 // flag to ask the target to give us the memory location of that argument if
12190 // available.
12191 ArgCopyElisionMapTy ArgCopyElisionCandidates;
12192 findArgumentCopyElisionCandidates(DL, FuncInfo: FuncInfo.get(),
12193 ArgCopyElisionCandidates);
12194
12195 // Set up the incoming argument description vector.
12196 for (const Argument &Arg : F.args()) {
12197 unsigned ArgNo = Arg.getArgNo();
12198 SmallVector<Type *, 4> Types;
12199 ComputeValueTypes(DL: DAG.getDataLayout(), Ty: Arg.getType(), Types);
12200 bool isArgValueUsed = !Arg.use_empty();
12201 Type *FinalType = Arg.getType();
12202 if (Arg.hasAttribute(Kind: Attribute::ByVal))
12203 FinalType = Arg.getParamByValType();
12204 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
12205 Ty: FinalType, CallConv: F.getCallingConv(), isVarArg: F.isVarArg(), DL);
12206 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
12207 ++Value) {
12208 Type *ArgTy = Types[Value];
12209 EVT VT = TLI->getValueType(DL, Ty: ArgTy);
12210 ISD::ArgFlagsTy Flags;
12211
12212 if (ArgTy->isPointerTy()) {
12213 Flags.setPointer();
12214 Flags.setPointerAddrSpace(cast<PointerType>(Val: ArgTy)->getAddressSpace());
12215 }
12216 if (Arg.hasAttribute(Kind: Attribute::ZExt))
12217 Flags.setZExt();
12218 if (Arg.hasAttribute(Kind: Attribute::SExt))
12219 Flags.setSExt();
12220 if (Arg.hasAttribute(Kind: Attribute::InReg)) {
12221 // If we are using vectorcall calling convention, a structure that is
12222 // passed InReg - is surely an HVA
12223 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
12224 isa<StructType>(Val: Arg.getType())) {
12225 // The first value of a structure is marked
12226 if (0 == Value)
12227 Flags.setHvaStart();
12228 Flags.setHva();
12229 }
12230 // Set InReg Flag
12231 Flags.setInReg();
12232 }
12233 if (Arg.hasAttribute(Kind: Attribute::StructRet))
12234 Flags.setSRet();
12235 if (Arg.hasAttribute(Kind: Attribute::SwiftSelf))
12236 Flags.setSwiftSelf();
12237 if (Arg.hasAttribute(Kind: Attribute::SwiftAsync))
12238 Flags.setSwiftAsync();
12239 if (Arg.hasAttribute(Kind: Attribute::SwiftError))
12240 Flags.setSwiftError();
12241 if (Arg.hasAttribute(Kind: Attribute::ByVal))
12242 Flags.setByVal();
12243 if (Arg.hasAttribute(Kind: Attribute::ByRef))
12244 Flags.setByRef();
12245 if (Arg.hasAttribute(Kind: Attribute::InAlloca)) {
12246 Flags.setInAlloca();
12247 // Set the byval flag for CCAssignFn callbacks that don't know about
12248 // inalloca. This way we can know how many bytes we should've allocated
12249 // and how many bytes a callee cleanup function will pop. If we port
12250 // inalloca to more targets, we'll have to add custom inalloca handling
12251 // in the various CC lowering callbacks.
12252 Flags.setByVal();
12253 }
12254 if (Arg.hasAttribute(Kind: Attribute::Preallocated)) {
12255 Flags.setPreallocated();
12256 // Set the byval flag for CCAssignFn callbacks that don't know about
12257 // preallocated. This way we can know how many bytes we should've
12258 // allocated and how many bytes a callee cleanup function will pop. If
12259 // we port preallocated to more targets, we'll have to add custom
12260 // preallocated handling in the various CC lowering callbacks.
12261 Flags.setByVal();
12262 }
12263
12264 // Certain targets (such as MIPS), may have a different ABI alignment
12265 // for a type depending on the context. Give the target a chance to
12266 // specify the alignment it wants.
12267 const Align OriginalAlignment(
12268 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
12269 Flags.setOrigAlign(OriginalAlignment);
12270
12271 Align MemAlign;
12272 Type *ArgMemTy = nullptr;
12273 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
12274 Flags.isByRef()) {
12275 if (!ArgMemTy)
12276 ArgMemTy = Arg.getPointeeInMemoryValueType();
12277
12278 uint64_t MemSize = DL.getTypeAllocSize(Ty: ArgMemTy);
12279
12280 // For in-memory arguments, size and alignment should be passed from FE.
12281 // BE will guess if this info is not there but there are cases it cannot
12282 // get right.
12283 if (auto ParamAlign = Arg.getParamStackAlign())
12284 MemAlign = *ParamAlign;
12285 else if ((ParamAlign = Arg.getParamAlign()))
12286 MemAlign = *ParamAlign;
12287 else
12288 MemAlign = TLI->getByValTypeAlignment(Ty: ArgMemTy, DL);
12289 if (Flags.isByRef())
12290 Flags.setByRefSize(MemSize);
12291 else
12292 Flags.setByValSize(MemSize);
12293 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
12294 MemAlign = *ParamAlign;
12295 } else {
12296 MemAlign = OriginalAlignment;
12297 }
12298 Flags.setMemAlign(MemAlign);
12299
12300 if (Arg.hasAttribute(Kind: Attribute::Nest))
12301 Flags.setNest();
12302 if (NeedsRegBlock)
12303 Flags.setInConsecutiveRegs();
12304 if (ArgCopyElisionCandidates.count(Val: &Arg))
12305 Flags.setCopyElisionCandidate();
12306 if (Arg.hasAttribute(Kind: Attribute::Returned))
12307 Flags.setReturned();
12308
12309 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
12310 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12311 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
12312 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12313 for (unsigned i = 0; i != NumRegs; ++i) {
12314 // For scalable vectors, use the minimum size; individual targets
12315 // are responsible for handling scalable vector arguments and
12316 // return values.
12317 ISD::InputArg MyFlags(
12318 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12319 i * RegisterVT.getStoreSize().getKnownMinValue());
12320 if (NumRegs > 1 && i == 0)
12321 MyFlags.Flags.setSplit();
12322 // if it isn't first piece, alignment must be 1
12323 else if (i > 0) {
12324 MyFlags.Flags.setOrigAlign(Align(1));
12325 if (i == NumRegs - 1)
12326 MyFlags.Flags.setSplitEnd();
12327 }
12328 Ins.push_back(Elt: MyFlags);
12329 }
12330 if (NeedsRegBlock && Value == NumValues - 1)
12331 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
12332 }
12333 }
12334
12335 // Call the target to set up the argument values.
12336 SmallVector<SDValue, 8> InVals;
12337 SDValue NewRoot = TLI->LowerFormalArguments(
12338 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
12339
12340 // Verify that the target's LowerFormalArguments behaved as expected.
12341 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
12342 "LowerFormalArguments didn't return a valid chain!");
12343 assert(InVals.size() == Ins.size() &&
12344 "LowerFormalArguments didn't emit the correct number of values!");
12345 assert(all_of(InVals, [](SDValue InVal) { return InVal.getNode(); }) &&
12346 "LowerFormalArguments emitted a null value!");
12347
12348 // Update the DAG with the new chain value resulting from argument lowering.
12349 DAG.setRoot(NewRoot);
12350
12351 // Set up the argument values.
12352 unsigned i = 0;
12353 if (!FuncInfo->CanLowerReturn) {
12354 // Create a virtual register for the sret pointer, and put in a copy
12355 // from the sret argument into it.
12356 MVT VT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
12357 MVT RegVT = TLI->getRegisterType(Context&: *CurDAG->getContext(), VT);
12358 std::optional<ISD::NodeType> AssertOp;
12359 SDValue ArgValue =
12360 getCopyFromParts(DAG, DL: dl, Parts: &InVals[0], NumParts: 1, PartVT: RegVT, ValueVT: VT, V: nullptr, InChain: NewRoot,
12361 CC: F.getCallingConv(), AssertOp);
12362
12363 MachineFunction& MF = SDB->DAG.getMachineFunction();
12364 MachineRegisterInfo& RegInfo = MF.getRegInfo();
12365 Register SRetReg =
12366 RegInfo.createVirtualRegister(RegClass: TLI->getRegClassFor(VT: RegVT));
12367 FuncInfo->DemoteRegister = SRetReg;
12368 NewRoot =
12369 SDB->DAG.getCopyToReg(Chain: NewRoot, dl: SDB->getCurSDLoc(), Reg: SRetReg, N: ArgValue);
12370 DAG.setRoot(NewRoot);
12371
12372 // i indexes lowered arguments. Bump it past the hidden sret argument.
12373 ++i;
12374 }
12375
12376 SmallVector<SDValue, 4> Chains;
12377 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12378 for (const Argument &Arg : F.args()) {
12379 SmallVector<SDValue, 4> ArgValues;
12380 SmallVector<EVT, 4> ValueVTs;
12381 ComputeValueVTs(TLI: *TLI, DL: DAG.getDataLayout(), Ty: Arg.getType(), ValueVTs);
12382 unsigned NumValues = ValueVTs.size();
12383 if (NumValues == 0)
12384 continue;
12385
12386 bool ArgHasUses = !Arg.use_empty();
12387
12388 // Elide the copying store if the target loaded this argument from a
12389 // suitable fixed stack object.
12390 if (Ins[i].Flags.isCopyElisionCandidate()) {
12391 unsigned NumParts = 0;
12392 for (EVT VT : ValueVTs)
12393 NumParts += TLI->getNumRegistersForCallingConv(Context&: *CurDAG->getContext(),
12394 CC: F.getCallingConv(), VT);
12395
12396 tryToElideArgumentCopy(FuncInfo&: *FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
12397 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
12398 ArgVals: ArrayRef(&InVals[i], NumParts), ArgHasUses);
12399 }
12400
12401 // If this argument is unused then remember its value. It is used to generate
12402 // debugging information.
12403 bool isSwiftErrorArg =
12404 TLI->supportSwiftError() &&
12405 Arg.hasAttribute(Kind: Attribute::SwiftError);
12406 if (!ArgHasUses && !isSwiftErrorArg) {
12407 SDB->setUnusedArgValue(V: &Arg, NewN: InVals[i]);
12408
12409 // Also remember any frame index for use in FastISel.
12410 if (FrameIndexSDNode *FI =
12411 dyn_cast<FrameIndexSDNode>(Val: InVals[i].getNode()))
12412 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12413 }
12414
12415 for (unsigned Val = 0; Val != NumValues; ++Val) {
12416 EVT VT = ValueVTs[Val];
12417 MVT PartVT = TLI->getRegisterTypeForCallingConv(Context&: *CurDAG->getContext(),
12418 CC: F.getCallingConv(), VT);
12419 unsigned NumParts = TLI->getNumRegistersForCallingConv(
12420 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12421
12422 // Even an apparent 'unused' swifterror argument needs to be returned. So
12423 // we do generate a copy for it that can be used on return from the
12424 // function.
12425 if (ArgHasUses || isSwiftErrorArg) {
12426 std::optional<ISD::NodeType> AssertOp;
12427 if (Arg.hasAttribute(Kind: Attribute::SExt))
12428 AssertOp = ISD::AssertSext;
12429 else if (Arg.hasAttribute(Kind: Attribute::ZExt))
12430 AssertOp = ISD::AssertZext;
12431
12432 SDValue OutVal =
12433 getCopyFromParts(DAG, DL: dl, Parts: &InVals[i], NumParts, PartVT, ValueVT: VT, V: nullptr,
12434 InChain: NewRoot, CC: F.getCallingConv(), AssertOp);
12435
12436 FPClassTest NoFPClass = Arg.getNoFPClass();
12437 if (NoFPClass != fcNone) {
12438 SDValue SDNoFPClass = DAG.getTargetConstant(
12439 Val: static_cast<uint64_t>(NoFPClass), DL: dl, VT: MVT::i32);
12440 OutVal = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: OutVal.getValueType(),
12441 N1: OutVal, N2: SDNoFPClass);
12442 }
12443 ArgValues.push_back(Elt: OutVal);
12444 }
12445
12446 i += NumParts;
12447 }
12448
12449 // We don't need to do anything else for unused arguments.
12450 if (ArgValues.empty())
12451 continue;
12452
12453 // Note down frame index.
12454 if (FrameIndexSDNode *FI =
12455 dyn_cast<FrameIndexSDNode>(Val: ArgValues[0].getNode()))
12456 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12457
12458 SDValue Res = DAG.getMergeValues(Ops: ArrayRef(ArgValues.data(), NumValues),
12459 dl: SDB->getCurSDLoc());
12460
12461 SDB->setValue(V: &Arg, NewN: Res);
12462 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
12463 // We want to associate the argument with the frame index, among
12464 // involved operands, that correspond to the lowest address. The
12465 // getCopyFromParts function, called earlier, is swapping the order of
12466 // the operands to BUILD_PAIR depending on endianness. The result of
12467 // that swapping is that the least significant bits of the argument will
12468 // be in the first operand of the BUILD_PAIR node, and the most
12469 // significant bits will be in the second operand.
12470 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
12471 if (LoadSDNode *LNode =
12472 dyn_cast<LoadSDNode>(Val: Res.getOperand(i: LowAddressOp).getNode()))
12473 if (FrameIndexSDNode *FI =
12474 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
12475 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12476 }
12477
12478 // Analyses past this point are naive and don't expect an assertion.
12479 if (Res.getOpcode() == ISD::AssertZext)
12480 Res = Res.getOperand(i: 0);
12481
12482 // Update the SwiftErrorVRegDefMap.
12483 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
12484 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
12485 if (Reg.isVirtual())
12486 SwiftError->setCurrentVReg(MBB: FuncInfo->MBB, SwiftError->getFunctionArg(),
12487 Reg);
12488 }
12489
12490 // If this argument is live outside of the entry block, insert a copy from
12491 // wherever we got it to the vreg that other BB's will reference it as.
12492 if (Res.getOpcode() == ISD::CopyFromReg) {
12493 // If we can, though, try to skip creating an unnecessary vreg.
12494 // FIXME: This isn't very clean... it would be nice to make this more
12495 // general.
12496 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
12497 if (Reg.isVirtual()) {
12498 FuncInfo->ValueMap[&Arg] = Reg;
12499 continue;
12500 }
12501 }
12502 if (!isOnlyUsedInEntryBlock(A: &Arg, FastISel)) {
12503 FuncInfo->InitializeRegForValue(V: &Arg);
12504 SDB->CopyToExportRegsIfNeeded(V: &Arg);
12505 }
12506 }
12507
12508 if (!Chains.empty()) {
12509 Chains.push_back(Elt: NewRoot);
12510 NewRoot = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
12511 }
12512
12513 DAG.setRoot(NewRoot);
12514
12515 assert(i == InVals.size() && "Argument register count mismatch!");
12516
12517 // If any argument copy elisions occurred and we have debug info, update the
12518 // stale frame indices used in the dbg.declare variable info table.
12519 if (!ArgCopyElisionFrameIndexMap.empty()) {
12520 for (MachineFunction::VariableDbgInfo &VI :
12521 MF->getInStackSlotVariableDbgInfo()) {
12522 auto I = ArgCopyElisionFrameIndexMap.find(Val: VI.getStackSlot());
12523 if (I != ArgCopyElisionFrameIndexMap.end())
12524 VI.updateStackSlot(NewSlot: I->second);
12525 }
12526 }
12527
12528 // Finally, if the target has anything special to do, allow it to do so.
12529 emitFunctionEntryCode();
12530}
12531
12532/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
12533/// ensure constants are generated when needed. Remember the virtual registers
12534/// that need to be added to the Machine PHI nodes as input. We cannot just
12535/// directly add them, because expansion might result in multiple MBB's for one
12536/// BB. As such, the start of the BB might correspond to a different MBB than
12537/// the end.
12538void
12539SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
12540 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12541
12542 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12543
12544 // Check PHI nodes in successors that expect a value to be available from this
12545 // block.
12546 for (const BasicBlock *SuccBB : successors(I: LLVMBB->getTerminator())) {
12547 if (!isa<PHINode>(Val: SuccBB->begin())) continue;
12548 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(BB: SuccBB);
12549
12550 // If this terminator has multiple identical successors (common for
12551 // switches), only handle each succ once.
12552 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
12553 continue;
12554
12555 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
12556
12557 // At this point we know that there is a 1-1 correspondence between LLVM PHI
12558 // nodes and Machine PHI nodes, but the incoming operands have not been
12559 // emitted yet.
12560 for (const PHINode &PN : SuccBB->phis()) {
12561 // Ignore dead phi's.
12562 if (PN.use_empty())
12563 continue;
12564
12565 // Skip empty types
12566 if (PN.getType()->isEmptyTy())
12567 continue;
12568
12569 Register Reg;
12570 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
12571
12572 if (const auto *C = dyn_cast<Constant>(Val: PHIOp)) {
12573 Register &RegOut = ConstantsOut[C];
12574 if (!RegOut) {
12575 RegOut = FuncInfo.CreateRegs(V: PHIOp);
12576 // We need to zero/sign extend ConstantInt phi operands to match
12577 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12578 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12579 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
12580 ExtendType = TLI.signExtendConstant(C: CI) ? ISD::SIGN_EXTEND
12581 : ISD::ZERO_EXTEND;
12582 CopyValueToVirtualRegister(V: C, Reg: RegOut, ExtendType);
12583 }
12584 Reg = RegOut;
12585 } else {
12586 auto I = FuncInfo.ValueMap.find(Val: PHIOp);
12587 if (I != FuncInfo.ValueMap.end())
12588 Reg = I->second;
12589 else {
12590 assert(isa<AllocaInst>(PHIOp) &&
12591 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12592 "Didn't codegen value into a register!??");
12593 Reg = FuncInfo.CreateRegs(V: PHIOp);
12594 CopyValueToVirtualRegister(V: PHIOp, Reg);
12595 }
12596 }
12597
12598 // Remember that this register needs to added to the machine PHI node as
12599 // the input for this MBB.
12600 SmallVector<EVT, 4> ValueVTs;
12601 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: PN.getType(), ValueVTs);
12602 for (EVT VT : ValueVTs) {
12603 const unsigned NumRegisters = TLI.getNumRegisters(Context&: *DAG.getContext(), VT);
12604 for (unsigned i = 0; i != NumRegisters; ++i)
12605 FuncInfo.PHINodesToUpdate.emplace_back(args: &*MBBI++, args: Reg + i);
12606 Reg += NumRegisters;
12607 }
12608 }
12609 }
12610
12611 ConstantsOut.clear();
12612}
12613
12614MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12615 MachineFunction::iterator I(MBB);
12616 if (++I == FuncInfo.MF->end())
12617 return nullptr;
12618 return &*I;
12619}
12620
12621/// During lowering new call nodes can be created (such as memset, etc.).
12622/// Those will become new roots of the current DAG, but complications arise
12623/// when they are tail calls. In such cases, the call lowering will update
12624/// the root, but the builder still needs to know that a tail call has been
12625/// lowered in order to avoid generating an additional return.
12626void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12627 // If the node is null, we do have a tail call.
12628 if (MaybeTC.getNode() != nullptr)
12629 DAG.setRoot(MaybeTC);
12630 else
12631 HasTailCall = true;
12632}
12633
12634void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12635 MachineBasicBlock *SwitchMBB,
12636 MachineBasicBlock *DefaultMBB) {
12637 MachineFunction *CurMF = FuncInfo.MF;
12638 MachineBasicBlock *NextMBB = nullptr;
12639 MachineFunction::iterator BBI(W.MBB);
12640 if (++BBI != FuncInfo.MF->end())
12641 NextMBB = &*BBI;
12642
12643 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12644
12645 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12646
12647 if (Size == 2 && W.MBB == SwitchMBB) {
12648 // If any two of the cases has the same destination, and if one value
12649 // is the same as the other, but has one bit unset that the other has set,
12650 // use bit manipulation to do two compares at once. For example:
12651 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12652 // TODO: This could be extended to merge any 2 cases in switches with 3
12653 // cases.
12654 // TODO: Handle cases where W.CaseBB != SwitchBB.
12655 CaseCluster &Small = *W.FirstCluster;
12656 CaseCluster &Big = *W.LastCluster;
12657
12658 if (Small.Low == Small.High && Big.Low == Big.High &&
12659 Small.MBB == Big.MBB) {
12660 const APInt &SmallValue = Small.Low->getValue();
12661 const APInt &BigValue = Big.Low->getValue();
12662
12663 // Check that there is only one bit different.
12664 APInt CommonBit = BigValue ^ SmallValue;
12665 if (CommonBit.isPowerOf2()) {
12666 SDValue CondLHS = getValue(V: Cond);
12667 EVT VT = CondLHS.getValueType();
12668 SDLoc DL = getCurSDLoc();
12669
12670 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: CondLHS,
12671 N2: DAG.getConstant(Val: CommonBit, DL, VT));
12672 SDValue Cond = DAG.getSetCC(
12673 DL, VT: MVT::i1, LHS: Or, RHS: DAG.getConstant(Val: BigValue | SmallValue, DL, VT),
12674 Cond: ISD::SETEQ);
12675
12676 // Update successor info.
12677 // Both Small and Big will jump to Small.BB, so we sum up the
12678 // probabilities.
12679 addSuccessorWithProb(Src: SwitchMBB, Dst: Small.MBB, Prob: Small.Prob + Big.Prob);
12680 if (BPI)
12681 addSuccessorWithProb(
12682 Src: SwitchMBB, Dst: DefaultMBB,
12683 // The default destination is the first successor in IR.
12684 Prob: BPI->getEdgeProbability(Src: SwitchMBB->getBasicBlock(), IndexInSuccessors: (unsigned)0));
12685 else
12686 addSuccessorWithProb(Src: SwitchMBB, Dst: DefaultMBB);
12687
12688 // Insert the true branch.
12689 SDValue BrCond =
12690 DAG.getNode(Opcode: ISD::BRCOND, DL, VT: MVT::Other, N1: getControlRoot(), N2: Cond,
12691 N3: DAG.getBasicBlock(MBB: Small.MBB));
12692 // Insert the false branch.
12693 BrCond = DAG.getNode(Opcode: ISD::BR, DL, VT: MVT::Other, N1: BrCond,
12694 N2: DAG.getBasicBlock(MBB: DefaultMBB));
12695
12696 DAG.setRoot(BrCond);
12697 return;
12698 }
12699 }
12700 }
12701
12702 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12703 // Here, we order cases by probability so the most likely case will be
12704 // checked first. However, two clusters can have the same probability in
12705 // which case their relative ordering is non-deterministic. So we use Low
12706 // as a tie-breaker as clusters are guaranteed to never overlap.
12707 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
12708 Comp: [](const CaseCluster &a, const CaseCluster &b) {
12709 return a.Prob != b.Prob ?
12710 a.Prob > b.Prob :
12711 a.Low->getValue().slt(RHS: b.Low->getValue());
12712 });
12713
12714 // Rearrange the case blocks so that the last one falls through if possible
12715 // without changing the order of probabilities.
12716 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12717 --I;
12718 if (I->Prob > W.LastCluster->Prob)
12719 break;
12720 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12721 std::swap(a&: *I, b&: *W.LastCluster);
12722 break;
12723 }
12724 }
12725 }
12726
12727 // Compute total probability.
12728 BranchProbability DefaultProb = W.DefaultProb;
12729 BranchProbability UnhandledProbs = DefaultProb;
12730 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12731 UnhandledProbs += I->Prob;
12732
12733 MachineBasicBlock *CurMBB = W.MBB;
12734 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12735 bool FallthroughUnreachable = false;
12736 MachineBasicBlock *Fallthrough;
12737 if (I == W.LastCluster) {
12738 // For the last cluster, fall through to the default destination.
12739 Fallthrough = DefaultMBB;
12740 FallthroughUnreachable =
12741 isa<UnreachableInst>(Val: DefaultMBB->getBasicBlock()->getFirstNonPHIIt());
12742 } else {
12743 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
12744 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
12745 // Put Cond in a virtual register to make it available from the new blocks.
12746 ExportFromCurrentBlock(V: Cond);
12747 }
12748 UnhandledProbs -= I->Prob;
12749
12750 switch (I->Kind) {
12751 case CC_JumpTable: {
12752 // FIXME: Optimize away range check based on pivot comparisons.
12753 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12754 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12755
12756 // The jump block hasn't been inserted yet; insert it here.
12757 MachineBasicBlock *JumpMBB = JT->MBB;
12758 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
12759
12760 auto JumpProb = I->Prob;
12761 auto FallthroughProb = UnhandledProbs;
12762
12763 // If the default statement is a target of the jump table, we evenly
12764 // distribute the default probability to successors of CurMBB. Also
12765 // update the probability on the edge from JumpMBB to Fallthrough.
12766 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12767 SE = JumpMBB->succ_end();
12768 SI != SE; ++SI) {
12769 if (*SI == DefaultMBB) {
12770 JumpProb += DefaultProb / 2;
12771 FallthroughProb -= DefaultProb / 2;
12772 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
12773 JumpMBB->normalizeSuccProbs();
12774 break;
12775 }
12776 }
12777
12778 // If the default clause is unreachable, propagate that knowledge into
12779 // JTH->FallthroughUnreachable which will use it to suppress the range
12780 // check.
12781 //
12782 // However, don't do this if we're doing branch target enforcement
12783 // (e.g. Arm BTI, or the cf-protection-branch module flag used by
12784 // RISC-V Zicfilp and X86 IBT), because a table branch _without_ a
12785 // range check can be a tempting JOP gadget - out-of-bounds inputs
12786 // that are impossible in correct execution become possible again if
12787 // an attacker can influence the control flow. So if an attacker
12788 // doesn't already have a branch target enforcement bypass available,
12789 // we don't want them to be able to get one out of this table branch.
12790 if (FallthroughUnreachable) {
12791 Function &CurFunc = CurMF->getFunction();
12792 bool HasCFBranch = false;
12793 if (auto *CFBranch = mdconst::extract_or_null<ConstantInt>(
12794 MD: CurFunc.getParent()->getModuleFlag(Key: "cf-protection-branch")))
12795 HasCFBranch = !CFBranch->isZero();
12796 if (!HasCFBranch &&
12797 !CurFunc.hasFnAttribute(Kind: "branch-target-enforcement"))
12798 JTH->FallthroughUnreachable = true;
12799 }
12800
12801 if (!JTH->FallthroughUnreachable)
12802 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
12803 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
12804 CurMBB->normalizeSuccProbs();
12805
12806 // The jump table header will be inserted in our current block, do the
12807 // range check, and fall through to our fallthrough block.
12808 JTH->HeaderBB = CurMBB;
12809 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12810
12811 // If we're in the right place, emit the jump table header right now.
12812 if (CurMBB == SwitchMBB) {
12813 visitJumpTableHeader(JT&: *JT, JTH&: *JTH, SwitchBB: SwitchMBB);
12814 JTH->Emitted = true;
12815 }
12816 break;
12817 }
12818 case CC_BitTests: {
12819 // FIXME: Optimize away range check based on pivot comparisons.
12820 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12821
12822 // The bit test blocks haven't been inserted yet; insert them here.
12823 for (BitTestCase &BTC : BTB->Cases)
12824 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
12825
12826 // Fill in fields of the BitTestBlock.
12827 BTB->Parent = CurMBB;
12828 BTB->Default = Fallthrough;
12829
12830 BTB->DefaultProb = UnhandledProbs;
12831 // If the cases in bit test don't form a contiguous range, we evenly
12832 // distribute the probability on the edge to Fallthrough to two
12833 // successors of CurMBB.
12834 if (!BTB->ContiguousRange) {
12835 BTB->Prob += DefaultProb / 2;
12836 BTB->DefaultProb -= DefaultProb / 2;
12837 }
12838
12839 if (FallthroughUnreachable)
12840 BTB->FallthroughUnreachable = true;
12841
12842 // If we're in the right place, emit the bit test header right now.
12843 if (CurMBB == SwitchMBB) {
12844 visitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
12845 BTB->Emitted = true;
12846 }
12847 break;
12848 }
12849 case CC_Range: {
12850 const Value *RHS, *LHS, *MHS;
12851 ISD::CondCode CC;
12852 if (I->Low == I->High) {
12853 // Check Cond == I->Low.
12854 CC = ISD::SETEQ;
12855 LHS = Cond;
12856 RHS=I->Low;
12857 MHS = nullptr;
12858 } else {
12859 // Check I->Low <= Cond <= I->High.
12860 CC = ISD::SETLE;
12861 LHS = I->Low;
12862 MHS = Cond;
12863 RHS = I->High;
12864 }
12865
12866 // If Fallthrough is unreachable, fold away the comparison.
12867 if (FallthroughUnreachable)
12868 CC = ISD::SETTRUE;
12869
12870 // The false probability is the sum of all unhandled cases.
12871 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12872 getCurSDLoc(), I->Prob, UnhandledProbs);
12873
12874 if (CurMBB == SwitchMBB)
12875 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12876 else
12877 SL->SwitchCases.push_back(x: CB);
12878
12879 break;
12880 }
12881 }
12882 CurMBB = Fallthrough;
12883 }
12884}
12885
12886void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12887 const SwitchWorkListItem &W,
12888 Value *Cond,
12889 MachineBasicBlock *SwitchMBB) {
12890 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12891 "Clusters not sorted?");
12892 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12893
12894 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12895 SL->computeSplitWorkItemInfo(W);
12896
12897 // Use the first element on the right as pivot since we will make less-than
12898 // comparisons against it.
12899 CaseClusterIt PivotCluster = FirstRight;
12900 assert(PivotCluster > W.FirstCluster);
12901 assert(PivotCluster <= W.LastCluster);
12902
12903 CaseClusterIt FirstLeft = W.FirstCluster;
12904 CaseClusterIt LastRight = W.LastCluster;
12905
12906 const ConstantInt *Pivot = PivotCluster->Low;
12907
12908 // New blocks will be inserted immediately after the current one.
12909 MachineFunction::iterator BBI(W.MBB);
12910 ++BBI;
12911
12912 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12913 // we can branch to its destination directly if it's squeezed exactly in
12914 // between the known lower bound and Pivot - 1.
12915 MachineBasicBlock *LeftMBB;
12916 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12917 FirstLeft->Low == W.GE &&
12918 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12919 LeftMBB = FirstLeft->MBB;
12920 } else {
12921 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12922 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
12923 WorkList.push_back(
12924 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
12925 // Put Cond in a virtual register to make it available from the new blocks.
12926 ExportFromCurrentBlock(V: Cond);
12927 }
12928
12929 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12930 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12931 // directly if RHS.High equals the current upper bound.
12932 MachineBasicBlock *RightMBB;
12933 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12934 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12935 RightMBB = FirstRight->MBB;
12936 } else {
12937 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12938 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
12939 WorkList.push_back(
12940 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
12941 // Put Cond in a virtual register to make it available from the new blocks.
12942 ExportFromCurrentBlock(V: Cond);
12943 }
12944
12945 // Create the CaseBlock record that will be used to lower the branch.
12946 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12947 getCurSDLoc(), LeftProb, RightProb);
12948
12949 if (W.MBB == SwitchMBB)
12950 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12951 else
12952 SL->SwitchCases.push_back(x: CB);
12953}
12954
12955// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12956// from the swith statement.
12957static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
12958 BranchProbability PeeledCaseProb) {
12959 if (PeeledCaseProb == BranchProbability::getOne())
12960 return BranchProbability::getZero();
12961 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12962
12963 uint32_t Numerator = CaseProb.getNumerator();
12964 uint32_t Denominator = SwitchProb.scale(Num: CaseProb.getDenominator());
12965 return BranchProbability(Numerator, std::max(a: Numerator, b: Denominator));
12966}
12967
12968// Try to peel the top probability case if it exceeds the threshold.
12969// Return current MachineBasicBlock for the switch statement if the peeling
12970// does not occur.
12971// If the peeling is performed, return the newly created MachineBasicBlock
12972// for the peeled switch statement. Also update Clusters to remove the peeled
12973// case. PeeledCaseProb is the BranchProbability for the peeled case.
12974MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12975 const SwitchInst &SI, CaseClusterVector &Clusters,
12976 BranchProbability &PeeledCaseProb) {
12977 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12978 // Don't perform if there is only one cluster or optimizing for size.
12979 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12980 TM.getOptLevel() == CodeGenOptLevel::None ||
12981 SwitchMBB->getParent()->getFunction().hasMinSize())
12982 return SwitchMBB;
12983
12984 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
12985 unsigned PeeledCaseIndex = 0;
12986 bool SwitchPeeled = false;
12987 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12988 CaseCluster &CC = Clusters[Index];
12989 if (CC.Prob < TopCaseProb)
12990 continue;
12991 TopCaseProb = CC.Prob;
12992 PeeledCaseIndex = Index;
12993 SwitchPeeled = true;
12994 }
12995 if (!SwitchPeeled)
12996 return SwitchMBB;
12997
12998 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12999 << TopCaseProb << "\n");
13000
13001 // Record the MBB for the peeled switch statement.
13002 MachineFunction::iterator BBI(SwitchMBB);
13003 ++BBI;
13004 MachineBasicBlock *PeeledSwitchMBB =
13005 FuncInfo.MF->CreateMachineBasicBlock(BB: SwitchMBB->getBasicBlock());
13006 FuncInfo.MF->insert(MBBI: BBI, MBB: PeeledSwitchMBB);
13007
13008 ExportFromCurrentBlock(V: SI.getCondition());
13009 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
13010 SwitchWorkListItem W = {.MBB: SwitchMBB, .FirstCluster: PeeledCaseIt, .LastCluster: PeeledCaseIt,
13011 .GE: nullptr, .LT: nullptr, .DefaultProb: TopCaseProb.getCompl()};
13012 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB: PeeledSwitchMBB);
13013
13014 Clusters.erase(position: PeeledCaseIt);
13015 for (CaseCluster &CC : Clusters) {
13016 LLVM_DEBUG(
13017 dbgs() << "Scale the probablity for one cluster, before scaling: "
13018 << CC.Prob << "\n");
13019 CC.Prob = scaleCaseProbality(CaseProb: CC.Prob, PeeledCaseProb: TopCaseProb);
13020 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
13021 }
13022 PeeledCaseProb = TopCaseProb;
13023 return PeeledSwitchMBB;
13024}
13025
13026void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
13027 // Extract cases from the switch.
13028 BranchProbabilityInfo *BPI = FuncInfo.BPI;
13029 CaseClusterVector Clusters;
13030 Clusters.reserve(n: SI.getNumCases());
13031 for (auto I : SI.cases()) {
13032 MachineBasicBlock *Succ = FuncInfo.getMBB(BB: I.getCaseSuccessor());
13033 const ConstantInt *CaseVal = I.getCaseValue();
13034 BranchProbability Prob =
13035 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
13036 : BranchProbability(1, SI.getNumCases() + 1);
13037 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
13038 }
13039
13040 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(BB: SI.getDefaultDest());
13041
13042 // Cluster adjacent cases with the same destination. We do this at all
13043 // optimization levels because it's cheap to do and will make codegen faster
13044 // if there are many clusters.
13045 sortAndRangeify(Clusters);
13046
13047 // The branch probablity of the peeled case.
13048 BranchProbability PeeledCaseProb = BranchProbability::getZero();
13049 MachineBasicBlock *PeeledSwitchMBB =
13050 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
13051
13052 // If there is only the default destination, jump there directly.
13053 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
13054 if (Clusters.empty()) {
13055 assert(PeeledSwitchMBB == SwitchMBB);
13056 SwitchMBB->addSuccessor(Succ: DefaultMBB);
13057 if (DefaultMBB != NextBlock(MBB: SwitchMBB)) {
13058 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other,
13059 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: DefaultMBB)));
13060 }
13061 return;
13062 }
13063
13064 SL->findJumpTables(Clusters, SI: &SI, SL: getCurSDLoc(), DefaultMBB, PSI: DAG.getPSI(),
13065 BFI: DAG.getBFI());
13066 SL->findBitTestClusters(Clusters, SI: &SI);
13067
13068 LLVM_DEBUG({
13069 dbgs() << "Case clusters: ";
13070 for (const CaseCluster &C : Clusters) {
13071 if (C.Kind == CC_JumpTable)
13072 dbgs() << "JT:";
13073 if (C.Kind == CC_BitTests)
13074 dbgs() << "BT:";
13075
13076 C.Low->getValue().print(dbgs(), true);
13077 if (C.Low != C.High) {
13078 dbgs() << '-';
13079 C.High->getValue().print(dbgs(), true);
13080 }
13081 dbgs() << ' ';
13082 }
13083 dbgs() << '\n';
13084 });
13085
13086 assert(!Clusters.empty());
13087 SwitchWorkList WorkList;
13088 CaseClusterIt First = Clusters.begin();
13089 CaseClusterIt Last = Clusters.end() - 1;
13090 auto DefaultProb = getEdgeProbability(Src: PeeledSwitchMBB, Dst: DefaultMBB);
13091 // Scale the branchprobability for DefaultMBB if the peel occurs and
13092 // DefaultMBB is not replaced.
13093 if (PeeledCaseProb != BranchProbability::getZero() &&
13094 DefaultMBB == FuncInfo.getMBB(BB: SI.getDefaultDest()))
13095 DefaultProb = scaleCaseProbality(CaseProb: DefaultProb, PeeledCaseProb);
13096 WorkList.push_back(
13097 Elt: {.MBB: PeeledSwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
13098
13099 while (!WorkList.empty()) {
13100 SwitchWorkListItem W = WorkList.pop_back_val();
13101 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
13102
13103 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
13104 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
13105 // For optimized builds, lower large range as a balanced binary tree.
13106 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB);
13107 continue;
13108 }
13109
13110 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB);
13111 }
13112}
13113
13114void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
13115 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13116 auto DL = getCurSDLoc();
13117 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13118 setValue(V: &I, NewN: DAG.getStepVector(DL, ResVT: ResultVT));
13119}
13120
13121void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
13122 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13123 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13124
13125 SDLoc DL = getCurSDLoc();
13126 SDValue V = getValue(V: I.getOperand(i_nocapture: 0));
13127 assert(VT == V.getValueType() && "Malformed vector.reverse!");
13128
13129 if (VT.isScalableVector()) {
13130 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V));
13131 return;
13132 }
13133
13134 // Use VECTOR_SHUFFLE for the fixed-length vector
13135 // to maintain existing behavior.
13136 SmallVector<int, 8> Mask;
13137 unsigned NumElts = VT.getVectorMinNumElements();
13138 for (unsigned i = 0; i != NumElts; ++i)
13139 Mask.push_back(Elt: NumElts - 1 - i);
13140
13141 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V, N2: DAG.getUNDEF(VT), Mask));
13142}
13143
13144void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
13145 unsigned Factor) {
13146 auto DL = getCurSDLoc();
13147 SDValue InVec = getValue(V: I.getOperand(i_nocapture: 0));
13148
13149 SmallVector<EVT, 4> ValueVTs;
13150 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
13151 ValueVTs);
13152
13153 EVT OutVT = ValueVTs[0];
13154 unsigned OutNumElts = OutVT.getVectorMinNumElements();
13155
13156 SmallVector<SDValue, 4> SubVecs(Factor);
13157 for (unsigned i = 0; i != Factor; ++i) {
13158 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
13159 SubVecs[i] = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OutVT, N1: InVec,
13160 N2: DAG.getVectorIdxConstant(Val: OutNumElts * i, DL));
13161 }
13162
13163 SDValue Res = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
13164 VTList: DAG.getVTList(VTs: ValueVTs), Ops: SubVecs);
13165 setValue(V: &I, NewN: Res);
13166}
13167
13168void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
13169 unsigned Factor) {
13170 auto DL = getCurSDLoc();
13171 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13172 EVT InVT = getValue(V: I.getOperand(i_nocapture: 0)).getValueType();
13173 EVT OutVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13174
13175 SmallVector<SDValue, 8> InVecs(Factor);
13176 for (unsigned i = 0; i < Factor; ++i) {
13177 InVecs[i] = getValue(V: I.getOperand(i_nocapture: i));
13178 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
13179 "Expected VTs to be the same");
13180 }
13181
13182 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
13183 SDValue Res =
13184 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, VTList: DAG.getVTList(VTs: ValueVTs), Ops: InVecs);
13185
13186 SmallVector<SDValue, 8> Results(Factor);
13187 for (unsigned i = 0; i < Factor; ++i)
13188 Results[i] = Res.getValue(R: i);
13189
13190 Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: Results);
13191 setValue(V: &I, NewN: Res);
13192}
13193
13194void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
13195 SmallVector<EVT, 4> ValueVTs;
13196 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
13197 ValueVTs);
13198 unsigned NumValues = ValueVTs.size();
13199 if (NumValues == 0) return;
13200
13201 SmallVector<SDValue, 4> Values(NumValues);
13202 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
13203
13204 for (unsigned i = 0; i != NumValues; ++i)
13205 Values[i] = DAG.getNode(Opcode: ISD::FREEZE, DL: getCurSDLoc(), VT: ValueVTs[i],
13206 Operand: SDValue(Op.getNode(), Op.getResNo() + i));
13207
13208 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
13209 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
13210}
13211
13212void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
13213 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13214 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13215
13216 SDLoc DL = getCurSDLoc();
13217 SDValue V1 = getValue(V: I.getOperand(i_nocapture: 0));
13218 SDValue V2 = getValue(V: I.getOperand(i_nocapture: 1));
13219 const bool IsLeft = I.getIntrinsicID() == Intrinsic::vector_splice_left;
13220
13221 // VECTOR_SHUFFLE doesn't support a scalable or non-constant mask.
13222 if (VT.isScalableVector() || !isa<ConstantInt>(Val: I.getOperand(i_nocapture: 2))) {
13223 SDValue Offset = DAG.getZExtOrTrunc(
13224 Op: getValue(V: I.getOperand(i_nocapture: 2)), DL, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
13225 setValue(V: &I, NewN: DAG.getNode(Opcode: IsLeft ? ISD::VECTOR_SPLICE_LEFT
13226 : ISD::VECTOR_SPLICE_RIGHT,
13227 DL, VT, N1: V1, N2: V2, N3: Offset));
13228 return;
13229 }
13230 uint64_t Imm = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->getZExtValue();
13231
13232 unsigned NumElts = VT.getVectorNumElements();
13233
13234 uint64_t Idx = IsLeft ? Imm : NumElts - Imm;
13235
13236 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
13237 SmallVector<int, 8> Mask;
13238 for (unsigned i = 0; i < NumElts; ++i)
13239 Mask.push_back(Elt: Idx + i);
13240 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask));
13241}
13242
13243// Consider the following MIR after SelectionDAG, which produces output in
13244// phyregs in the first case or virtregs in the second case.
13245//
13246// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
13247// %5:gr32 = COPY $ebx
13248// %6:gr32 = COPY $edx
13249// %1:gr32 = COPY %6:gr32
13250// %0:gr32 = COPY %5:gr32
13251//
13252// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
13253// %1:gr32 = COPY %6:gr32
13254// %0:gr32 = COPY %5:gr32
13255//
13256// Given %0, we'd like to return $ebx in the first case and %5 in the second.
13257// Given %1, we'd like to return $edx in the first case and %6 in the second.
13258//
13259// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
13260// to a single virtreg (such as %0). The remaining outputs monotonically
13261// increase in virtreg number from there. If a callbr has no outputs, then it
13262// should not have a corresponding callbr landingpad; in fact, the callbr
13263// landingpad would not even be able to refer to such a callbr.
13264static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
13265 MachineInstr *MI = MRI.def_begin(RegNo: Reg)->getParent();
13266 // There is definitely at least one copy.
13267 assert(MI->getOpcode() == TargetOpcode::COPY &&
13268 "start of copy chain MUST be COPY");
13269 Reg = MI->getOperand(i: 1).getReg();
13270
13271 // If the copied register in the first copy must be virtual.
13272 assert(Reg.isVirtual() && "expected COPY of virtual register");
13273 MI = MRI.def_begin(RegNo: Reg)->getParent();
13274
13275 // There may be an optional second copy.
13276 if (MI->getOpcode() == TargetOpcode::COPY) {
13277 assert(Reg.isVirtual() && "expected COPY of virtual register");
13278 Reg = MI->getOperand(i: 1).getReg();
13279 assert(Reg.isPhysical() && "expected COPY of physical register");
13280 } else {
13281 // The start of the chain must be an INLINEASM_BR.
13282 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13283 "end of copy chain MUST be INLINEASM_BR");
13284 }
13285
13286 return Reg;
13287}
13288
13289// We must do this walk rather than the simpler
13290// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
13291// otherwise we will end up with copies of virtregs only valid along direct
13292// edges.
13293void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
13294 SmallVector<EVT, 8> ResultVTs;
13295 SmallVector<SDValue, 8> ResultValues;
13296 const auto *CBR =
13297 cast<CallBrInst>(Val: I.getParent()->getUniquePredecessor()->getTerminator());
13298
13299 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13300 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
13301 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
13302
13303 Register InitialDef = FuncInfo.ValueMap[CBR];
13304 SDValue Chain = DAG.getRoot();
13305
13306 // Re-parse the asm constraints string.
13307 TargetLowering::AsmOperandInfoVector TargetConstraints =
13308 TLI.ParseConstraints(DL: DAG.getDataLayout(), TRI, Call: *CBR);
13309 for (auto &T : TargetConstraints) {
13310 SDISelAsmOperandInfo OpInfo(T);
13311 if (OpInfo.Type != InlineAsm::isOutput)
13312 continue;
13313
13314 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
13315 // individual constraint.
13316 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
13317
13318 switch (OpInfo.ConstraintType) {
13319 case TargetLowering::C_Register:
13320 case TargetLowering::C_RegisterClass: {
13321 // Fill in OpInfo.AssignedRegs.Regs.
13322 getRegistersForValue(DAG, DL: getCurSDLoc(), OpInfo, RefOpInfo&: OpInfo);
13323
13324 // getRegistersForValue may produce 1 to many registers based on whether
13325 // the OpInfo.ConstraintVT is legal on the target or not.
13326 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
13327 Register OriginalDef = FollowCopyChain(MRI, Reg: InitialDef++);
13328 if (OriginalDef.isPhysical())
13329 FuncInfo.MBB->addLiveIn(PhysReg: OriginalDef);
13330 // Update the assigned registers to use the original defs.
13331 Reg = OriginalDef;
13332 }
13333
13334 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
13335 DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V: CBR);
13336 ResultValues.push_back(Elt: V);
13337 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
13338 break;
13339 }
13340 case TargetLowering::C_Other: {
13341 SDValue Flag;
13342 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Glue&: Flag, DL: getCurSDLoc(),
13343 OpInfo, DAG);
13344 ++InitialDef;
13345 ResultValues.push_back(Elt: V);
13346 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
13347 break;
13348 }
13349 default:
13350 break;
13351 }
13352 }
13353 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
13354 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
13355 setValue(V: &I, NewN: V);
13356}
13357