1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/Analysis/AliasAnalysis.h"
24#include "llvm/Analysis/BranchProbabilityInfo.h"
25#include "llvm/Analysis/ConstantFolding.h"
26#include "llvm/Analysis/Loads.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/TargetTransformInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/Analysis/VectorUtils.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
34#include "llvm/CodeGen/CodeGenCommonISel.h"
35#include "llvm/CodeGen/FunctionLoweringInfo.h"
36#include "llvm/CodeGen/GCMetadata.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/MachineBasicBlock.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineInstrBuilder.h"
42#include "llvm/CodeGen/MachineInstrBundleIterator.h"
43#include "llvm/CodeGen/MachineMemOperand.h"
44#include "llvm/CodeGen/MachineModuleInfo.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/MachineRegisterInfo.h"
47#include "llvm/CodeGen/SelectionDAG.h"
48#include "llvm/CodeGen/SelectionDAGNodes.h"
49#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
50#include "llvm/CodeGen/StackMaps.h"
51#include "llvm/CodeGen/SwiftErrorValueTracking.h"
52#include "llvm/CodeGen/TargetFrameLowering.h"
53#include "llvm/CodeGen/TargetInstrInfo.h"
54#include "llvm/CodeGen/TargetOpcodes.h"
55#include "llvm/CodeGen/TargetRegisterInfo.h"
56#include "llvm/CodeGen/TargetSubtargetInfo.h"
57#include "llvm/CodeGen/WinEHFuncInfo.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
64#include "llvm/IR/ConstantRange.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
68#include "llvm/IR/DebugInfoMetadata.h"
69#include "llvm/IR/DerivedTypes.h"
70#include "llvm/IR/DiagnosticInfo.h"
71#include "llvm/IR/EHPersonalities.h"
72#include "llvm/IR/Function.h"
73#include "llvm/IR/GetElementPtrTypeIterator.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
76#include "llvm/IR/Instructions.h"
77#include "llvm/IR/IntrinsicInst.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
84#include "llvm/IR/Metadata.h"
85#include "llvm/IR/Module.h"
86#include "llvm/IR/Operator.h"
87#include "llvm/IR/PatternMatch.h"
88#include "llvm/IR/Statepoint.h"
89#include "llvm/IR/Type.h"
90#include "llvm/IR/User.h"
91#include "llvm/IR/Value.h"
92#include "llvm/MC/MCContext.h"
93#include "llvm/Support/AtomicOrdering.h"
94#include "llvm/Support/Casting.h"
95#include "llvm/Support/CommandLine.h"
96#include "llvm/Support/Compiler.h"
97#include "llvm/Support/Debug.h"
98#include "llvm/Support/InstructionCost.h"
99#include "llvm/Support/MathExtras.h"
100#include "llvm/Support/raw_ostream.h"
101#include "llvm/Target/TargetMachine.h"
102#include "llvm/Target/TargetOptions.h"
103#include "llvm/TargetParser/Triple.h"
104#include "llvm/Transforms/Utils/Local.h"
105#include <cstddef>
106#include <limits>
107#include <optional>
108#include <tuple>
109
110using namespace llvm;
111using namespace PatternMatch;
112using namespace SwitchCG;
113
114#define DEBUG_TYPE "isel"
115
116/// LimitFloatPrecision - Generate low-precision inline sequences for
117/// some float libcalls (6, 8 or 12 bits).
118static unsigned LimitFloatPrecision;
119
120static cl::opt<bool>
121 InsertAssertAlign("insert-assert-align", cl::init(Val: true),
122 cl::desc("Insert the experimental `assertalign` node."),
123 cl::ReallyHidden);
124
125static cl::opt<unsigned, true>
126 LimitFPPrecision("limit-float-precision",
127 cl::desc("Generate low-precision inline sequences "
128 "for some float libcalls"),
129 cl::location(L&: LimitFloatPrecision), cl::Hidden,
130 cl::init(Val: 0));
131
132static cl::opt<unsigned> SwitchPeelThreshold(
133 "switch-peel-threshold", cl::Hidden, cl::init(Val: 66),
134 cl::desc("Set the case probability threshold for peeling the case from a "
135 "switch statement. A value greater than 100 will void this "
136 "optimization"));
137
138// Limit the width of DAG chains. This is important in general to prevent
139// DAG-based analysis from blowing up. For example, alias analysis and
140// load clustering may not complete in reasonable time. It is difficult to
141// recognize and avoid this situation within each individual analysis, and
142// future analyses are likely to have the same behavior. Limiting DAG width is
143// the safe approach and will be especially important with global DAGs.
144//
145// MaxParallelChains default is arbitrarily high to avoid affecting
146// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
147// sequence over this should have been converted to llvm.memcpy by the
148// frontend. It is easy to induce this behavior with .ll code such as:
149// %buffer = alloca [4096 x i8]
150// %data = load [4096 x i8]* %argPtr
151// store [4096 x i8] %data, [4096 x i8]* %buffer
152static const unsigned MaxParallelChains = 64;
153
154static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
155 const SDValue *Parts, unsigned NumParts,
156 MVT PartVT, EVT ValueVT, const Value *V,
157 SDValue InChain,
158 std::optional<CallingConv::ID> CC);
159
160/// getCopyFromParts - Create a value that contains the specified legal parts
161/// combined into the value they represent. If the parts combine to a type
162/// larger than ValueVT then AssertOp can be used to specify whether the extra
163/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
164/// (ISD::AssertSext).
165static SDValue
166getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
167 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
168 SDValue InChain,
169 std::optional<CallingConv::ID> CC = std::nullopt,
170 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
171 // Let the target assemble the parts if it wants to
172 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
173 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
174 PartVT, ValueVT, CC))
175 return Val;
176
177 if (ValueVT.isVector())
178 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
179 InChain, CC);
180
181 assert(NumParts > 0 && "No parts to assemble!");
182 SDValue Val = Parts[0];
183
184 if (NumParts > 1) {
185 // Assemble the value from multiple parts.
186 if (ValueVT.isInteger()) {
187 unsigned PartBits = PartVT.getSizeInBits();
188 unsigned ValueBits = ValueVT.getSizeInBits();
189
190 // Assemble the power of 2 part.
191 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
192 unsigned RoundBits = PartBits * RoundParts;
193 EVT RoundVT = RoundBits == ValueBits ?
194 ValueVT : EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits);
195 SDValue Lo, Hi;
196
197 EVT HalfVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits/2);
198
199 if (RoundParts > 2) {
200 Lo = getCopyFromParts(DAG, DL, Parts, NumParts: RoundParts / 2, PartVT, ValueVT: HalfVT, V,
201 InChain);
202 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts / 2, NumParts: RoundParts / 2,
203 PartVT, ValueVT: HalfVT, V, InChain);
204 } else {
205 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[0]);
206 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[1]);
207 }
208
209 if (DAG.getDataLayout().isBigEndian())
210 std::swap(a&: Lo, b&: Hi);
211
212 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: RoundVT, N1: Lo, N2: Hi);
213
214 if (RoundParts < NumParts) {
215 // Assemble the trailing non-power-of-2 part.
216 unsigned OddParts = NumParts - RoundParts;
217 EVT OddVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OddParts * PartBits);
218 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts, NumParts: OddParts, PartVT,
219 ValueVT: OddVT, V, InChain, CC);
220
221 // Combine the round and odd parts.
222 Lo = Val;
223 if (DAG.getDataLayout().isBigEndian())
224 std::swap(a&: Lo, b&: Hi);
225 EVT TotalVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
226 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: TotalVT, Operand: Hi);
227 Hi = DAG.getNode(
228 Opcode: ISD::SHL, DL, VT: TotalVT, N1: Hi,
229 N2: DAG.getShiftAmountConstant(Val: Lo.getValueSizeInBits(), VT: TotalVT, DL));
230 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: TotalVT, Operand: Lo);
231 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: TotalVT, N1: Lo, N2: Hi);
232 }
233 } else if (PartVT.isFloatingPoint()) {
234 // FP split into multiple FP parts (for ppcf128)
235 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
236 "Unexpected split");
237 SDValue Lo, Hi;
238 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EVT(MVT::f64), Operand: Parts[0]);
239 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EVT(MVT::f64), Operand: Parts[1]);
240 if (TLI.hasBigEndianPartOrdering(VT: ValueVT, DL: DAG.getDataLayout()))
241 std::swap(a&: Lo, b&: Hi);
242 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: ValueVT, N1: Lo, N2: Hi);
243 } else {
244 // FP split into integer parts (soft fp)
245 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
246 !PartVT.isVector() && "Unexpected split");
247 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
248 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, ValueVT: IntVT, V,
249 InChain, CC);
250 }
251 }
252
253 // There is now one part, held in Val. Correct it to match ValueVT.
254 // PartEVT is the type of the register class that holds the value.
255 // ValueVT is the type of the inline asm operation.
256 EVT PartEVT = Val.getValueType();
257
258 if (PartEVT == ValueVT)
259 return Val;
260
261 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
262 ValueVT.bitsLT(VT: PartEVT)) {
263 // For an FP value in an integer part, we need to truncate to the right
264 // width first.
265 PartEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
266 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: PartEVT, Operand: Val);
267 }
268
269 // Handle types that have the same size.
270 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
271 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
272
273 // Handle types with different sizes.
274 if (PartEVT.isInteger() && ValueVT.isInteger()) {
275 if (ValueVT.bitsLT(VT: PartEVT)) {
276 // For a truncate, see if we have any information to
277 // indicate whether the truncated bits will always be
278 // zero or sign-extension.
279 if (AssertOp)
280 Val = DAG.getNode(Opcode: *AssertOp, DL, VT: PartEVT, N1: Val,
281 N2: DAG.getValueType(ValueVT));
282 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
283 }
284 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
285 }
286
287 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
288 // FP_ROUND's are always exact here.
289 if (ValueVT.bitsLT(VT: Val.getValueType())) {
290
291 SDValue NoChange =
292 DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
293
294 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
295 Kind: llvm::Attribute::StrictFP)) {
296 return DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL,
297 VTList: DAG.getVTList(VT1: ValueVT, VT2: MVT::Other), N1: InChain, N2: Val,
298 N3: NoChange);
299 }
300
301 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ValueVT, N1: Val, N2: NoChange);
302 }
303
304 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: ValueVT, Operand: Val);
305 }
306
307 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
308 // then truncating.
309 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
310 ValueVT.bitsLT(VT: PartEVT)) {
311 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Val);
312 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
313 }
314
315 report_fatal_error(reason: "Unknown mismatch in getCopyFromParts!");
316}
317
318static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
319 const Twine &ErrMsg) {
320 const Instruction *I = dyn_cast_or_null<Instruction>(Val: V);
321 if (!I)
322 return Ctx.emitError(ErrorStr: ErrMsg);
323
324 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
325 if (CI->isInlineAsm()) {
326 return Ctx.diagnose(DI: DiagnosticInfoInlineAsm(
327 *CI, ErrMsg + ", possible invalid constraint for vector type"));
328 }
329
330 return Ctx.emitError(I, ErrorStr: ErrMsg);
331}
332
333/// getCopyFromPartsVector - Create a value that contains the specified legal
334/// parts combined into the value they represent. If the parts combine to a
335/// type larger than ValueVT then AssertOp can be used to specify whether the
336/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
337/// ValueVT (ISD::AssertSext).
338static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
339 const SDValue *Parts, unsigned NumParts,
340 MVT PartVT, EVT ValueVT, const Value *V,
341 SDValue InChain,
342 std::optional<CallingConv::ID> CallConv) {
343 assert(ValueVT.isVector() && "Not a vector value");
344 assert(NumParts > 0 && "No parts to assemble!");
345 const bool IsABIRegCopy = CallConv.has_value();
346
347 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
348 SDValue Val = Parts[0];
349
350 // Handle a multi-element vector.
351 if (NumParts > 1) {
352 EVT IntermediateVT;
353 MVT RegisterVT;
354 unsigned NumIntermediates;
355 unsigned NumRegs;
356
357 if (IsABIRegCopy) {
358 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
359 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT,
360 NumIntermediates, RegisterVT);
361 } else {
362 NumRegs =
363 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
364 NumIntermediates, RegisterVT);
365 }
366
367 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
368 NumParts = NumRegs; // Silence a compiler warning.
369 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
370 assert(RegisterVT.getSizeInBits() ==
371 Parts[0].getSimpleValueType().getSizeInBits() &&
372 "Part type sizes don't match!");
373
374 // Assemble the parts into intermediate operands.
375 SmallVector<SDValue, 8> Ops(NumIntermediates);
376 if (NumIntermediates == NumParts) {
377 // If the register was not expanded, truncate or copy the value,
378 // as appropriate.
379 for (unsigned i = 0; i != NumParts; ++i)
380 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i], NumParts: 1, PartVT, ValueVT: IntermediateVT,
381 V, InChain, CC: CallConv);
382 } else if (NumParts > 0) {
383 // If the intermediate type was expanded, build the intermediate
384 // operands from the parts.
385 assert(NumParts % NumIntermediates == 0 &&
386 "Must expand into a divisible number of parts!");
387 unsigned Factor = NumParts / NumIntermediates;
388 for (unsigned i = 0; i != NumIntermediates; ++i)
389 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i * Factor], NumParts: Factor, PartVT,
390 ValueVT: IntermediateVT, V, InChain, CC: CallConv);
391 }
392
393 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
394 // intermediate operands.
395 EVT BuiltVectorTy =
396 IntermediateVT.isVector()
397 ? EVT::getVectorVT(
398 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(),
399 EC: IntermediateVT.getVectorElementCount() * NumParts)
400 : EVT::getVectorVT(Context&: *DAG.getContext(),
401 VT: IntermediateVT.getScalarType(),
402 NumElements: NumIntermediates);
403 Val = DAG.getNode(Opcode: IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
404 : ISD::BUILD_VECTOR,
405 DL, VT: BuiltVectorTy, Ops);
406 }
407
408 // There is now one part, held in Val. Correct it to match ValueVT.
409 EVT PartEVT = Val.getValueType();
410
411 if (PartEVT == ValueVT)
412 return Val;
413
414 if (PartEVT.isVector()) {
415 // Vector/Vector bitcast.
416 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
417 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
418
419 // If the parts vector has more elements than the value vector, then we
420 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
421 // Extract the elements we want.
422 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
423 assert((PartEVT.getVectorElementCount().getKnownMinValue() >
424 ValueVT.getVectorElementCount().getKnownMinValue()) &&
425 (PartEVT.getVectorElementCount().isScalable() ==
426 ValueVT.getVectorElementCount().isScalable()) &&
427 "Cannot narrow, it would be a lossy transformation");
428 PartEVT =
429 EVT::getVectorVT(Context&: *DAG.getContext(), VT: PartEVT.getVectorElementType(),
430 EC: ValueVT.getVectorElementCount());
431 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: PartEVT, N1: Val,
432 N2: DAG.getVectorIdxConstant(Val: 0, DL));
433 if (PartEVT == ValueVT)
434 return Val;
435 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
436 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
437
438 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
439 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
440 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
441 }
442
443 // Promoted vector extract
444 return DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueVT);
445 }
446
447 // Trivial bitcast if the types are the same size and the destination
448 // vector type is legal.
449 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
450 TLI.isTypeLegal(VT: ValueVT))
451 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
452
453 if (ValueVT.getVectorNumElements() != 1) {
454 // Certain ABIs require that vectors are passed as integers. For vectors
455 // are the same size, this is an obvious bitcast.
456 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
457 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
458 } else if (ValueVT.bitsLT(VT: PartEVT)) {
459 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
460 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
461 // Drop the extra bits.
462 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
463 return DAG.getBitcast(VT: ValueVT, V: Val);
464 }
465
466 diagnosePossiblyInvalidConstraint(
467 Ctx&: *DAG.getContext(), V, ErrMsg: "non-trivial scalar-to-vector conversion");
468 return DAG.getUNDEF(VT: ValueVT);
469 }
470
471 // Handle cases such as i8 -> <1 x i1>
472 EVT ValueSVT = ValueVT.getVectorElementType();
473 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
474 unsigned ValueSize = ValueSVT.getSizeInBits();
475 if (ValueSize == PartEVT.getSizeInBits()) {
476 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueSVT, Operand: Val);
477 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
478 // It's possible a scalar floating point type gets softened to integer and
479 // then promoted to a larger integer. If PartEVT is the larger integer
480 // we need to truncate it and then bitcast to the FP type.
481 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
482 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
483 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
484 Val = DAG.getBitcast(VT: ValueSVT, V: Val);
485 } else {
486 Val = ValueVT.isFloatingPoint()
487 ? DAG.getFPExtendOrRound(Op: Val, DL, VT: ValueSVT)
488 : DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueSVT);
489 }
490 }
491
492 return DAG.getBuildVector(VT: ValueVT, DL, Ops: Val);
493}
494
495static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
496 SDValue Val, SDValue *Parts, unsigned NumParts,
497 MVT PartVT, const Value *V,
498 std::optional<CallingConv::ID> CallConv);
499
500/// getCopyToParts - Create a series of nodes that contain the specified value
501/// split into legal parts. If the parts contain more bits than Val, then, for
502/// integers, ExtendKind can be used to specify how to generate the extra bits.
503static void
504getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
505 unsigned NumParts, MVT PartVT, const Value *V,
506 std::optional<CallingConv::ID> CallConv = std::nullopt,
507 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
508 // Let the target split the parts if it wants to
509 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
510 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
511 CC: CallConv))
512 return;
513 EVT ValueVT = Val.getValueType();
514
515 // Handle the vector case separately.
516 if (ValueVT.isVector())
517 return getCopyToPartsVector(DAG, dl: DL, Val, Parts, NumParts, PartVT, V,
518 CallConv);
519
520 unsigned OrigNumParts = NumParts;
521 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
522 "Copying to an illegal type!");
523
524 if (NumParts == 0)
525 return;
526
527 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
528 EVT PartEVT = PartVT;
529 if (PartEVT == ValueVT) {
530 assert(NumParts == 1 && "No-op copy with multiple parts!");
531 Parts[0] = Val;
532 return;
533 }
534
535 unsigned PartBits = PartVT.getSizeInBits();
536 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
537 // If the parts cover more bits than the value has, promote the value.
538 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
539 assert(NumParts == 1 && "Do not know what to promote to!");
540 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
541 } else {
542 if (ValueVT.isFloatingPoint()) {
543 // FP values need to be bitcast, then extended if they are being put
544 // into a larger container.
545 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
546 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
547 }
548 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
549 ValueVT.isInteger() &&
550 "Unknown mismatch!");
551 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
552 Val = DAG.getNode(Opcode: ExtendKind, DL, VT: ValueVT, Operand: Val);
553 if (PartVT == MVT::x86mmx)
554 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
555 }
556 } else if (PartBits == ValueVT.getSizeInBits()) {
557 // Different types of the same size.
558 assert(NumParts == 1 && PartEVT != ValueVT);
559 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
560 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
561 // If the parts cover less bits than value has, truncate the value.
562 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
563 ValueVT.isInteger() &&
564 "Unknown mismatch!");
565 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
566 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
567 if (PartVT == MVT::x86mmx)
568 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
569 }
570
571 // The value may have changed - recompute ValueVT.
572 ValueVT = Val.getValueType();
573 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
574 "Failed to tile the value with PartVT!");
575
576 if (NumParts == 1) {
577 if (PartEVT != ValueVT) {
578 diagnosePossiblyInvalidConstraint(Ctx&: *DAG.getContext(), V,
579 ErrMsg: "scalar-to-vector conversion failed");
580 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
581 }
582
583 Parts[0] = Val;
584 return;
585 }
586
587 // Expand the value into multiple parts.
588 if (NumParts & (NumParts - 1)) {
589 // The number of parts is not a power of 2. Split off and copy the tail.
590 assert(PartVT.isInteger() && ValueVT.isInteger() &&
591 "Do not know what to expand to!");
592 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
593 unsigned RoundBits = RoundParts * PartBits;
594 unsigned OddParts = NumParts - RoundParts;
595 SDValue OddVal = DAG.getNode(Opcode: ISD::SRL, DL, VT: ValueVT, N1: Val,
596 N2: DAG.getShiftAmountConstant(Val: RoundBits, VT: ValueVT, DL));
597
598 getCopyToParts(DAG, DL, Val: OddVal, Parts: Parts + RoundParts, NumParts: OddParts, PartVT, V,
599 CallConv);
600
601 if (DAG.getDataLayout().isBigEndian())
602 // The odd parts were reversed by getCopyToParts - unreverse them.
603 std::reverse(first: Parts + RoundParts, last: Parts + NumParts);
604
605 NumParts = RoundParts;
606 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
607 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
608 }
609
610 // The number of parts is a power of 2. Repeatedly bisect the value using
611 // EXTRACT_ELEMENT.
612 Parts[0] = DAG.getNode(Opcode: ISD::BITCAST, DL,
613 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
614 BitWidth: ValueVT.getSizeInBits()),
615 Operand: Val);
616
617 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
618 for (unsigned i = 0; i < NumParts; i += StepSize) {
619 unsigned ThisBits = StepSize * PartBits / 2;
620 EVT ThisVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ThisBits);
621 SDValue &Part0 = Parts[i];
622 SDValue &Part1 = Parts[i+StepSize/2];
623
624 Part1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
625 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 1, DL));
626 Part0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
627 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 0, DL));
628
629 if (ThisBits == PartBits && ThisVT != PartVT) {
630 Part0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part0);
631 Part1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part1);
632 }
633 }
634 }
635
636 if (DAG.getDataLayout().isBigEndian())
637 std::reverse(first: Parts, last: Parts + OrigNumParts);
638}
639
640static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
641 const SDLoc &DL, EVT PartVT) {
642 if (!PartVT.isVector())
643 return SDValue();
644
645 EVT ValueVT = Val.getValueType();
646 EVT PartEVT = PartVT.getVectorElementType();
647 EVT ValueEVT = ValueVT.getVectorElementType();
648 ElementCount PartNumElts = PartVT.getVectorElementCount();
649 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
650
651 // We only support widening vectors with equivalent element types and
652 // fixed/scalable properties. If a target needs to widen a fixed-length type
653 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
654 if (ElementCount::isKnownLE(LHS: PartNumElts, RHS: ValueNumElts) ||
655 PartNumElts.isScalable() != ValueNumElts.isScalable())
656 return SDValue();
657
658 // Have a try for bf16 because some targets share its ABI with fp16.
659 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
660 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
661 "Cannot widen to illegal type");
662 Val = DAG.getNode(
663 Opcode: ISD::BITCAST, DL,
664 VT: ValueVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::f16), Operand: Val);
665 } else if (PartEVT != ValueEVT) {
666 return SDValue();
667 }
668
669 // Widening a scalable vector to another scalable vector is done by inserting
670 // the vector into a larger undef one.
671 if (PartNumElts.isScalable())
672 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: PartVT, N1: DAG.getUNDEF(VT: PartVT),
673 N2: Val, N3: DAG.getVectorIdxConstant(Val: 0, DL));
674
675 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
676 // undef elements.
677 SmallVector<SDValue, 16> Ops;
678 DAG.ExtractVectorElements(Op: Val, Args&: Ops);
679 SDValue EltUndef = DAG.getUNDEF(VT: PartEVT);
680 Ops.append(NumInputs: (PartNumElts - ValueNumElts).getFixedValue(), Elt: EltUndef);
681
682 // FIXME: Use CONCAT for 2x -> 4x.
683 return DAG.getBuildVector(VT: PartVT, DL, Ops);
684}
685
686/// getCopyToPartsVector - Create a series of nodes that contain the specified
687/// value split into legal parts.
688static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
689 SDValue Val, SDValue *Parts, unsigned NumParts,
690 MVT PartVT, const Value *V,
691 std::optional<CallingConv::ID> CallConv) {
692 EVT ValueVT = Val.getValueType();
693 assert(ValueVT.isVector() && "Not a vector");
694 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
695 const bool IsABIRegCopy = CallConv.has_value();
696
697 if (NumParts == 1) {
698 EVT PartEVT = PartVT;
699 if (PartEVT == ValueVT) {
700 // Nothing to do.
701 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
702 // Bitconvert vector->vector case.
703 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
704 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
705 Val = Widened;
706 } else if (PartVT.isVector() &&
707 PartEVT.getVectorElementType().bitsGE(
708 VT: ValueVT.getVectorElementType()) &&
709 PartEVT.getVectorElementCount() ==
710 ValueVT.getVectorElementCount()) {
711
712 // Promoted vector extract
713 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
714 } else if (PartEVT.isVector() &&
715 PartEVT.getVectorElementType() !=
716 ValueVT.getVectorElementType() &&
717 TLI.getTypeAction(Context&: *DAG.getContext(), VT: ValueVT) ==
718 TargetLowering::TypeWidenVector) {
719 // Combination of widening and promotion.
720 EVT WidenVT =
721 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
722 EC: PartVT.getVectorElementCount());
723 SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: WidenVT);
724 Val = DAG.getAnyExtOrTrunc(Op: Widened, DL, VT: PartVT);
725 } else {
726 // Don't extract an integer from a float vector. This can happen if the
727 // FP type gets softened to integer and then promoted. The promotion
728 // prevents it from being picked up by the earlier bitcast case.
729 if (ValueVT.getVectorElementCount().isScalar() &&
730 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
731 // If we reach this condition and PartVT is FP, this means that
732 // ValueVT is also FP and both have a different size, otherwise we
733 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
734 // would be invalid since that would mean the smaller FP type has to
735 // be extended to the larger one.
736 if (PartVT.isFloatingPoint()) {
737 Val = DAG.getBitcast(VT: ValueVT.getScalarType(), V: Val);
738 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
739 } else
740 Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: PartVT, N1: Val,
741 N2: DAG.getVectorIdxConstant(Val: 0, DL));
742 } else {
743 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
744 assert(PartVT.getFixedSizeInBits() > ValueSize &&
745 "lossy conversion of vector to scalar type");
746 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
747 Val = DAG.getBitcast(VT: IntermediateType, V: Val);
748 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
749 }
750 }
751
752 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
753 Parts[0] = Val;
754 return;
755 }
756
757 // Handle a multi-element vector.
758 EVT IntermediateVT;
759 MVT RegisterVT;
760 unsigned NumIntermediates;
761 unsigned NumRegs;
762 if (IsABIRegCopy) {
763 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
764 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT, NumIntermediates,
765 RegisterVT);
766 } else {
767 NumRegs =
768 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
769 NumIntermediates, RegisterVT);
770 }
771
772 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
773 NumParts = NumRegs; // Silence a compiler warning.
774 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
775
776 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
777 "Mixing scalable and fixed vectors when copying in parts");
778
779 std::optional<ElementCount> DestEltCnt;
780
781 if (IntermediateVT.isVector())
782 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
783 else
784 DestEltCnt = ElementCount::getFixed(MinVal: NumIntermediates);
785
786 EVT BuiltVectorTy = EVT::getVectorVT(
787 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(), EC: *DestEltCnt);
788
789 if (ValueVT == BuiltVectorTy) {
790 // Nothing to do.
791 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
792 // Bitconvert vector->vector case.
793 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BuiltVectorTy, Operand: Val);
794 } else {
795 if (BuiltVectorTy.getVectorElementType().bitsGT(
796 VT: ValueVT.getVectorElementType())) {
797 // Integer promotion.
798 ValueVT = EVT::getVectorVT(Context&: *DAG.getContext(),
799 VT: BuiltVectorTy.getVectorElementType(),
800 EC: ValueVT.getVectorElementCount());
801 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
802 }
803
804 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: BuiltVectorTy)) {
805 Val = Widened;
806 }
807 }
808
809 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
810
811 // Split the vector into intermediate operands.
812 SmallVector<SDValue, 8> Ops(NumIntermediates);
813 for (unsigned i = 0; i != NumIntermediates; ++i) {
814 if (IntermediateVT.isVector()) {
815 // This does something sensible for scalable vectors - see the
816 // definition of EXTRACT_SUBVECTOR for further details.
817 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
818 Ops[i] =
819 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: IntermediateVT, N1: Val,
820 N2: DAG.getVectorIdxConstant(Val: i * IntermediateNumElts, DL));
821 } else {
822 Ops[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IntermediateVT, N1: Val,
823 N2: DAG.getVectorIdxConstant(Val: i, DL));
824 }
825 }
826
827 // Split the intermediate operands into legal parts.
828 if (NumParts == NumIntermediates) {
829 // If the register was not expanded, promote or copy the value,
830 // as appropriate.
831 for (unsigned i = 0; i != NumParts; ++i)
832 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i], NumParts: 1, PartVT, V, CallConv);
833 } else if (NumParts > 0) {
834 // If the intermediate type was expanded, split each the value into
835 // legal parts.
836 assert(NumIntermediates != 0 && "division by zero");
837 assert(NumParts % NumIntermediates == 0 &&
838 "Must expand into a divisible number of parts!");
839 unsigned Factor = NumParts / NumIntermediates;
840 for (unsigned i = 0; i != NumIntermediates; ++i)
841 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i * Factor], NumParts: Factor, PartVT, V,
842 CallConv);
843 }
844}
845
846static void failForInvalidBundles(const CallBase &I, StringRef Name,
847 ArrayRef<uint32_t> AllowedBundles) {
848 if (I.hasOperandBundlesOtherThan(IDs: AllowedBundles)) {
849 ListSeparator LS;
850 std::string Error;
851 raw_string_ostream OS(Error);
852 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
853 OperandBundleUse U = I.getOperandBundleAt(Index: i);
854 if (!is_contained(Range&: AllowedBundles, Element: U.getTagID()))
855 OS << LS << U.getTagName();
856 }
857 reportFatalUsageError(
858 reason: Twine("cannot lower ", Name)
859 .concat(Suffix: Twine(" with arbitrary operand bundles: ", Error)));
860 }
861}
862
863RegsForValue::RegsForValue(const SmallVector<Register, 4> &regs, MVT regvt,
864 EVT valuevt, std::optional<CallingConv::ID> CC)
865 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
866 RegCount(1, regs.size()), CallConv(CC) {}
867
868RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
869 const DataLayout &DL, Register Reg, Type *Ty,
870 std::optional<CallingConv::ID> CC) {
871 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
872
873 CallConv = CC;
874
875 for (EVT ValueVT : ValueVTs) {
876 unsigned NumRegs =
877 isABIMangled()
878 ? TLI.getNumRegistersForCallingConv(Context, CC: *CC, VT: ValueVT)
879 : TLI.getNumRegisters(Context, VT: ValueVT);
880 MVT RegisterVT =
881 isABIMangled()
882 ? TLI.getRegisterTypeForCallingConv(Context, CC: *CC, VT: ValueVT)
883 : TLI.getRegisterType(Context, VT: ValueVT);
884 for (unsigned i = 0; i != NumRegs; ++i)
885 Regs.push_back(Elt: Reg + i);
886 RegVTs.push_back(Elt: RegisterVT);
887 RegCount.push_back(Elt: NumRegs);
888 Reg = Reg.id() + NumRegs;
889 }
890}
891
892SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
893 FunctionLoweringInfo &FuncInfo,
894 const SDLoc &dl, SDValue &Chain,
895 SDValue *Glue, const Value *V) const {
896 // A Value with type {} or [0 x %t] needs no registers.
897 if (ValueVTs.empty())
898 return SDValue();
899
900 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
901
902 // Assemble the legal parts into the final values.
903 SmallVector<SDValue, 4> Values(ValueVTs.size());
904 SmallVector<SDValue, 8> Parts;
905 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
906 // Copy the legal parts from the registers.
907 EVT ValueVT = ValueVTs[Value];
908 unsigned NumRegs = RegCount[Value];
909 MVT RegisterVT = isABIMangled()
910 ? TLI.getRegisterTypeForCallingConv(
911 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
912 : RegVTs[Value];
913
914 Parts.resize(N: NumRegs);
915 for (unsigned i = 0; i != NumRegs; ++i) {
916 SDValue P;
917 if (!Glue) {
918 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT);
919 } else {
920 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT, Glue: *Glue);
921 *Glue = P.getValue(R: 2);
922 }
923
924 Chain = P.getValue(R: 1);
925 Parts[i] = P;
926
927 // If the source register was virtual and if we know something about it,
928 // add an assert node.
929 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
930 continue;
931
932 const FunctionLoweringInfo::LiveOutInfo *LOI =
933 FuncInfo.GetLiveOutRegInfo(Reg: Regs[Part+i]);
934 if (!LOI)
935 continue;
936
937 unsigned RegSize = RegisterVT.getScalarSizeInBits();
938 unsigned NumSignBits = LOI->NumSignBits;
939 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
940
941 if (NumZeroBits == RegSize) {
942 // The current value is a zero.
943 // Explicitly express that as it would be easier for
944 // optimizations to kick in.
945 Parts[i] = DAG.getConstant(Val: 0, DL: dl, VT: RegisterVT);
946 continue;
947 }
948
949 // FIXME: We capture more information than the dag can represent. For
950 // now, just use the tightest assertzext/assertsext possible.
951 bool isSExt;
952 EVT FromVT(MVT::Other);
953 if (NumZeroBits) {
954 FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
955 isSExt = false;
956 } else if (NumSignBits > 1) {
957 FromVT =
958 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumSignBits + 1);
959 isSExt = true;
960 } else {
961 continue;
962 }
963 // Add an assertion node.
964 assert(FromVT != MVT::Other);
965 Parts[i] = DAG.getNode(Opcode: isSExt ? ISD::AssertSext : ISD::AssertZext, DL: dl,
966 VT: RegisterVT, N1: P, N2: DAG.getValueType(FromVT));
967 }
968
969 Values[Value] = getCopyFromParts(DAG, DL: dl, Parts: Parts.begin(), NumParts: NumRegs,
970 PartVT: RegisterVT, ValueVT, V, InChain: Chain, CC: CallConv);
971 Part += NumRegs;
972 Parts.clear();
973 }
974
975 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values);
976}
977
978void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
979 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
980 const Value *V,
981 ISD::NodeType PreferredExtendType) const {
982 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
983 ISD::NodeType ExtendKind = PreferredExtendType;
984
985 // Get the list of the values's legal parts.
986 unsigned NumRegs = Regs.size();
987 SmallVector<SDValue, 8> Parts(NumRegs);
988 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
989 unsigned NumParts = RegCount[Value];
990
991 MVT RegisterVT = isABIMangled()
992 ? TLI.getRegisterTypeForCallingConv(
993 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
994 : RegVTs[Value];
995
996 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, VT2: RegisterVT))
997 ExtendKind = ISD::ZERO_EXTEND;
998
999 getCopyToParts(DAG, DL: dl, Val: Val.getValue(R: Val.getResNo() + Value), Parts: &Parts[Part],
1000 NumParts, PartVT: RegisterVT, V, CallConv, ExtendKind);
1001 Part += NumParts;
1002 }
1003
1004 // Copy the parts into the registers.
1005 SmallVector<SDValue, 8> Chains(NumRegs);
1006 for (unsigned i = 0; i != NumRegs; ++i) {
1007 SDValue Part;
1008 if (!Glue) {
1009 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i]);
1010 } else {
1011 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i], Glue: *Glue);
1012 *Glue = Part.getValue(R: 1);
1013 }
1014
1015 Chains[i] = Part.getValue(R: 0);
1016 }
1017
1018 if (NumRegs == 1 || Glue)
1019 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1020 // flagged to it. That is the CopyToReg nodes and the user are considered
1021 // a single scheduling unit. If we create a TokenFactor and return it as
1022 // chain, then the TokenFactor is both a predecessor (operand) of the
1023 // user as well as a successor (the TF operands are flagged to the user).
1024 // c1, f1 = CopyToReg
1025 // c2, f2 = CopyToReg
1026 // c3 = TokenFactor c1, c2
1027 // ...
1028 // = op c3, ..., f2
1029 Chain = Chains[NumRegs-1];
1030 else
1031 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
1032}
1033
1034void RegsForValue::AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching,
1035 unsigned MatchingIdx, const SDLoc &dl,
1036 SelectionDAG &DAG,
1037 std::vector<SDValue> &Ops) const {
1038 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1039
1040 InlineAsm::Flag Flag(Code, Regs.size());
1041 if (HasMatching)
1042 Flag.setMatchingOp(MatchingIdx);
1043 else if (!Regs.empty() && Regs.front().isVirtual()) {
1044 // Put the register class of the virtual registers in the flag word. That
1045 // way, later passes can recompute register class constraints for inline
1046 // assembly as well as normal instructions.
1047 // Don't do this for tied operands that can use the regclass information
1048 // from the def.
1049 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
1050 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Regs.front());
1051 Flag.setRegClass(RC->getID());
1052 }
1053
1054 SDValue Res = DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32);
1055 Ops.push_back(x: Res);
1056
1057 if (Code == InlineAsm::Kind::Clobber) {
1058 // Clobbers should always have a 1:1 mapping with registers, and may
1059 // reference registers that have illegal (e.g. vector) types. Hence, we
1060 // shouldn't try to apply any sort of splitting logic to them.
1061 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1062 "No 1:1 mapping from clobbers to regs?");
1063 Register SP = TLI.getStackPointerRegisterToSaveRestore();
1064 (void)SP;
1065 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1066 Ops.push_back(x: DAG.getRegister(Reg: Regs[I], VT: RegVTs[I]));
1067 assert(
1068 (Regs[I] != SP ||
1069 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
1070 "If we clobbered the stack pointer, MFI should know about it.");
1071 }
1072 return;
1073 }
1074
1075 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1076 MVT RegisterVT = RegVTs[Value];
1077 unsigned NumRegs = TLI.getNumRegisters(Context&: *DAG.getContext(), VT: ValueVTs[Value],
1078 RegisterVT);
1079 for (unsigned i = 0; i != NumRegs; ++i) {
1080 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1081 Register TheReg = Regs[Reg++];
1082 Ops.push_back(x: DAG.getRegister(Reg: TheReg, VT: RegisterVT));
1083 }
1084 }
1085}
1086
1087SmallVector<std::pair<Register, TypeSize>, 4>
1088RegsForValue::getRegsAndSizes() const {
1089 SmallVector<std::pair<Register, TypeSize>, 4> OutVec;
1090 unsigned I = 0;
1091 for (auto CountAndVT : zip_first(t: RegCount, u: RegVTs)) {
1092 unsigned RegCount = std::get<0>(t&: CountAndVT);
1093 MVT RegisterVT = std::get<1>(t&: CountAndVT);
1094 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1095 for (unsigned E = I + RegCount; I != E; ++I)
1096 OutVec.push_back(Elt: std::make_pair(x: Regs[I], y&: RegisterSize));
1097 }
1098 return OutVec;
1099}
1100
1101void SelectionDAGBuilder::init(GCFunctionInfo *gfi, BatchAAResults *aa,
1102 AssumptionCache *ac, const TargetLibraryInfo *li,
1103 const TargetTransformInfo &TTI) {
1104 BatchAA = aa;
1105 AC = ac;
1106 GFI = gfi;
1107 LibInfo = li;
1108 Context = DAG.getContext();
1109 LPadToCallSiteMap.clear();
1110 this->TTI = &TTI;
1111 SL->init(tli: DAG.getTargetLoweringInfo(), tm: TM, dl: DAG.getDataLayout());
1112 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1113 M: *DAG.getMachineFunction().getFunction().getParent());
1114}
1115
1116void SelectionDAGBuilder::clear() {
1117 NodeMap.clear();
1118 UnusedArgNodeMap.clear();
1119 PendingLoads.clear();
1120 PendingExports.clear();
1121 PendingConstrainedFP.clear();
1122 PendingConstrainedFPStrict.clear();
1123 CurInst = nullptr;
1124 HasTailCall = false;
1125 SDNodeOrder = LowestSDNodeOrder;
1126 StatepointLowering.clear();
1127}
1128
1129void SelectionDAGBuilder::clearDanglingDebugInfo() {
1130 DanglingDebugInfoMap.clear();
1131}
1132
1133// Update DAG root to include dependencies on Pending chains.
1134SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1135 SDValue Root = DAG.getRoot();
1136
1137 if (Pending.empty())
1138 return Root;
1139
1140 // Add current root to PendingChains, unless we already indirectly
1141 // depend on it.
1142 if (Root.getOpcode() != ISD::EntryToken) {
1143 unsigned i = 0, e = Pending.size();
1144 for (; i != e; ++i) {
1145 assert(Pending[i].getNode()->getNumOperands() > 1);
1146 if (Pending[i].getNode()->getOperand(Num: 0) == Root)
1147 break; // Don't add the root if we already indirectly depend on it.
1148 }
1149
1150 if (i == e)
1151 Pending.push_back(Elt: Root);
1152 }
1153
1154 if (Pending.size() == 1)
1155 Root = Pending[0];
1156 else
1157 Root = DAG.getTokenFactor(DL: getCurSDLoc(), Vals&: Pending);
1158
1159 DAG.setRoot(Root);
1160 Pending.clear();
1161 return Root;
1162}
1163
1164SDValue SelectionDAGBuilder::getMemoryRoot() {
1165 return updateRoot(Pending&: PendingLoads);
1166}
1167
1168SDValue SelectionDAGBuilder::getFPOperationRoot(fp::ExceptionBehavior EB) {
1169 // If the new exception behavior differs from that of the pending
1170 // ones, chain up them and update the root.
1171 switch (EB) {
1172 case fp::ExceptionBehavior::ebMayTrap:
1173 case fp::ExceptionBehavior::ebIgnore:
1174 // Floating-point exceptions produced by such operations are not intended
1175 // to be observed, so the sequence of these operations does not need to be
1176 // preserved.
1177 //
1178 // They however must not be mixed with the instructions that have strict
1179 // exception behavior. Placing an operation with 'ebIgnore' behavior between
1180 // 'ebStrict' operations could distort the observed exception behavior.
1181 if (!PendingConstrainedFPStrict.empty()) {
1182 assert(PendingConstrainedFP.empty());
1183 updateRoot(Pending&: PendingConstrainedFPStrict);
1184 }
1185 break;
1186 case fp::ExceptionBehavior::ebStrict:
1187 // Floating-point exception produced by these operations may be observed, so
1188 // they must be correctly chained. If trapping on FP exceptions is
1189 // disabled, the exceptions can be observed only by functions that read
1190 // exception flags, like 'llvm.get_fpenv' or 'fetestexcept'. It means that
1191 // the order of operations is not significant between barriers.
1192 //
1193 // If trapping is enabled, each operation becomes an implicit observation
1194 // point, so the operations must be sequenced according their original
1195 // source order.
1196 if (!PendingConstrainedFP.empty()) {
1197 assert(PendingConstrainedFPStrict.empty());
1198 updateRoot(Pending&: PendingConstrainedFP);
1199 }
1200 // TODO: Add support for trapping-enabled scenarios.
1201 }
1202 return DAG.getRoot();
1203}
1204
1205SDValue SelectionDAGBuilder::getRoot() {
1206 // Chain up all pending constrained intrinsics together with all
1207 // pending loads, by simply appending them to PendingLoads and
1208 // then calling getMemoryRoot().
1209 PendingLoads.reserve(N: PendingLoads.size() +
1210 PendingConstrainedFP.size() +
1211 PendingConstrainedFPStrict.size());
1212 PendingLoads.append(in_start: PendingConstrainedFP.begin(),
1213 in_end: PendingConstrainedFP.end());
1214 PendingLoads.append(in_start: PendingConstrainedFPStrict.begin(),
1215 in_end: PendingConstrainedFPStrict.end());
1216 PendingConstrainedFP.clear();
1217 PendingConstrainedFPStrict.clear();
1218 return getMemoryRoot();
1219}
1220
1221SDValue SelectionDAGBuilder::getControlRoot() {
1222 // We need to emit pending fpexcept.strict constrained intrinsics,
1223 // so append them to the PendingExports list.
1224 PendingExports.append(in_start: PendingConstrainedFPStrict.begin(),
1225 in_end: PendingConstrainedFPStrict.end());
1226 PendingConstrainedFPStrict.clear();
1227 return updateRoot(Pending&: PendingExports);
1228}
1229
1230void SelectionDAGBuilder::handleDebugDeclare(Value *Address,
1231 DILocalVariable *Variable,
1232 DIExpression *Expression,
1233 DebugLoc DL) {
1234 assert(Variable && "Missing variable");
1235
1236 // Check if address has undef value.
1237 if (!Address || isa<UndefValue>(Val: Address) ||
1238 (Address->use_empty() && !isa<Argument>(Val: Address))) {
1239 LLVM_DEBUG(
1240 dbgs()
1241 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1242 return;
1243 }
1244
1245 bool IsParameter = Variable->isParameter() || isa<Argument>(Val: Address);
1246
1247 SDValue &N = NodeMap[Address];
1248 if (!N.getNode() && isa<Argument>(Val: Address))
1249 // Check unused arguments map.
1250 N = UnusedArgNodeMap[Address];
1251 SDDbgValue *SDV;
1252 if (N.getNode()) {
1253 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Val: Address))
1254 Address = BCI->getOperand(i_nocapture: 0);
1255 // Parameters are handled specially.
1256 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: N.getNode());
1257 if (IsParameter && FINode) {
1258 // Byval parameter. We have a frame index at this point.
1259 SDV = DAG.getFrameIndexDbgValue(Var: Variable, Expr: Expression, FI: FINode->getIndex(),
1260 /*IsIndirect*/ true, DL, O: SDNodeOrder);
1261 } else if (isa<Argument>(Val: Address)) {
1262 // Address is an argument, so try to emit its dbg value using
1263 // virtual register info from the FuncInfo.ValueMap.
1264 EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1265 Kind: FuncArgumentDbgValueKind::Declare, N);
1266 return;
1267 } else {
1268 SDV = DAG.getDbgValue(Var: Variable, Expr: Expression, N: N.getNode(), R: N.getResNo(),
1269 IsIndirect: true, DL, O: SDNodeOrder);
1270 }
1271 DAG.AddDbgValue(DB: SDV, isParameter: IsParameter);
1272 } else {
1273 // If Address is an argument then try to emit its dbg value using
1274 // virtual register info from the FuncInfo.ValueMap.
1275 if (!EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1276 Kind: FuncArgumentDbgValueKind::Declare, N)) {
1277 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1278 << " (could not emit func-arg dbg_value)\n");
1279 }
1280 }
1281}
1282
1283void SelectionDAGBuilder::visitDbgInfo(const Instruction &I) {
1284 // Add SDDbgValue nodes for any var locs here. Do so before updating
1285 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1286 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1287 // Add SDDbgValue nodes for any var locs here. Do so before updating
1288 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1289 for (auto It = FnVarLocs->locs_begin(Before: &I), End = FnVarLocs->locs_end(Before: &I);
1290 It != End; ++It) {
1291 auto *Var = FnVarLocs->getDILocalVariable(ID: It->VariableID);
1292 dropDanglingDebugInfo(Variable: Var, Expr: It->Expr);
1293 if (It->Values.isKillLocation(Expression: It->Expr)) {
1294 handleKillDebugValue(Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder);
1295 continue;
1296 }
1297 SmallVector<Value *> Values(It->Values.location_ops());
1298 if (!handleDebugValue(Values, Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder,
1299 IsVariadic: It->Values.hasArgList())) {
1300 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1301 addDanglingDebugInfo(Values&: Vals,
1302 Var: FnVarLocs->getDILocalVariable(ID: It->VariableID),
1303 Expr: It->Expr, IsVariadic: Vals.size() > 1, DL: It->DL, Order: SDNodeOrder);
1304 }
1305 }
1306 }
1307
1308 // We must skip DbgVariableRecords if they've already been processed above as
1309 // we have just emitted the debug values resulting from assignment tracking
1310 // analysis, making any existing DbgVariableRecords redundant (and probably
1311 // less correct). We still need to process DbgLabelRecords. This does sink
1312 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1313 // be important as it does so deterministcally and ordering between
1314 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1315 // printing).
1316 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1317 // Is there is any debug-info attached to this instruction, in the form of
1318 // DbgRecord non-instruction debug-info records.
1319 for (DbgRecord &DR : I.getDbgRecordRange()) {
1320 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1321 assert(DLR->getLabel() && "Missing label");
1322 SDDbgLabel *SDV =
1323 DAG.getDbgLabel(Label: DLR->getLabel(), DL: DLR->getDebugLoc(), O: SDNodeOrder);
1324 DAG.AddDbgLabel(DB: SDV);
1325 continue;
1326 }
1327
1328 if (SkipDbgVariableRecords)
1329 continue;
1330 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1331 DILocalVariable *Variable = DVR.getVariable();
1332 DIExpression *Expression = DVR.getExpression();
1333 dropDanglingDebugInfo(Variable, Expr: Expression);
1334
1335 if (DVR.getType() == DbgVariableRecord::LocationType::Declare) {
1336 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1337 continue;
1338 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1339 << "\n");
1340 handleDebugDeclare(Address: DVR.getVariableLocationOp(OpIdx: 0), Variable, Expression,
1341 DL: DVR.getDebugLoc());
1342 continue;
1343 }
1344
1345 // A DbgVariableRecord with no locations is a kill location.
1346 SmallVector<Value *, 4> Values(DVR.location_ops());
1347 if (Values.empty()) {
1348 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1349 Order: SDNodeOrder);
1350 continue;
1351 }
1352
1353 // A DbgVariableRecord with an undef or absent location is also a kill
1354 // location.
1355 if (llvm::any_of(Range&: Values,
1356 P: [](Value *V) { return !V || isa<UndefValue>(Val: V); })) {
1357 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1358 Order: SDNodeOrder);
1359 continue;
1360 }
1361
1362 bool IsVariadic = DVR.hasArgList();
1363 if (!handleDebugValue(Values, Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1364 Order: SDNodeOrder, IsVariadic)) {
1365 addDanglingDebugInfo(Values, Var: Variable, Expr: Expression, IsVariadic,
1366 DL: DVR.getDebugLoc(), Order: SDNodeOrder);
1367 }
1368 }
1369}
1370
1371void SelectionDAGBuilder::visit(const Instruction &I) {
1372 visitDbgInfo(I);
1373
1374 // Set up outgoing PHI node register values before emitting the terminator.
1375 if (I.isTerminator()) {
1376 HandlePHINodesInSuccessorBlocks(LLVMBB: I.getParent());
1377 }
1378
1379 ++SDNodeOrder;
1380 CurInst = &I;
1381
1382 // Set inserted listener only if required.
1383 bool NodeInserted = false;
1384 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1385 MDNode *PCSectionsMD = I.getMetadata(KindID: LLVMContext::MD_pcsections);
1386 MDNode *MMRA = I.getMetadata(KindID: LLVMContext::MD_mmra);
1387 if (PCSectionsMD || MMRA) {
1388 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1389 args&: DAG, args: [&](SDNode *) { NodeInserted = true; });
1390 }
1391
1392 visit(Opcode: I.getOpcode(), I);
1393
1394 if (!I.isTerminator() && !HasTailCall &&
1395 !isa<GCStatepointInst>(Val: I)) // statepoints handle their exports internally
1396 CopyToExportRegsIfNeeded(V: &I);
1397
1398 // Handle metadata.
1399 if (PCSectionsMD || MMRA) {
1400 auto It = NodeMap.find(Val: &I);
1401 if (It != NodeMap.end()) {
1402 if (PCSectionsMD)
1403 DAG.addPCSections(Node: It->second.getNode(), MD: PCSectionsMD);
1404 if (MMRA)
1405 DAG.addMMRAMetadata(Node: It->second.getNode(), MMRA);
1406 } else if (NodeInserted) {
1407 // This should not happen; if it does, don't let it go unnoticed so we can
1408 // fix it. Relevant visit*() function is probably missing a setValue().
1409 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1410 << I.getModule()->getName() << "]\n";
1411 LLVM_DEBUG(I.dump());
1412 assert(false);
1413 }
1414 }
1415
1416 CurInst = nullptr;
1417}
1418
1419void SelectionDAGBuilder::visitPHI(const PHINode &) {
1420 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1421}
1422
1423void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1424 // Note: this doesn't use InstVisitor, because it has to work with
1425 // ConstantExpr's in addition to instructions.
1426 switch (Opcode) {
1427 default: llvm_unreachable("Unknown instruction type encountered!");
1428 // Build the switch statement using the Instruction.def file.
1429#define HANDLE_INST(NUM, OPCODE, CLASS) \
1430 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1431#include "llvm/IR/Instruction.def"
1432 }
1433}
1434
1435static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,
1436 DILocalVariable *Variable,
1437 DebugLoc DL, unsigned Order,
1438 SmallVectorImpl<Value *> &Values,
1439 DIExpression *Expression) {
1440 // For variadic dbg_values we will now insert poison.
1441 // FIXME: We can potentially recover these!
1442 SmallVector<SDDbgOperand, 2> Locs;
1443 for (const Value *V : Values) {
1444 auto *Poison = PoisonValue::get(T: V->getType());
1445 Locs.push_back(Elt: SDDbgOperand::fromConst(Const: Poison));
1446 }
1447 SDDbgValue *SDV = DAG.getDbgValueList(Var: Variable, Expr: Expression, Locs, Dependencies: {},
1448 /*IsIndirect=*/false, DL, O: Order,
1449 /*IsVariadic=*/true);
1450 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1451 return true;
1452}
1453
1454void SelectionDAGBuilder::addDanglingDebugInfo(SmallVectorImpl<Value *> &Values,
1455 DILocalVariable *Var,
1456 DIExpression *Expr,
1457 bool IsVariadic, DebugLoc DL,
1458 unsigned Order) {
1459 if (IsVariadic) {
1460 handleDanglingVariadicDebugInfo(DAG, Variable: Var, DL, Order, Values, Expression: Expr);
1461 return;
1462 }
1463 // TODO: Dangling debug info will eventually either be resolved or produce
1464 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1465 // between the original dbg.value location and its resolved DBG_VALUE,
1466 // which we should ideally fill with an extra poison DBG_VALUE.
1467 assert(Values.size() == 1);
1468 DanglingDebugInfoMap[Values[0]].emplace_back(args&: Var, args&: Expr, args&: DL, args&: Order);
1469}
1470
1471void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1472 const DIExpression *Expr) {
1473 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1474 DIVariable *DanglingVariable = DDI.getVariable();
1475 DIExpression *DanglingExpr = DDI.getExpression();
1476 if (DanglingVariable == Variable && Expr->fragmentsOverlap(Other: DanglingExpr)) {
1477 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1478 << printDDI(nullptr, DDI) << "\n");
1479 return true;
1480 }
1481 return false;
1482 };
1483
1484 for (auto &DDIMI : DanglingDebugInfoMap) {
1485 DanglingDebugInfoVector &DDIV = DDIMI.second;
1486
1487 // If debug info is to be dropped, run it through final checks to see
1488 // whether it can be salvaged.
1489 for (auto &DDI : DDIV)
1490 if (isMatchingDbgValue(DDI))
1491 salvageUnresolvedDbgValue(V: DDIMI.first, DDI);
1492
1493 erase_if(C&: DDIV, P: isMatchingDbgValue);
1494 }
1495}
1496
1497// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1498// generate the debug data structures now that we've seen its definition.
1499void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1500 SDValue Val) {
1501 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(Key: V);
1502 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1503 return;
1504
1505 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1506 for (auto &DDI : DDIV) {
1507 DebugLoc DL = DDI.getDebugLoc();
1508 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1509 DILocalVariable *Variable = DDI.getVariable();
1510 DIExpression *Expr = DDI.getExpression();
1511 assert(Variable->isValidLocationForIntrinsic(DL) &&
1512 "Expected inlined-at fields to agree");
1513 SDDbgValue *SDV;
1514 if (Val.getNode()) {
1515 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1516 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1517 // we couldn't resolve it directly when examining the DbgValue intrinsic
1518 // in the first place we should not be more successful here). Unless we
1519 // have some test case that prove this to be correct we should avoid
1520 // calling EmitFuncArgumentDbgValue here.
1521 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1522 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1523 Kind: FuncArgumentDbgValueKind::Value, N: Val)) {
1524 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1525 << printDDI(V, DDI) << "\n");
1526 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1527 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1528 // inserted after the definition of Val when emitting the instructions
1529 // after ISel. An alternative could be to teach
1530 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1531 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1532 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1533 << ValSDNodeOrder << "\n");
1534 SDV = getDbgValue(N: Val, Variable, Expr, dl: DL,
1535 DbgSDNodeOrder: std::max(a: DbgSDNodeOrder, b: ValSDNodeOrder));
1536 DAG.AddDbgValue(DB: SDV, isParameter: false);
1537 } else
1538 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1539 << printDDI(V, DDI)
1540 << " in EmitFuncArgumentDbgValue\n");
1541 } else {
1542 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1543 << "\n");
1544 auto Poison = PoisonValue::get(T: V->getType());
1545 auto SDV =
1546 DAG.getConstantDbgValue(Var: Variable, Expr, C: Poison, DL, O: DbgSDNodeOrder);
1547 DAG.AddDbgValue(DB: SDV, isParameter: false);
1548 }
1549 }
1550 DDIV.clear();
1551}
1552
1553void SelectionDAGBuilder::salvageUnresolvedDbgValue(const Value *V,
1554 DanglingDebugInfo &DDI) {
1555 // TODO: For the variadic implementation, instead of only checking the fail
1556 // state of `handleDebugValue`, we need know specifically which values were
1557 // invalid, so that we attempt to salvage only those values when processing
1558 // a DIArgList.
1559 const Value *OrigV = V;
1560 DILocalVariable *Var = DDI.getVariable();
1561 DIExpression *Expr = DDI.getExpression();
1562 DebugLoc DL = DDI.getDebugLoc();
1563 unsigned SDOrder = DDI.getSDNodeOrder();
1564
1565 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1566 // that DW_OP_stack_value is desired.
1567 bool StackValue = true;
1568
1569 // Can this Value can be encoded without any further work?
1570 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false))
1571 return;
1572
1573 // Attempt to salvage back through as many instructions as possible. Bail if
1574 // a non-instruction is seen, such as a constant expression or global
1575 // variable. FIXME: Further work could recover those too.
1576 while (isa<Instruction>(Val: V)) {
1577 const Instruction &VAsInst = *cast<const Instruction>(Val: V);
1578 // Temporary "0", awaiting real implementation.
1579 SmallVector<uint64_t, 16> Ops;
1580 SmallVector<Value *, 4> AdditionalValues;
1581 V = salvageDebugInfoImpl(I&: const_cast<Instruction &>(VAsInst),
1582 CurrentLocOps: Expr->getNumLocationOperands(), Ops,
1583 AdditionalValues);
1584 // If we cannot salvage any further, and haven't yet found a suitable debug
1585 // expression, bail out.
1586 if (!V)
1587 break;
1588
1589 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1590 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1591 // here for variadic dbg_values, remove that condition.
1592 if (!AdditionalValues.empty())
1593 break;
1594
1595 // New value and expr now represent this debuginfo.
1596 Expr = DIExpression::appendOpsToArg(Expr, Ops, ArgNo: 0, StackValue);
1597
1598 // Some kind of simplification occurred: check whether the operand of the
1599 // salvaged debug expression can be encoded in this DAG.
1600 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false)) {
1601 LLVM_DEBUG(
1602 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1603 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1604 return;
1605 }
1606 }
1607
1608 // This was the final opportunity to salvage this debug information, and it
1609 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1610 // any earlier variable location.
1611 assert(OrigV && "V shouldn't be null");
1612 auto *Poison = PoisonValue::get(T: OrigV->getType());
1613 auto *SDV = DAG.getConstantDbgValue(Var, Expr, C: Poison, DL, O: SDNodeOrder);
1614 DAG.AddDbgValue(DB: SDV, isParameter: false);
1615 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1616 << printDDI(OrigV, DDI) << "\n");
1617}
1618
1619void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var,
1620 DIExpression *Expr,
1621 DebugLoc DbgLoc,
1622 unsigned Order) {
1623 Value *Poison = PoisonValue::get(T: Type::getInt1Ty(C&: *Context));
1624 DIExpression *NewExpr =
1625 const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr));
1626 handleDebugValue(Values: Poison, Var, Expr: NewExpr, DbgLoc, Order,
1627 /*IsVariadic*/ false);
1628}
1629
1630bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1631 DILocalVariable *Var,
1632 DIExpression *Expr, DebugLoc DbgLoc,
1633 unsigned Order, bool IsVariadic) {
1634 if (Values.empty())
1635 return true;
1636
1637 // Filter EntryValue locations out early.
1638 if (visitEntryValueDbgValue(Values, Variable: Var, Expr, DbgLoc))
1639 return true;
1640
1641 SmallVector<SDDbgOperand> LocationOps;
1642 SmallVector<SDNode *> Dependencies;
1643 for (const Value *V : Values) {
1644 // Constant value.
1645 if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V) || isa<UndefValue>(Val: V) ||
1646 isa<ConstantPointerNull>(Val: V)) {
1647 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: V));
1648 continue;
1649 }
1650
1651 // Look through IntToPtr constants.
1652 if (auto *CE = dyn_cast<ConstantExpr>(Val: V))
1653 if (CE->getOpcode() == Instruction::IntToPtr) {
1654 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: CE->getOperand(i_nocapture: 0)));
1655 continue;
1656 }
1657
1658 // If the Value is a frame index, we can create a FrameIndex debug value
1659 // without relying on the DAG at all.
1660 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
1661 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
1662 if (SI != FuncInfo.StaticAllocaMap.end()) {
1663 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: SI->second));
1664 continue;
1665 }
1666 }
1667
1668 // Do not use getValue() in here; we don't want to generate code at
1669 // this point if it hasn't been done yet.
1670 SDValue N = NodeMap[V];
1671 if (!N.getNode() && isa<Argument>(Val: V)) // Check unused arguments map.
1672 N = UnusedArgNodeMap[V];
1673
1674 if (N.getNode()) {
1675 // Only emit func arg dbg value for non-variadic dbg.values for now.
1676 if (!IsVariadic &&
1677 EmitFuncArgumentDbgValue(V, Variable: Var, Expr, DL: DbgLoc,
1678 Kind: FuncArgumentDbgValueKind::Value, N))
1679 return true;
1680 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
1681 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1682 // describe stack slot locations.
1683 //
1684 // Consider "int x = 0; int *px = &x;". There are two kinds of
1685 // interesting debug values here after optimization:
1686 //
1687 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1688 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1689 //
1690 // Both describe the direct values of their associated variables.
1691 Dependencies.push_back(Elt: N.getNode());
1692 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex()));
1693 continue;
1694 }
1695 LocationOps.emplace_back(
1696 Args: SDDbgOperand::fromNode(Node: N.getNode(), ResNo: N.getResNo()));
1697 continue;
1698 }
1699
1700 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1701 // Special rules apply for the first dbg.values of parameter variables in a
1702 // function. Identify them by the fact they reference Argument Values, that
1703 // they're parameters, and they are parameters of the current function. We
1704 // need to let them dangle until they get an SDNode.
1705 bool IsParamOfFunc =
1706 isa<Argument>(Val: V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1707 if (IsParamOfFunc)
1708 return false;
1709
1710 // The value is not used in this block yet (or it would have an SDNode).
1711 // We still want the value to appear for the user if possible -- if it has
1712 // an associated VReg, we can refer to that instead.
1713 auto VMI = FuncInfo.ValueMap.find(Val: V);
1714 if (VMI != FuncInfo.ValueMap.end()) {
1715 Register Reg = VMI->second;
1716 // If this is a PHI node, it may be split up into several MI PHI nodes
1717 // (in FunctionLoweringInfo::set).
1718 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1719 V->getType(), std::nullopt);
1720 if (RFV.occupiesMultipleRegs()) {
1721 // FIXME: We could potentially support variadic dbg_values here.
1722 if (IsVariadic)
1723 return false;
1724 unsigned Offset = 0;
1725 unsigned BitsToDescribe = 0;
1726 if (auto VarSize = Var->getSizeInBits())
1727 BitsToDescribe = *VarSize;
1728 if (auto Fragment = Expr->getFragmentInfo())
1729 BitsToDescribe = Fragment->SizeInBits;
1730 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1731 // Bail out if all bits are described already.
1732 if (Offset >= BitsToDescribe)
1733 break;
1734 // TODO: handle scalable vectors.
1735 unsigned RegisterSize = RegAndSize.second;
1736 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1737 ? BitsToDescribe - Offset
1738 : RegisterSize;
1739 auto FragmentExpr = DIExpression::createFragmentExpression(
1740 Expr, OffsetInBits: Offset, SizeInBits: FragmentSize);
1741 if (!FragmentExpr)
1742 continue;
1743 SDDbgValue *SDV = DAG.getVRegDbgValue(
1744 Var, Expr: *FragmentExpr, VReg: RegAndSize.first, IsIndirect: false, DL: DbgLoc, O: Order);
1745 DAG.AddDbgValue(DB: SDV, isParameter: false);
1746 Offset += RegisterSize;
1747 }
1748 return true;
1749 }
1750 // We can use simple vreg locations for variadic dbg_values as well.
1751 LocationOps.emplace_back(Args: SDDbgOperand::fromVReg(VReg: Reg));
1752 continue;
1753 }
1754 // We failed to create a SDDbgOperand for V.
1755 return false;
1756 }
1757
1758 // We have created a SDDbgOperand for each Value in Values.
1759 assert(!LocationOps.empty());
1760 SDDbgValue *SDV =
1761 DAG.getDbgValueList(Var, Expr, Locs: LocationOps, Dependencies,
1762 /*IsIndirect=*/false, DL: DbgLoc, O: Order, IsVariadic);
1763 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1764 return true;
1765}
1766
1767void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1768 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1769 for (auto &Pair : DanglingDebugInfoMap)
1770 for (auto &DDI : Pair.second)
1771 salvageUnresolvedDbgValue(V: const_cast<Value *>(Pair.first), DDI);
1772 clearDanglingDebugInfo();
1773}
1774
1775/// getCopyFromRegs - If there was virtual register allocated for the value V
1776/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1777SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1778 DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(Val: V);
1779 SDValue Result;
1780
1781 if (It != FuncInfo.ValueMap.end()) {
1782 Register InReg = It->second;
1783
1784 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1785 DAG.getDataLayout(), InReg, Ty,
1786 std::nullopt); // This is not an ABI copy.
1787 SDValue Chain = DAG.getEntryNode();
1788 Result = RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr,
1789 V);
1790 resolveDanglingDebugInfo(V, Val: Result);
1791 }
1792
1793 return Result;
1794}
1795
1796/// getValue - Return an SDValue for the given Value.
1797SDValue SelectionDAGBuilder::getValue(const Value *V) {
1798 // If we already have an SDValue for this value, use it. It's important
1799 // to do this first, so that we don't create a CopyFromReg if we already
1800 // have a regular SDValue.
1801 SDValue &N = NodeMap[V];
1802 if (N.getNode()) return N;
1803
1804 // If there's a virtual register allocated and initialized for this
1805 // value, use it.
1806 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
1807 return copyFromReg;
1808
1809 // Otherwise create a new SDValue and remember it.
1810 SDValue Val = getValueImpl(V);
1811 NodeMap[V] = Val;
1812 resolveDanglingDebugInfo(V, Val);
1813 return Val;
1814}
1815
1816/// getNonRegisterValue - Return an SDValue for the given Value, but
1817/// don't look in FuncInfo.ValueMap for a virtual register.
1818SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1819 // If we already have an SDValue for this value, use it.
1820 SDValue &N = NodeMap[V];
1821 if (N.getNode()) {
1822 if (isIntOrFPConstant(V: N)) {
1823 // Remove the debug location from the node as the node is about to be used
1824 // in a location which may differ from the original debug location. This
1825 // is relevant to Constant and ConstantFP nodes because they can appear
1826 // as constant expressions inside PHI nodes.
1827 N->setDebugLoc(DebugLoc());
1828 }
1829 return N;
1830 }
1831
1832 // Otherwise create a new SDValue and remember it.
1833 SDValue Val = getValueImpl(V);
1834 NodeMap[V] = Val;
1835 resolveDanglingDebugInfo(V, Val);
1836 return Val;
1837}
1838
1839/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1840/// Create an SDValue for the given value.
1841SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1842 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1843
1844 if (const Constant *C = dyn_cast<Constant>(Val: V)) {
1845 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: V->getType(), AllowUnknown: true);
1846
1847 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
1848 SDLoc DL = getCurSDLoc();
1849
1850 // DAG.getConstant() may attempt to legalise the vector constant which can
1851 // significantly change the combines applied to the DAG. To reduce the
1852 // divergence when enabling ConstantInt based vectors we try to construct
1853 // the DAG in the same way as shufflevector based splats. TODO: The
1854 // divergence sometimes leads to better optimisations. Ideally we should
1855 // prevent DAG.getConstant() from legalising too early but there are some
1856 // degradations preventing this.
1857 if (VT.isScalableVector())
1858 return DAG.getNode(
1859 Opcode: ISD::SPLAT_VECTOR, DL, VT,
1860 Operand: DAG.getConstant(Val: CI->getValue(), DL, VT: VT.getVectorElementType()));
1861 if (VT.isFixedLengthVector())
1862 return DAG.getSplatBuildVector(
1863 VT, DL,
1864 Op: DAG.getConstant(Val: CI->getValue(), DL, VT: VT.getVectorElementType()));
1865 return DAG.getConstant(Val: *CI, DL, VT);
1866 }
1867
1868 if (const ConstantByte *CB = dyn_cast<ConstantByte>(Val: C))
1869 return DAG.getConstant(Val: CB->getValue(), DL: getCurSDLoc(), VT);
1870
1871 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: C))
1872 return DAG.getGlobalAddress(GV, DL: getCurSDLoc(), VT);
1873
1874 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
1875 return DAG.getNode(Opcode: ISD::PtrAuthGlobalAddress, DL: getCurSDLoc(), VT,
1876 N1: getValue(V: CPA->getPointer()), N2: getValue(V: CPA->getKey()),
1877 N3: getValue(V: CPA->getAddrDiscriminator()),
1878 N4: getValue(V: CPA->getDiscriminator()));
1879 }
1880
1881 if (isa<ConstantPointerNull>(Val: C))
1882 return DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT);
1883
1884 if (match(V: C, P: m_VScale()))
1885 return DAG.getVScale(DL: getCurSDLoc(), VT, MulImm: APInt(VT.getSizeInBits(), 1));
1886
1887 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C))
1888 return DAG.getConstantFP(V: *CFP, DL: getCurSDLoc(), VT);
1889
1890 if (isa<UndefValue>(Val: C) && !V->getType()->isAggregateType())
1891 return isa<PoisonValue>(Val: C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1892
1893 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
1894 visit(Opcode: CE->getOpcode(), I: *CE);
1895 SDValue N1 = NodeMap[V];
1896 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1897 return N1;
1898 }
1899
1900 if (isa<ConstantStruct>(Val: C) || isa<ConstantArray>(Val: C)) {
1901 SmallVector<SDValue, 4> Constants;
1902 for (const Use &U : C->operands()) {
1903 SDNode *Val = getValue(V: U).getNode();
1904 // If the operand is an empty aggregate, there are no values.
1905 if (!Val) continue;
1906 // Add each leaf value from the operand to the Constants list
1907 // to form a flattened list of all the values.
1908 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1909 Constants.push_back(Elt: SDValue(Val, i));
1910 }
1911
1912 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
1913 }
1914
1915 if (const ConstantDataSequential *CDS =
1916 dyn_cast<ConstantDataSequential>(Val: C)) {
1917 SmallVector<SDValue, 4> Ops;
1918 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1919 SDNode *Val = getValue(V: CDS->getElementAsConstant(i)).getNode();
1920 // Add each leaf value from the operand to the Constants list
1921 // to form a flattened list of all the values.
1922 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1923 Ops.push_back(Elt: SDValue(Val, i));
1924 }
1925
1926 if (isa<ArrayType>(Val: CDS->getType()))
1927 return DAG.getMergeValues(Ops, dl: getCurSDLoc());
1928 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
1929 }
1930
1931 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1932 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1933 "Unknown struct or array constant!");
1934
1935 SmallVector<EVT, 4> ValueVTs;
1936 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: C->getType(), ValueVTs);
1937 unsigned NumElts = ValueVTs.size();
1938 if (NumElts == 0)
1939 return SDValue(); // empty struct
1940 SmallVector<SDValue, 4> Constants(NumElts);
1941 for (unsigned i = 0; i != NumElts; ++i) {
1942 EVT EltVT = ValueVTs[i];
1943 if (isa<UndefValue>(Val: C))
1944 Constants[i] = DAG.getUNDEF(VT: EltVT);
1945 else if (EltVT.isFloatingPoint())
1946 Constants[i] = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
1947 else
1948 Constants[i] = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
1949 }
1950
1951 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
1952 }
1953
1954 if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C))
1955 return DAG.getBlockAddress(BA, VT);
1956
1957 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C))
1958 return getValue(V: Equiv->getGlobalValue());
1959
1960 if (const auto *NC = dyn_cast<NoCFIValue>(Val: C))
1961 return getValue(V: NC->getGlobalValue());
1962
1963 if (VT == MVT::aarch64svcount) {
1964 assert(C->isNullValue() && "Can only zero this target type!");
1965 return DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT,
1966 Operand: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::nxv16i1));
1967 }
1968
1969 if (VT.isRISCVVectorTuple()) {
1970 assert(C->isNullValue() && "Can only zero this target type!");
1971 return DAG.getNode(
1972 Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT,
1973 Operand: DAG.getNode(
1974 Opcode: ISD::SPLAT_VECTOR, DL: getCurSDLoc(),
1975 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8,
1976 NumElements: VT.getSizeInBits().getKnownMinValue() / 8, IsScalable: true),
1977 Operand: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::getIntegerVT(BitWidth: 8))));
1978 }
1979
1980 VectorType *VecTy = cast<VectorType>(Val: V->getType());
1981
1982 // Now that we know the number and type of the elements, get that number of
1983 // elements into the Ops array based on what kind of constant it is.
1984 if (const ConstantVector *CV = dyn_cast<ConstantVector>(Val: C)) {
1985 SmallVector<SDValue, 16> Ops;
1986 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
1987 for (unsigned i = 0; i != NumElements; ++i)
1988 Ops.push_back(Elt: getValue(V: CV->getOperand(i_nocapture: i)));
1989
1990 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
1991 }
1992
1993 if (isa<ConstantAggregateZero>(Val: C)) {
1994 EVT EltVT =
1995 TLI.getValueType(DL: DAG.getDataLayout(), Ty: VecTy->getElementType());
1996
1997 SDValue Op;
1998 if (EltVT.isFloatingPoint())
1999 Op = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2000 else
2001 Op = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2002
2003 return DAG.getSplat(VT, DL: getCurSDLoc(), Op);
2004 }
2005
2006 llvm_unreachable("Unknown vector constant");
2007 }
2008
2009 // If this is a static alloca, generate it as the frameindex instead of
2010 // computation.
2011 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
2012 DenseMap<const AllocaInst*, int>::iterator SI =
2013 FuncInfo.StaticAllocaMap.find(Val: AI);
2014 if (SI != FuncInfo.StaticAllocaMap.end())
2015 return DAG.getFrameIndex(
2016 FI: SI->second, VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: AI->getType()));
2017 }
2018
2019 // If this is an instruction which fast-isel has deferred, select it now.
2020 if (const Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
2021 Register InReg = FuncInfo.InitializeRegForValue(V: Inst);
2022 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
2023 Inst->getType(), std::nullopt);
2024 SDValue Chain = DAG.getEntryNode();
2025 return RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V);
2026 }
2027
2028 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(Val: V))
2029 return DAG.getMDNode(MD: cast<MDNode>(Val: MD->getMetadata()));
2030
2031 if (const auto *BB = dyn_cast<BasicBlock>(Val: V))
2032 return DAG.getBasicBlock(MBB: FuncInfo.getMBB(BB));
2033
2034 llvm_unreachable("Can't get register for value!");
2035}
2036
2037void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
2038 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2039 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
2040 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
2041 bool IsSEH = isAsynchronousEHPersonality(Pers);
2042 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
2043 if (IsSEH) {
2044 // For SEH, EHCont Guard needs to know that this catchpad is a target.
2045 CatchPadMBB->setIsEHContTarget(true);
2046 DAG.getMachineFunction().setHasEHContTarget(true);
2047 } else
2048 CatchPadMBB->setIsEHScopeEntry();
2049 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
2050 if (IsMSVCCXX || IsCoreCLR)
2051 CatchPadMBB->setIsEHFuncletEntry();
2052}
2053
2054void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
2055 // Update machine-CFG edge.
2056 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(BB: I.getSuccessor());
2057 FuncInfo.MBB->addSuccessor(Succ: TargetMBB);
2058
2059 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2060 bool IsSEH = isAsynchronousEHPersonality(Pers);
2061 if (IsSEH) {
2062 // If this is not a fall-through branch or optimizations are switched off,
2063 // emit the branch.
2064 if (TargetMBB != NextBlock(MBB: FuncInfo.MBB) ||
2065 TM.getOptLevel() == CodeGenOptLevel::None)
2066 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other,
2067 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: TargetMBB)));
2068 return;
2069 }
2070
2071 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2072 TargetMBB->setIsEHContTarget(true);
2073 DAG.getMachineFunction().setHasEHContTarget(true);
2074
2075 // Figure out the funclet membership for the catchret's successor.
2076 // This will be used by the FuncletLayout pass to determine how to order the
2077 // BB's.
2078 // A 'catchret' returns to the outer scope's color.
2079 Value *ParentPad = I.getCatchSwitchParentPad();
2080 const BasicBlock *SuccessorColor;
2081 if (isa<ConstantTokenNone>(Val: ParentPad))
2082 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2083 else
2084 SuccessorColor = cast<Instruction>(Val: ParentPad)->getParent();
2085 assert(SuccessorColor && "No parent funclet for catchret!");
2086 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(BB: SuccessorColor);
2087 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2088
2089 // Create the terminator node.
2090 SDValue Ret = DAG.getNode(Opcode: ISD::CATCHRET, DL: getCurSDLoc(), VT: MVT::Other,
2091 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: TargetMBB),
2092 N3: DAG.getBasicBlock(MBB: SuccessorColorMBB));
2093 DAG.setRoot(Ret);
2094}
2095
2096void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2097 // Don't emit any special code for the cleanuppad instruction. It just marks
2098 // the start of an EH scope/funclet.
2099 FuncInfo.MBB->setIsEHScopeEntry();
2100 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2101 if (Pers != EHPersonality::Wasm_CXX) {
2102 FuncInfo.MBB->setIsEHFuncletEntry();
2103 FuncInfo.MBB->setIsCleanupFuncletEntry();
2104 }
2105}
2106
2107/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2108/// many places it could ultimately go. In the IR, we have a single unwind
2109/// destination, but in the machine CFG, we enumerate all the possible blocks.
2110/// This function skips over imaginary basic blocks that hold catchswitch
2111/// instructions, and finds all the "real" machine
2112/// basic block destinations. As those destinations may not be successors of
2113/// EHPadBB, here we also calculate the edge probability to those destinations.
2114/// The passed-in Prob is the edge probability to EHPadBB.
2115static void findUnwindDestinations(
2116 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2117 BranchProbability Prob,
2118 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2119 &UnwindDests) {
2120 EHPersonality Personality =
2121 classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2122 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2123 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2124 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2125 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2126
2127 while (EHPadBB) {
2128 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
2129 BasicBlock *NewEHPadBB = nullptr;
2130 if (isa<LandingPadInst>(Val: Pad)) {
2131 // Stop on landingpads. They are not funclets.
2132 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2133 break;
2134 } else if (isa<CleanupPadInst>(Val: Pad)) {
2135 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2136 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2137 // which always catches an exception.
2138 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2139 UnwindDests.back().first->setIsEHScopeEntry();
2140 // In Wasm, EH scopes are not funclets
2141 if (!IsWasmCXX)
2142 UnwindDests.back().first->setIsEHFuncletEntry();
2143 break;
2144 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
2145 // Add the catchpad handlers to the possible destinations.
2146 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2147 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: CatchPadBB), Args&: Prob);
2148 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2149 if (IsMSVCCXX || IsCoreCLR)
2150 UnwindDests.back().first->setIsEHFuncletEntry();
2151 if (!IsSEH)
2152 UnwindDests.back().first->setIsEHScopeEntry();
2153 }
2154 NewEHPadBB = CatchSwitch->getUnwindDest();
2155 } else {
2156 continue;
2157 }
2158
2159 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2160 if (BPI && NewEHPadBB)
2161 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
2162 EHPadBB = NewEHPadBB;
2163 }
2164}
2165
2166void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2167 // Update successor info.
2168 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2169 auto UnwindDest = I.getUnwindDest();
2170 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2171 BranchProbability UnwindDestProb =
2172 (BPI && UnwindDest)
2173 ? BPI->getEdgeProbability(Src: FuncInfo.MBB->getBasicBlock(), Dst: UnwindDest)
2174 : BranchProbability::getZero();
2175 findUnwindDestinations(FuncInfo, EHPadBB: UnwindDest, Prob: UnwindDestProb, UnwindDests);
2176 for (auto &UnwindDest : UnwindDests) {
2177 UnwindDest.first->setIsEHPad();
2178 addSuccessorWithProb(Src: FuncInfo.MBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
2179 }
2180 FuncInfo.MBB->normalizeSuccProbs();
2181
2182 // Create the terminator node.
2183 MachineBasicBlock *CleanupPadMBB =
2184 FuncInfo.getMBB(BB: I.getCleanupPad()->getParent());
2185 SDValue Ret = DAG.getNode(Opcode: ISD::CLEANUPRET, DL: getCurSDLoc(), VT: MVT::Other,
2186 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: CleanupPadMBB));
2187 DAG.setRoot(Ret);
2188}
2189
2190void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2191 report_fatal_error(reason: "visitCatchSwitch not yet implemented!");
2192}
2193
2194void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2195 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2196 auto &DL = DAG.getDataLayout();
2197 SDValue Chain = getControlRoot();
2198 SmallVector<ISD::OutputArg, 8> Outs;
2199 SmallVector<SDValue, 8> OutVals;
2200
2201 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2202 // lower
2203 //
2204 // %val = call <ty> @llvm.experimental.deoptimize()
2205 // ret <ty> %val
2206 //
2207 // differently.
2208 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2209 LowerDeoptimizingReturn();
2210 return;
2211 }
2212
2213 if (!FuncInfo.CanLowerReturn) {
2214 Register DemoteReg = FuncInfo.DemoteRegister;
2215
2216 // Emit a store of the return value through the virtual register.
2217 // Leave Outs empty so that LowerReturn won't try to load return
2218 // registers the usual way.
2219 MVT PtrValueVT = TLI.getPointerTy(DL, AS: DL.getAllocaAddrSpace());
2220 SDValue RetPtr =
2221 DAG.getCopyFromReg(Chain, dl: getCurSDLoc(), Reg: DemoteReg, VT: PtrValueVT);
2222 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2223
2224 SmallVector<EVT, 4> ValueVTs, MemVTs;
2225 SmallVector<uint64_t, 4> Offsets;
2226 ComputeValueVTs(TLI, DL, Ty: I.getOperand(i_nocapture: 0)->getType(), ValueVTs, MemVTs: &MemVTs,
2227 FixedOffsets: &Offsets, StartingOffset: 0);
2228 unsigned NumValues = ValueVTs.size();
2229
2230 SmallVector<SDValue, 4> Chains(NumValues);
2231 Align BaseAlign = DL.getPrefTypeAlign(Ty: I.getOperand(i_nocapture: 0)->getType());
2232 for (unsigned i = 0; i != NumValues; ++i) {
2233 // An aggregate return value cannot wrap around the address space, so
2234 // offsets to its parts don't wrap either.
2235 SDValue Ptr = DAG.getObjectPtrOffset(SL: getCurSDLoc(), Ptr: RetPtr,
2236 Offset: TypeSize::getFixed(ExactSize: Offsets[i]));
2237
2238 SDValue Val = RetOp.getValue(R: RetOp.getResNo() + i);
2239 if (MemVTs[i] != ValueVTs[i])
2240 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: getCurSDLoc(), VT: MemVTs[i]);
2241 Chains[i] = DAG.getStore(
2242 Chain, dl: getCurSDLoc(), Val,
2243 // FIXME: better loc info would be nice.
2244 Ptr, PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
2245 Alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]));
2246 }
2247
2248 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: getCurSDLoc(),
2249 VT: MVT::Other, Ops: Chains);
2250 } else if (I.getNumOperands() != 0) {
2251 SmallVector<Type *, 4> Types;
2252 ComputeValueTypes(DL, Ty: I.getOperand(i_nocapture: 0)->getType(), Types);
2253 unsigned NumValues = Types.size();
2254 if (NumValues) {
2255 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2256
2257 const Function *F = I.getParent()->getParent();
2258
2259 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2260 Ty: I.getOperand(i_nocapture: 0)->getType(), CallConv: F->getCallingConv(),
2261 /*IsVarArg*/ isVarArg: false, DL);
2262
2263 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2264 if (F->getAttributes().hasRetAttr(Kind: Attribute::SExt))
2265 ExtendKind = ISD::SIGN_EXTEND;
2266 else if (F->getAttributes().hasRetAttr(Kind: Attribute::ZExt))
2267 ExtendKind = ISD::ZERO_EXTEND;
2268
2269 LLVMContext &Context = F->getContext();
2270 bool RetInReg = F->getAttributes().hasRetAttr(Kind: Attribute::InReg);
2271
2272 for (unsigned j = 0; j != NumValues; ++j) {
2273 EVT VT = TLI.getValueType(DL, Ty: Types[j]);
2274
2275 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2276 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2277
2278 CallingConv::ID CC = F->getCallingConv();
2279
2280 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2281 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2282 SmallVector<SDValue, 4> Parts(NumParts);
2283 getCopyToParts(DAG, DL: getCurSDLoc(),
2284 Val: SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2285 Parts: &Parts[0], NumParts, PartVT, V: &I, CallConv: CC, ExtendKind);
2286
2287 // 'inreg' on function refers to return value
2288 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2289 if (RetInReg)
2290 Flags.setInReg();
2291
2292 if (I.getOperand(i_nocapture: 0)->getType()->isPointerTy()) {
2293 Flags.setPointer();
2294 Flags.setPointerAddrSpace(
2295 cast<PointerType>(Val: I.getOperand(i_nocapture: 0)->getType())->getAddressSpace());
2296 }
2297
2298 if (NeedsRegBlock) {
2299 Flags.setInConsecutiveRegs();
2300 if (j == NumValues - 1)
2301 Flags.setInConsecutiveRegsLast();
2302 }
2303
2304 // Propagate extension type if any
2305 if (ExtendKind == ISD::SIGN_EXTEND)
2306 Flags.setSExt();
2307 else if (ExtendKind == ISD::ZERO_EXTEND)
2308 Flags.setZExt();
2309 else if (F->getAttributes().hasRetAttr(Kind: Attribute::NoExt))
2310 Flags.setNoExt();
2311
2312 for (unsigned i = 0; i < NumParts; ++i) {
2313 Outs.push_back(Elt: ISD::OutputArg(Flags,
2314 Parts[i].getValueType().getSimpleVT(),
2315 VT, Types[j], 0, 0));
2316 OutVals.push_back(Elt: Parts[i]);
2317 }
2318 }
2319 }
2320 }
2321
2322 // Push in swifterror virtual register as the last element of Outs. This makes
2323 // sure swifterror virtual register will be returned in the swifterror
2324 // physical register.
2325 const Function *F = I.getParent()->getParent();
2326 if (TLI.supportSwiftError() &&
2327 F->getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError)) {
2328 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2329 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2330 Flags.setSwiftError();
2331 Outs.push_back(Elt: ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
2332 /*argvt=*/EVT(TLI.getPointerTy(DL)),
2333 PointerType::getUnqual(C&: *DAG.getContext()),
2334 /*origidx=*/1, /*partOffs=*/0));
2335 // Create SDNode for the swifterror virtual register.
2336 OutVals.push_back(
2337 Elt: DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(
2338 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2339 VT: EVT(TLI.getPointerTy(DL))));
2340 }
2341
2342 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2343 CallingConv::ID CallConv =
2344 DAG.getMachineFunction().getFunction().getCallingConv();
2345 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2346 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2347
2348 // Verify that the target's LowerReturn behaved as expected.
2349 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2350 "LowerReturn didn't return a valid chain!");
2351
2352 // Update the DAG with the new chain value resulting from return lowering.
2353 DAG.setRoot(Chain);
2354}
2355
2356/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2357/// created for it, emit nodes to copy the value into the virtual
2358/// registers.
2359void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2360 // Skip empty types
2361 if (V->getType()->isEmptyTy())
2362 return;
2363
2364 DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(Val: V);
2365 if (VMI != FuncInfo.ValueMap.end()) {
2366 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2367 "Unused value assigned virtual registers!");
2368 CopyValueToVirtualRegister(V, Reg: VMI->second);
2369 }
2370}
2371
2372/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2373/// the current basic block, add it to ValueMap now so that we'll get a
2374/// CopyTo/FromReg.
2375void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2376 // No need to export constants.
2377 if (!isa<Instruction>(Val: V) && !isa<Argument>(Val: V)) return;
2378
2379 // Already exported?
2380 if (FuncInfo.isExportedInst(V)) return;
2381
2382 Register Reg = FuncInfo.InitializeRegForValue(V);
2383 CopyValueToVirtualRegister(V, Reg);
2384}
2385
2386bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2387 const BasicBlock *FromBB) {
2388 // The operands of the setcc have to be in this block. We don't know
2389 // how to export them from some other block.
2390 if (const Instruction *VI = dyn_cast<Instruction>(Val: V)) {
2391 // Can export from current BB.
2392 if (VI->getParent() == FromBB)
2393 return true;
2394
2395 // Is already exported, noop.
2396 return FuncInfo.isExportedInst(V);
2397 }
2398
2399 // If this is an argument, we can export it if the BB is the entry block or
2400 // if it is already exported.
2401 if (isa<Argument>(Val: V)) {
2402 if (FromBB->isEntryBlock())
2403 return true;
2404
2405 // Otherwise, can only export this if it is already exported.
2406 return FuncInfo.isExportedInst(V);
2407 }
2408
2409 // Otherwise, constants can always be exported.
2410 return true;
2411}
2412
2413/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2414BranchProbability
2415SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2416 const MachineBasicBlock *Dst) const {
2417 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2418 const BasicBlock *SrcBB = Src->getBasicBlock();
2419 const BasicBlock *DstBB = Dst->getBasicBlock();
2420 if (!BPI) {
2421 // If BPI is not available, set the default probability as 1 / N, where N is
2422 // the number of successors.
2423 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
2424 return BranchProbability(1, SuccSize);
2425 }
2426 return BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
2427}
2428
2429void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2430 MachineBasicBlock *Dst,
2431 BranchProbability Prob) {
2432 if (!FuncInfo.BPI)
2433 Src->addSuccessorWithoutProb(Succ: Dst);
2434 else {
2435 if (Prob.isUnknown())
2436 Prob = getEdgeProbability(Src, Dst);
2437 Src->addSuccessor(Succ: Dst, Prob);
2438 }
2439}
2440
2441static bool InBlock(const Value *V, const BasicBlock *BB) {
2442 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
2443 return I->getParent() == BB;
2444 return true;
2445}
2446
2447/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2448/// This function emits a branch and is used at the leaves of an OR or an
2449/// AND operator tree.
2450void
2451SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2452 MachineBasicBlock *TBB,
2453 MachineBasicBlock *FBB,
2454 MachineBasicBlock *CurBB,
2455 MachineBasicBlock *SwitchBB,
2456 BranchProbability TProb,
2457 BranchProbability FProb,
2458 bool InvertCond) {
2459 const BasicBlock *BB = CurBB->getBasicBlock();
2460
2461 // If the leaf of the tree is a comparison, merge the condition into
2462 // the caseblock.
2463 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
2464 // The operands of the cmp have to be in this block. We don't know
2465 // how to export them from some other block. If this is the first block
2466 // of the sequence, no exporting is needed.
2467 if (CurBB == SwitchBB ||
2468 (isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 0), FromBB: BB) &&
2469 isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 1), FromBB: BB))) {
2470 ISD::CondCode Condition;
2471 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
2472 ICmpInst::Predicate Pred =
2473 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2474 Condition = getICmpCondCode(Pred);
2475 } else {
2476 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
2477 FCmpInst::Predicate Pred =
2478 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2479 Condition = getFCmpCondCode(Pred);
2480 if (FC->hasNoNaNs() ||
2481 (isKnownNeverNaN(V: FC->getOperand(i_nocapture: 0),
2482 SQ: SimplifyQuery(DAG.getDataLayout(), FC)) &&
2483 isKnownNeverNaN(V: FC->getOperand(i_nocapture: 1),
2484 SQ: SimplifyQuery(DAG.getDataLayout(), FC))))
2485 Condition = getFCmpCodeWithoutNaN(CC: Condition);
2486 }
2487
2488 CaseBlock CB(Condition, BOp->getOperand(i_nocapture: 0), BOp->getOperand(i_nocapture: 1), nullptr,
2489 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2490 SL->SwitchCases.push_back(x: CB);
2491 return;
2492 }
2493 }
2494
2495 // Create a CaseBlock record representing this branch.
2496 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2497 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(Context&: *DAG.getContext()),
2498 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2499 SL->SwitchCases.push_back(x: CB);
2500}
2501
2502// Collect dependencies on V recursively. This is used for the cost analysis in
2503// `shouldKeepJumpConditionsTogether`.
2504static bool collectInstructionDeps(
2505 SmallMapVector<const Instruction *, bool, 8> *Deps, const Value *V,
2506 SmallMapVector<const Instruction *, bool, 8> *Necessary = nullptr,
2507 unsigned Depth = 0) {
2508 // Return false if we have an incomplete count.
2509 if (Depth >= SelectionDAG::MaxRecursionDepth)
2510 return false;
2511
2512 auto *I = dyn_cast<Instruction>(Val: V);
2513 if (I == nullptr)
2514 return true;
2515
2516 if (Necessary != nullptr) {
2517 // This instruction is necessary for the other side of the condition so
2518 // don't count it.
2519 if (Necessary->contains(Key: I))
2520 return true;
2521 }
2522
2523 // Already added this dep.
2524 if (!Deps->try_emplace(Key: I, Args: false).second)
2525 return true;
2526
2527 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2528 if (!collectInstructionDeps(Deps, V: I->getOperand(i: OpIdx), Necessary,
2529 Depth: Depth + 1))
2530 return false;
2531 return true;
2532}
2533
2534bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether(
2535 const FunctionLoweringInfo &FuncInfo, const CondBrInst &I,
2536 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2537 TargetLoweringBase::CondMergingParams Params) const {
2538 if (Params.BaseCost < 0)
2539 return false;
2540
2541 // Baseline cost.
2542 InstructionCost CostThresh = Params.BaseCost;
2543
2544 BranchProbabilityInfo *BPI = nullptr;
2545 if (Params.LikelyBias || Params.UnlikelyBias)
2546 BPI = FuncInfo.BPI;
2547 if (BPI != nullptr) {
2548 // See if we are either likely to get an early out or compute both lhs/rhs
2549 // of the condition.
2550 BasicBlock *IfFalse = I.getSuccessor(i: 0);
2551 BasicBlock *IfTrue = I.getSuccessor(i: 1);
2552
2553 std::optional<bool> Likely;
2554 if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfTrue))
2555 Likely = true;
2556 else if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfFalse))
2557 Likely = false;
2558
2559 if (Likely) {
2560 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2561 // Its likely we will have to compute both lhs and rhs of condition
2562 CostThresh += Params.LikelyBias;
2563 else {
2564 if (Params.UnlikelyBias < 0)
2565 return false;
2566 // Its likely we will get an early out.
2567 CostThresh -= Params.UnlikelyBias;
2568 }
2569 }
2570 }
2571
2572 if (CostThresh <= 0)
2573 return false;
2574
2575 // Collect "all" instructions that lhs condition is dependent on.
2576 // Use map for stable iteration (to avoid non-determanism of iteration of
2577 // SmallPtrSet). The `bool` value is just a dummy.
2578 SmallMapVector<const Instruction *, bool, 8> LhsDeps, RhsDeps;
2579 collectInstructionDeps(Deps: &LhsDeps, V: Lhs);
2580 // Collect "all" instructions that rhs condition is dependent on AND are
2581 // dependencies of lhs. This gives us an estimate on which instructions we
2582 // stand to save by splitting the condition.
2583 if (!collectInstructionDeps(Deps: &RhsDeps, V: Rhs, Necessary: &LhsDeps))
2584 return false;
2585 // Add the compare instruction itself unless its a dependency on the LHS.
2586 if (const auto *RhsI = dyn_cast<Instruction>(Val: Rhs))
2587 if (!LhsDeps.contains(Key: RhsI))
2588 RhsDeps.try_emplace(Key: RhsI, Args: false);
2589
2590 InstructionCost CostOfIncluding = 0;
2591 // See if this instruction will need to computed independently of whether RHS
2592 // is.
2593 Value *BrCond = I.getCondition();
2594 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2595 for (const auto *U : Ins->users()) {
2596 // If user is independent of RHS calculation we don't need to count it.
2597 if (auto *UIns = dyn_cast<Instruction>(Val: U))
2598 if (UIns != BrCond && !RhsDeps.contains(Key: UIns))
2599 return false;
2600 }
2601 return true;
2602 };
2603
2604 // Prune instructions from RHS Deps that are dependencies of unrelated
2605 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2606 // arbitrary and just meant to cap the how much time we spend in the pruning
2607 // loop. Its highly unlikely to come into affect.
2608 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2609 // Stop after a certain point. No incorrectness from including too many
2610 // instructions.
2611 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2612 const Instruction *ToDrop = nullptr;
2613 for (const auto &InsPair : RhsDeps) {
2614 if (!ShouldCountInsn(InsPair.first)) {
2615 ToDrop = InsPair.first;
2616 break;
2617 }
2618 }
2619 if (ToDrop == nullptr)
2620 break;
2621 RhsDeps.erase(Key: ToDrop);
2622 }
2623
2624 for (const auto &InsPair : RhsDeps) {
2625 // Finally accumulate latency that we can only attribute to computing the
2626 // RHS condition. Use latency because we are essentially trying to calculate
2627 // the cost of the dependency chain.
2628 // Possible TODO: We could try to estimate ILP and make this more precise.
2629 CostOfIncluding += TTI->getInstructionCost(
2630 U: InsPair.first, CostKind: TargetTransformInfo::TCK_Latency);
2631
2632 if (CostOfIncluding > CostThresh)
2633 return false;
2634 }
2635 return true;
2636}
2637
2638void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2639 MachineBasicBlock *TBB,
2640 MachineBasicBlock *FBB,
2641 MachineBasicBlock *CurBB,
2642 MachineBasicBlock *SwitchBB,
2643 Instruction::BinaryOps Opc,
2644 BranchProbability TProb,
2645 BranchProbability FProb,
2646 bool InvertCond) {
2647 // Skip over not part of the tree and remember to invert op and operands at
2648 // next level.
2649 Value *NotCond;
2650 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
2651 InBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
2652 FindMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2653 InvertCond: !InvertCond);
2654 return;
2655 }
2656
2657 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
2658 const Value *BOpOp0, *BOpOp1;
2659 // Compute the effective opcode for Cond, taking into account whether it needs
2660 // to be inverted, e.g.
2661 // and (not (or A, B)), C
2662 // gets lowered as
2663 // and (and (not A, not B), C)
2664 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2665 if (BOp) {
2666 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2667 ? Instruction::And
2668 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2669 ? Instruction::Or
2670 : (Instruction::BinaryOps)0);
2671 if (InvertCond) {
2672 if (BOpc == Instruction::And)
2673 BOpc = Instruction::Or;
2674 else if (BOpc == Instruction::Or)
2675 BOpc = Instruction::And;
2676 }
2677 }
2678
2679 // If this node is not part of the or/and tree, emit it as a branch.
2680 // Note that all nodes in the tree should have same opcode.
2681 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2682 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2683 !InBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
2684 !InBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
2685 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2686 TProb, FProb, InvertCond);
2687 return;
2688 }
2689
2690 // Create TmpBB after CurBB.
2691 MachineFunction::iterator BBI(CurBB);
2692 MachineFunction &MF = DAG.getMachineFunction();
2693 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
2694 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
2695
2696 if (Opc == Instruction::Or) {
2697 // Codegen X | Y as:
2698 // BB1:
2699 // jmp_if_X TBB
2700 // jmp TmpBB
2701 // TmpBB:
2702 // jmp_if_Y TBB
2703 // jmp FBB
2704 //
2705
2706 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2707 // The requirement is that
2708 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2709 // = TrueProb for original BB.
2710 // Assuming the original probabilities are A and B, one choice is to set
2711 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2712 // A/(1+B) and 2B/(1+B). This choice assumes that
2713 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2714 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2715 // TmpBB, but the math is more complicated.
2716
2717 auto NewTrueProb = TProb / 2;
2718 auto NewFalseProb = TProb / 2 + FProb;
2719 // Emit the LHS condition.
2720 FindMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2721 FProb: NewFalseProb, InvertCond);
2722
2723 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2724 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2725 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2726 // Emit the RHS condition into TmpBB.
2727 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2728 FProb: Probs[1], InvertCond);
2729 } else {
2730 assert(Opc == Instruction::And && "Unknown merge op!");
2731 // Codegen X & Y as:
2732 // BB1:
2733 // jmp_if_X TmpBB
2734 // jmp FBB
2735 // TmpBB:
2736 // jmp_if_Y TBB
2737 // jmp FBB
2738 //
2739 // This requires creation of TmpBB after CurBB.
2740
2741 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2742 // The requirement is that
2743 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2744 // = FalseProb for original BB.
2745 // Assuming the original probabilities are A and B, one choice is to set
2746 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2747 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2748 // TrueProb for BB1 * FalseProb for TmpBB.
2749
2750 auto NewTrueProb = TProb + FProb / 2;
2751 auto NewFalseProb = FProb / 2;
2752 // Emit the LHS condition.
2753 FindMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2754 FProb: NewFalseProb, InvertCond);
2755
2756 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2757 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2758 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2759 // Emit the RHS condition into TmpBB.
2760 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2761 FProb: Probs[1], InvertCond);
2762 }
2763}
2764
2765/// If the set of cases should be emitted as a series of branches, return true.
2766/// If we should emit this as a bunch of and/or'd together conditions, return
2767/// false.
2768bool
2769SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2770 if (Cases.size() != 2) return true;
2771
2772 // If this is two comparisons of the same values or'd or and'd together, they
2773 // will get folded into a single comparison, so don't emit two blocks.
2774 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2775 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2776 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2777 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2778 return false;
2779 }
2780
2781 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2782 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2783 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2784 Cases[0].CC == Cases[1].CC &&
2785 isa<Constant>(Val: Cases[0].CmpRHS) &&
2786 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
2787 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2788 return false;
2789 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2790 return false;
2791 }
2792
2793 return true;
2794}
2795
2796void SelectionDAGBuilder::visitUncondBr(const UncondBrInst &I) {
2797 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2798
2799 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2800
2801 // Update machine-CFG edges.
2802 BrMBB->addSuccessor(Succ: Succ0MBB);
2803
2804 // If this is not a fall-through branch or optimizations are switched off,
2805 // emit the branch.
2806 if (Succ0MBB != NextBlock(MBB: BrMBB) ||
2807 TM.getOptLevel() == CodeGenOptLevel::None) {
2808 auto Br = DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other, N1: getControlRoot(),
2809 N2: DAG.getBasicBlock(MBB: Succ0MBB));
2810 setValue(V: &I, NewN: Br);
2811 DAG.setRoot(Br);
2812 }
2813}
2814
2815void SelectionDAGBuilder::visitCondBr(const CondBrInst &I) {
2816 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2817
2818 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2819
2820 // If this condition is one of the special cases we handle, do special stuff
2821 // now.
2822 const Value *CondVal = I.getCondition();
2823 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 1));
2824
2825 // If this is a series of conditions that are or'd or and'd together, emit
2826 // this as a sequence of branches instead of setcc's with and/or operations.
2827 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2828 // unpredictable branches, and vector extracts because those jumps are likely
2829 // expensive for any target), this should improve performance.
2830 // For example, instead of something like:
2831 // cmp A, B
2832 // C = seteq
2833 // cmp D, E
2834 // F = setle
2835 // or C, F
2836 // jnz foo
2837 // Emit:
2838 // cmp A, B
2839 // je foo
2840 // cmp D, E
2841 // jle foo
2842 bool IsUnpredictable = I.hasMetadata(KindID: LLVMContext::MD_unpredictable);
2843 const Instruction *BOp = dyn_cast<Instruction>(Val: CondVal);
2844 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2845 BOp->hasOneUse() && !IsUnpredictable) {
2846 Value *Vec;
2847 const Value *BOp0, *BOp1;
2848 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2849 if (match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2850 Opcode = Instruction::And;
2851 else if (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2852 Opcode = Instruction::Or;
2853
2854 if (Opcode &&
2855 !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
2856 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value()))) &&
2857 !shouldKeepJumpConditionsTogether(
2858 FuncInfo, I, Opc: Opcode, Lhs: BOp0, Rhs: BOp1,
2859 Params: DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2860 Opcode, BOp0, BOp1))) {
2861 FindMergedConditions(Cond: BOp, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: BrMBB, SwitchBB: BrMBB, Opc: Opcode,
2862 TProb: getEdgeProbability(Src: BrMBB, Dst: Succ0MBB),
2863 FProb: getEdgeProbability(Src: BrMBB, Dst: Succ1MBB),
2864 /*InvertCond=*/false);
2865 // If the compares in later blocks need to use values not currently
2866 // exported from this block, export them now. This block should always
2867 // be the first entry.
2868 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2869
2870 // Allow some cases to be rejected.
2871 if (ShouldEmitAsBranches(Cases: SL->SwitchCases)) {
2872 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2873 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpLHS);
2874 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpRHS);
2875 }
2876
2877 // Emit the branch for this block.
2878 visitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: BrMBB);
2879 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
2880 return;
2881 }
2882
2883 // Okay, we decided not to do this, remove any inserted MBB's and clear
2884 // SwitchCases.
2885 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2886 FuncInfo.MF->erase(MBBI: SL->SwitchCases[i].ThisBB);
2887
2888 SL->SwitchCases.clear();
2889 }
2890 }
2891
2892 // Create a CaseBlock record representing this branch.
2893 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(Context&: *DAG.getContext()),
2894 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2895 BranchProbability::getUnknown(), BranchProbability::getUnknown(),
2896 IsUnpredictable);
2897
2898 // Use visitSwitchCase to actually insert the fast branch sequence for this
2899 // cond branch.
2900 visitSwitchCase(CB, SwitchBB: BrMBB);
2901}
2902
2903/// visitSwitchCase - Emits the necessary code to represent a single node in
2904/// the binary search tree resulting from lowering a switch instruction.
2905void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2906 MachineBasicBlock *SwitchBB) {
2907 SDValue Cond;
2908 SDValue CondLHS = getValue(V: CB.CmpLHS);
2909 SDLoc dl = CB.DL;
2910
2911 if (CB.CC == ISD::SETTRUE) {
2912 // Branch or fall through to TrueBB.
2913 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
2914 SwitchBB->normalizeSuccProbs();
2915 if (CB.TrueBB != NextBlock(MBB: SwitchBB)) {
2916 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: getControlRoot(),
2917 N2: DAG.getBasicBlock(MBB: CB.TrueBB)));
2918 }
2919 return;
2920 }
2921
2922 auto &TLI = DAG.getTargetLoweringInfo();
2923 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: CB.CmpLHS->getType());
2924
2925 // Build the setcc now.
2926 if (!CB.CmpMHS) {
2927 // Fold "(X == true)" to X and "(X == false)" to !X to
2928 // handle common cases produced by branch lowering.
2929 if (CB.CmpRHS == ConstantInt::getTrue(Context&: *DAG.getContext()) &&
2930 CB.CC == ISD::SETEQ)
2931 Cond = CondLHS;
2932 else if (CB.CmpRHS == ConstantInt::getFalse(Context&: *DAG.getContext()) &&
2933 CB.CC == ISD::SETEQ) {
2934 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: CondLHS.getValueType());
2935 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CondLHS.getValueType(), N1: CondLHS, N2: True);
2936 } else {
2937 SDValue CondRHS = getValue(V: CB.CmpRHS);
2938
2939 // If a pointer's DAG type is larger than its memory type then the DAG
2940 // values are zero-extended. This breaks signed comparisons so truncate
2941 // back to the underlying type before doing the compare.
2942 if (CondLHS.getValueType() != MemVT) {
2943 CondLHS = DAG.getPtrExtOrTrunc(Op: CondLHS, DL: getCurSDLoc(), VT: MemVT);
2944 CondRHS = DAG.getPtrExtOrTrunc(Op: CondRHS, DL: getCurSDLoc(), VT: MemVT);
2945 }
2946 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: CondLHS, RHS: CondRHS, Cond: CB.CC);
2947 }
2948 } else {
2949 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2950
2951 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
2952 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
2953
2954 SDValue CmpOp = getValue(V: CB.CmpMHS);
2955 EVT VT = CmpOp.getValueType();
2956
2957 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
2958 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: CmpOp, RHS: DAG.getConstant(Val: High, DL: dl, VT),
2959 Cond: ISD::SETLE);
2960 } else {
2961 SDValue SUB = DAG.getNode(Opcode: ISD::SUB, DL: dl,
2962 VT, N1: CmpOp, N2: DAG.getConstant(Val: Low, DL: dl, VT));
2963 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: SUB,
2964 RHS: DAG.getConstant(Val: High-Low, DL: dl, VT), Cond: ISD::SETULE);
2965 }
2966 }
2967
2968 // Update successor info
2969 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
2970 // TrueBB and FalseBB are always different unless the incoming IR is
2971 // degenerate. This only happens when running llc on weird IR.
2972 if (CB.TrueBB != CB.FalseBB)
2973 addSuccessorWithProb(Src: SwitchBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
2974 SwitchBB->normalizeSuccProbs();
2975
2976 // If the lhs block is the next block, invert the condition so that we can
2977 // fall through to the lhs instead of the rhs block.
2978 if (CB.TrueBB == NextBlock(MBB: SwitchBB)) {
2979 std::swap(a&: CB.TrueBB, b&: CB.FalseBB);
2980 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: Cond.getValueType());
2981 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: Cond.getValueType(), N1: Cond, N2: True);
2982 }
2983
2984 SDNodeFlags Flags;
2985 Flags.setUnpredictable(CB.IsUnpredictable);
2986 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: getControlRoot(),
2987 N2: Cond, N3: DAG.getBasicBlock(MBB: CB.TrueBB), Flags);
2988
2989 setValue(V: CurInst, NewN: BrCond);
2990
2991 // Insert the false branch. Do this even if it's a fall through branch,
2992 // this makes it easier to do DAG optimizations which require inverting
2993 // the branch condition.
2994 BrCond = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrCond,
2995 N2: DAG.getBasicBlock(MBB: CB.FalseBB));
2996
2997 DAG.setRoot(BrCond);
2998}
2999
3000/// visitJumpTable - Emit JumpTable node in the current MBB
3001void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
3002 // Emit the code for the jump table
3003 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3004 assert(JT.Reg && "Should lower JT Header first!");
3005 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DL: DAG.getDataLayout());
3006 SDValue Index = DAG.getCopyFromReg(Chain: getControlRoot(), dl: *JT.SL, Reg: JT.Reg, VT: PTy);
3007 SDValue Table = DAG.getJumpTable(JTI: JT.JTI, VT: PTy);
3008 SDValue BrJumpTable = DAG.getNode(Opcode: ISD::BR_JT, DL: *JT.SL, VT: MVT::Other,
3009 N1: Index.getValue(R: 1), N2: Table, N3: Index);
3010 DAG.setRoot(BrJumpTable);
3011}
3012
3013/// visitJumpTableHeader - This function emits necessary code to produce index
3014/// in the JumpTable from switch case.
3015void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
3016 JumpTableHeader &JTH,
3017 MachineBasicBlock *SwitchBB) {
3018 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3019 const SDLoc &dl = *JT.SL;
3020
3021 // Subtract the lowest switch case value from the value being switched on.
3022 SDValue SwitchOp = getValue(V: JTH.SValue);
3023 EVT VT = SwitchOp.getValueType();
3024 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp,
3025 N2: DAG.getConstant(Val: JTH.First, DL: dl, VT));
3026
3027 // The SDNode we just created, which holds the value being switched on minus
3028 // the smallest case value, needs to be copied to a virtual register so it
3029 // can be used as an index into the jump table in a subsequent basic block.
3030 // This value may be smaller or larger than the target's pointer type, and
3031 // therefore require extension or truncating.
3032 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3033 SwitchOp =
3034 DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
3035
3036 Register JumpTableReg =
3037 FuncInfo.CreateReg(VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
3038 SDValue CopyTo =
3039 DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: JumpTableReg, N: SwitchOp);
3040 JT.Reg = JumpTableReg;
3041
3042 if (!JTH.FallthroughUnreachable) {
3043 // Emit the range check for the jump table, and branch to the default block
3044 // for the switch statement if the value being switched on exceeds the
3045 // largest case in the switch.
3046 SDValue CMP = DAG.getSetCC(
3047 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3048 VT: Sub.getValueType()),
3049 LHS: Sub, RHS: DAG.getConstant(Val: JTH.Last - JTH.First, DL: dl, VT), Cond: ISD::SETUGT);
3050
3051 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl,
3052 VT: MVT::Other, N1: CopyTo, N2: CMP,
3053 N3: DAG.getBasicBlock(MBB: JT.Default));
3054
3055 // Avoid emitting unnecessary branches to the next block.
3056 if (JT.MBB != NextBlock(MBB: SwitchBB))
3057 BrCond = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrCond,
3058 N2: DAG.getBasicBlock(MBB: JT.MBB));
3059
3060 DAG.setRoot(BrCond);
3061 } else {
3062 // Avoid emitting unnecessary branches to the next block.
3063 if (JT.MBB != NextBlock(MBB: SwitchBB))
3064 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: CopyTo,
3065 N2: DAG.getBasicBlock(MBB: JT.MBB)));
3066 else
3067 DAG.setRoot(CopyTo);
3068 }
3069}
3070
3071/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3072/// variable if there exists one.
3073static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
3074 SDValue &Chain) {
3075 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3076 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
3077 EVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout());
3078 MachineFunction &MF = DAG.getMachineFunction();
3079 Value *Global =
3080 TLI.getSDagStackGuard(M: *MF.getFunction().getParent(), Libcalls: DAG.getLibcalls());
3081 MachineSDNode *Node =
3082 DAG.getMachineNode(Opcode: TargetOpcode::LOAD_STACK_GUARD, dl: DL, VT: PtrTy, Op1: Chain);
3083 if (Global) {
3084 MachinePointerInfo MPInfo(Global);
3085 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
3086 MachineMemOperand::MODereferenceable;
3087 MachineMemOperand *MemRef = MF.getMachineMemOperand(
3088 PtrInfo: MPInfo, F: Flags, Size: PtrTy.getSizeInBits() / 8, BaseAlignment: DAG.getEVTAlign(MemoryVT: PtrTy));
3089 DAG.setNodeMemRefs(N: Node, NewMemRefs: {MemRef});
3090 }
3091 if (PtrTy != PtrMemTy)
3092 return DAG.getPtrExtOrTrunc(Op: SDValue(Node, 0), DL, VT: PtrMemTy);
3093 return SDValue(Node, 0);
3094}
3095
3096/// Codegen a new tail for a stack protector check ParentMBB which has had its
3097/// tail spliced into a stack protector check success bb.
3098///
3099/// For a high level explanation of how this fits into the stack protector
3100/// generation see the comment on the declaration of class
3101/// StackProtectorDescriptor.
3102void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
3103 MachineBasicBlock *ParentBB) {
3104
3105 // First create the loads to the guard/stack slot for the comparison.
3106 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3107 auto &DL = DAG.getDataLayout();
3108 EVT PtrTy = TLI.getFrameIndexTy(DL);
3109 EVT PtrMemTy = TLI.getPointerMemTy(DL, AS: DL.getAllocaAddrSpace());
3110
3111 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3112 int FI = MFI.getStackProtectorIndex();
3113
3114 SDValue Guard;
3115 SDLoc dl = getCurSDLoc();
3116 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3117 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3118 Align Align = DL.getPrefTypeAlign(
3119 Ty: PointerType::get(C&: M.getContext(), AddressSpace: DL.getAllocaAddrSpace()));
3120
3121 // Generate code to load the content of the guard slot.
3122 SDValue GuardVal = DAG.getLoad(
3123 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3124 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3125 MMOFlags: MachineMemOperand::MOVolatile);
3126
3127 if (TLI.useStackGuardXorFP())
3128 GuardVal = TLI.emitStackGuardXorFP(DAG, Val: GuardVal, DL: dl);
3129
3130 // If we're using function-based instrumentation, call the guard check
3131 // function
3132 if (SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3133 // Get the guard check function from the target and verify it exists since
3134 // we're using function-based instrumentation
3135 const Function *GuardCheckFn =
3136 TLI.getSSPStackGuardCheck(M, Libcalls: DAG.getLibcalls());
3137 assert(GuardCheckFn && "Guard check function is null");
3138
3139 // The target provides a guard check function to validate the guard value.
3140 // Generate a call to that function with the content of the guard slot as
3141 // argument.
3142 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3143 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3144
3145 TargetLowering::ArgListTy Args;
3146 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(i: 0));
3147 if (GuardCheckFn->hasParamAttribute(ArgNo: 0, Kind: Attribute::AttrKind::InReg))
3148 Entry.IsInReg = true;
3149 Args.push_back(x: Entry);
3150
3151 TargetLowering::CallLoweringInfo CLI(DAG);
3152 CLI.setDebugLoc(getCurSDLoc())
3153 .setChain(DAG.getEntryNode())
3154 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3155 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3156
3157 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3158 DAG.setRoot(Result.second);
3159 return;
3160 }
3161
3162 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3163 // Otherwise, emit a volatile load to retrieve the stack guard value.
3164 SDValue Chain = DAG.getEntryNode();
3165 if (TLI.useLoadStackGuardNode(M)) {
3166 Guard = getLoadStackGuard(DAG, DL: dl, Chain);
3167 } else {
3168 if (const Value *IRGuard = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls())) {
3169 SDValue GuardPtr = getValue(V: IRGuard);
3170 Guard = DAG.getLoad(VT: PtrMemTy, dl, Chain, Ptr: GuardPtr,
3171 PtrInfo: MachinePointerInfo(IRGuard, 0), Alignment: Align,
3172 MMOFlags: MachineMemOperand::MOVolatile);
3173 } else {
3174 LLVMContext &Ctx = *DAG.getContext();
3175 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
3176 Guard = DAG.getPOISON(VT: PtrMemTy);
3177 }
3178 }
3179
3180 // Perform the comparison via a getsetcc.
3181 SDValue Cmp = DAG.getSetCC(
3182 DL: dl, VT: TLI.getSetCCResultType(DL, Context&: *DAG.getContext(), VT: Guard.getValueType()),
3183 LHS: Guard, RHS: GuardVal, Cond: ISD::SETNE);
3184
3185 // If the guard/stackslot do not equal, branch to failure MBB.
3186 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: getControlRoot(),
3187 N2: Cmp, N3: DAG.getBasicBlock(MBB: SPD.getFailureMBB()));
3188 // Otherwise branch to success MBB.
3189 SDValue Br = DAG.getNode(Opcode: ISD::BR, DL: dl,
3190 VT: MVT::Other, N1: BrCond,
3191 N2: DAG.getBasicBlock(MBB: SPD.getSuccessMBB()));
3192
3193 DAG.setRoot(Br);
3194}
3195
3196/// Codegen the failure basic block for a stack protector check.
3197///
3198/// A failure stack protector machine basic block consists simply of a call to
3199/// __stack_chk_fail().
3200///
3201/// For a high level explanation of how this fits into the stack protector
3202/// generation see the comment on the declaration of class
3203/// StackProtectorDescriptor.
3204void SelectionDAGBuilder::visitSPDescriptorFailure(
3205 StackProtectorDescriptor &SPD) {
3206
3207 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3208 MachineBasicBlock *ParentBB = SPD.getParentMBB();
3209 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3210 SDValue Chain;
3211
3212 // For -Oz builds with a guard check function, we use function-based
3213 // instrumentation. Otherwise, if we have a guard check function, we call it
3214 // in the failure block.
3215 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M, Libcalls: DAG.getLibcalls());
3216 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3217 // First create the loads to the guard/stack slot for the comparison.
3218 auto &DL = DAG.getDataLayout();
3219 EVT PtrTy = TLI.getFrameIndexTy(DL);
3220 EVT PtrMemTy = TLI.getPointerMemTy(DL, AS: DL.getAllocaAddrSpace());
3221
3222 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3223 int FI = MFI.getStackProtectorIndex();
3224
3225 SDLoc dl = getCurSDLoc();
3226 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3227 Align Align = DL.getPrefTypeAlign(
3228 Ty: PointerType::get(C&: M.getContext(), AddressSpace: DL.getAllocaAddrSpace()));
3229
3230 // Generate code to load the content of the guard slot.
3231 SDValue GuardVal = DAG.getLoad(
3232 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3233 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3234 MMOFlags: MachineMemOperand::MOVolatile);
3235
3236 if (TLI.useStackGuardXorFP())
3237 GuardVal = TLI.emitStackGuardXorFP(DAG, Val: GuardVal, DL: dl);
3238
3239 // The target provides a guard check function to validate the guard value.
3240 // Generate a call to that function with the content of the guard slot as
3241 // argument.
3242 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3243 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3244
3245 TargetLowering::ArgListTy Args;
3246 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(i: 0));
3247 if (GuardCheckFn->hasParamAttribute(ArgNo: 0, Kind: Attribute::AttrKind::InReg))
3248 Entry.IsInReg = true;
3249 Args.push_back(x: Entry);
3250
3251 TargetLowering::CallLoweringInfo CLI(DAG);
3252 CLI.setDebugLoc(getCurSDLoc())
3253 .setChain(DAG.getEntryNode())
3254 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3255 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3256
3257 Chain = TLI.LowerCallTo(CLI).second;
3258 } else {
3259 TargetLowering::MakeLibCallOptions CallOptions;
3260 CallOptions.setDiscardResult(true);
3261 Chain = TLI.makeLibCall(DAG, LC: RTLIB::STACKPROTECTOR_CHECK_FAIL, RetVT: MVT::isVoid,
3262 Ops: {}, CallOptions, dl: getCurSDLoc())
3263 .second;
3264 }
3265
3266 // Emit a trap instruction if we are required to do so.
3267 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3268 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3269 Chain = DAG.getNode(Opcode: ISD::TRAP, DL: getCurSDLoc(), VT: MVT::Other, Operand: Chain);
3270
3271 DAG.setRoot(Chain);
3272}
3273
3274/// visitBitTestHeader - This function emits necessary code to produce value
3275/// suitable for "bit tests"
3276void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
3277 MachineBasicBlock *SwitchBB) {
3278 SDLoc dl = getCurSDLoc();
3279
3280 // Subtract the minimum value.
3281 SDValue SwitchOp = getValue(V: B.SValue);
3282 EVT VT = SwitchOp.getValueType();
3283 SDValue RangeSub =
3284 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp, N2: DAG.getConstant(Val: B.First, DL: dl, VT));
3285
3286 // Determine the type of the test operands.
3287 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3288 bool UsePtrType = false;
3289 if (!TLI.isTypeLegal(VT)) {
3290 UsePtrType = true;
3291 } else {
3292 for (const BitTestCase &Case : B.Cases)
3293 if (!isUIntN(N: VT.getSizeInBits(), x: Case.Mask)) {
3294 // Switch table case range are encoded into series of masks.
3295 // Just use pointer type, it's guaranteed to fit.
3296 UsePtrType = true;
3297 break;
3298 }
3299 }
3300 SDValue Sub = RangeSub;
3301 if (UsePtrType) {
3302 VT = TLI.getPointerTy(DL: DAG.getDataLayout());
3303 Sub = DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT);
3304 }
3305
3306 B.RegVT = VT.getSimpleVT();
3307 B.Reg = FuncInfo.CreateReg(VT: B.RegVT);
3308 SDValue CopyTo = DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: B.Reg, N: Sub);
3309
3310 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3311
3312 if (!B.FallthroughUnreachable)
3313 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
3314 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
3315 SwitchBB->normalizeSuccProbs();
3316
3317 SDValue Root = CopyTo;
3318 if (!B.FallthroughUnreachable) {
3319 // Conditional branch to the default block.
3320 SDValue RangeCmp = DAG.getSetCC(DL: dl,
3321 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3322 VT: RangeSub.getValueType()),
3323 LHS: RangeSub, RHS: DAG.getConstant(Val: B.Range, DL: dl, VT: RangeSub.getValueType()),
3324 Cond: ISD::SETUGT);
3325
3326 Root = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: Root, N2: RangeCmp,
3327 N3: DAG.getBasicBlock(MBB: B.Default));
3328 }
3329
3330 // Avoid emitting unnecessary branches to the next block.
3331 if (MBB != NextBlock(MBB: SwitchBB))
3332 Root = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: Root, N2: DAG.getBasicBlock(MBB));
3333
3334 DAG.setRoot(Root);
3335}
3336
3337/// visitBitTestCase - this function produces one "bit test"
3338void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
3339 MachineBasicBlock *NextMBB,
3340 BranchProbability BranchProbToNext,
3341 Register Reg, BitTestCase &B,
3342 MachineBasicBlock *SwitchBB) {
3343 SDLoc dl = getCurSDLoc();
3344 MVT VT = BB.RegVT;
3345 SDValue ShiftOp = DAG.getCopyFromReg(Chain: getControlRoot(), dl, Reg, VT);
3346 SDValue Cmp;
3347 unsigned PopCount = llvm::popcount(Value: B.Mask);
3348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3349 if (PopCount == 1) {
3350 // Testing for a single bit; just compare the shift count with what it
3351 // would need to be to shift a 1 bit in that position.
3352 Cmp = DAG.getSetCC(
3353 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3354 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_zero(Val: B.Mask), DL: dl, VT),
3355 Cond: ISD::SETEQ);
3356 } else if (PopCount == BB.Range) {
3357 // There is only one zero bit in the range, test for it directly.
3358 Cmp = DAG.getSetCC(
3359 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3360 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_one(Value: B.Mask), DL: dl, VT), Cond: ISD::SETNE);
3361 } else {
3362 // Make desired shift
3363 SDValue SwitchVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT,
3364 N1: DAG.getConstant(Val: 1, DL: dl, VT), N2: ShiftOp);
3365
3366 // Emit bit tests and jumps
3367 SDValue AndOp = DAG.getNode(Opcode: ISD::AND, DL: dl,
3368 VT, N1: SwitchVal, N2: DAG.getConstant(Val: B.Mask, DL: dl, VT));
3369 Cmp = DAG.getSetCC(
3370 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3371 LHS: AndOp, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
3372 }
3373
3374 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3375 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
3376 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3377 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
3378 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3379 // one as they are relative probabilities (and thus work more like weights),
3380 // and hence we need to normalize them to let the sum of them become one.
3381 SwitchBB->normalizeSuccProbs();
3382
3383 SDValue BrAnd = DAG.getNode(Opcode: ISD::BRCOND, DL: dl,
3384 VT: MVT::Other, N1: getControlRoot(),
3385 N2: Cmp, N3: DAG.getBasicBlock(MBB: B.TargetBB));
3386
3387 // Avoid emitting unnecessary branches to the next block.
3388 if (NextMBB != NextBlock(MBB: SwitchBB))
3389 BrAnd = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrAnd,
3390 N2: DAG.getBasicBlock(MBB: NextMBB));
3391
3392 DAG.setRoot(BrAnd);
3393}
3394
3395void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3396 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3397
3398 // Retrieve successors. Look through artificial IR level blocks like
3399 // catchswitch for successors.
3400 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
3401 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3402 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(BB: EHPadBB);
3403
3404 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3405 // have to do anything here to lower funclet bundles.
3406 failForInvalidBundles(I, Name: "invokes",
3407 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3408 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3409 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3410 LLVMContext::OB_clang_arc_attachedcall,
3411 LLVMContext::OB_kcfi});
3412
3413 const Value *Callee(I.getCalledOperand());
3414 const Function *Fn = dyn_cast<Function>(Val: Callee);
3415 if (isa<InlineAsm>(Val: Callee))
3416 visitInlineAsm(Call: I, EHPadBB);
3417 else if (Fn && Fn->isIntrinsic()) {
3418 switch (Fn->getIntrinsicID()) {
3419 default:
3420 llvm_unreachable("Cannot invoke this intrinsic");
3421 case Intrinsic::donothing:
3422 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3423 case Intrinsic::seh_try_begin:
3424 case Intrinsic::seh_scope_begin:
3425 case Intrinsic::seh_try_end:
3426 case Intrinsic::seh_scope_end:
3427 if (EHPadMBB)
3428 // a block referenced by EH table
3429 // so dtor-funclet not removed by opts
3430 EHPadMBB->setMachineBlockAddressTaken();
3431 break;
3432 case Intrinsic::experimental_patchpoint_void:
3433 case Intrinsic::experimental_patchpoint:
3434 visitPatchpoint(CB: I, EHPadBB);
3435 break;
3436 case Intrinsic::experimental_gc_statepoint:
3437 LowerStatepoint(I: cast<GCStatepointInst>(Val: I), EHPadBB);
3438 break;
3439 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3440 // but these intrinsics are special because they can be invoked, so we
3441 // manually lower it to a DAG node here.
3442 case Intrinsic::wasm_throw: {
3443 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3444 std::array<SDValue, 4> Ops = {
3445 getControlRoot(), // inchain for the terminator node
3446 DAG.getTargetConstant(Val: Intrinsic::wasm_throw, DL: getCurSDLoc(),
3447 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3448 getValue(V: I.getArgOperand(i: 0)), // tag
3449 getValue(V: I.getArgOperand(i: 1)) // thrown value
3450 };
3451 SDVTList VTs = DAG.getVTList(VTs: ArrayRef<EVT>({MVT::Other})); // outchain
3452 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3453 break;
3454 }
3455 case Intrinsic::wasm_rethrow: {
3456 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3457 std::array<SDValue, 2> Ops = {
3458 getControlRoot(), // inchain for the terminator node
3459 DAG.getTargetConstant(Val: Intrinsic::wasm_rethrow, DL: getCurSDLoc(),
3460 VT: TLI.getPointerTy(DL: DAG.getDataLayout()))};
3461 SDVTList VTs = DAG.getVTList(VTs: ArrayRef<EVT>({MVT::Other})); // outchain
3462 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3463 break;
3464 }
3465 }
3466 } else if (I.hasDeoptState()) {
3467 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3468 // Eventually we will support lowering the @llvm.experimental.deoptimize
3469 // intrinsic, and right now there are no plans to support other intrinsics
3470 // with deopt state.
3471 LowerCallSiteWithDeoptBundle(Call: &I, Callee: getValue(V: Callee), EHPadBB);
3472 } else if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
3473 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), EHPadBB);
3474 } else {
3475 LowerCallTo(CB: I, Callee: getValue(V: Callee), IsTailCall: false, IsMustTailCall: false, EHPadBB);
3476 }
3477
3478 // If the value of the invoke is used outside of its defining block, make it
3479 // available as a virtual register.
3480 // We already took care of the exported value for the statepoint instruction
3481 // during call to the LowerStatepoint.
3482 if (!isa<GCStatepointInst>(Val: I)) {
3483 CopyToExportRegsIfNeeded(V: &I);
3484 }
3485
3486 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3487 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3488 BranchProbability EHPadBBProb =
3489 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3490 : BranchProbability::getZero();
3491 findUnwindDestinations(FuncInfo, EHPadBB, Prob: EHPadBBProb, UnwindDests);
3492
3493 // Update successor info.
3494 addSuccessorWithProb(Src: InvokeMBB, Dst: Return);
3495 for (auto &UnwindDest : UnwindDests) {
3496 UnwindDest.first->setIsEHPad();
3497 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3498 }
3499 InvokeMBB->normalizeSuccProbs();
3500
3501 // Drop into normal successor.
3502 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other, N1: getControlRoot(),
3503 N2: DAG.getBasicBlock(MBB: Return)));
3504}
3505
3506/// The intrinsics currently supported by callbr are implicit control flow
3507/// intrinsics such as amdgcn.kill.
3508/// - they should be called (no "dontcall-" attributes)
3509/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
3510/// - they do not need custom argument handling (no
3511/// TLI.CollectTargetIntrinsicOperands())
3512void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
3513#ifndef NDEBUG
3514 SmallVector<TargetLowering::IntrinsicInfo, 2> Infos;
3515 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3516 Infos, I, DAG.getMachineFunction(), I.getIntrinsicID());
3517 assert(Infos.empty() && "Intrinsic touches memory");
3518#endif
3519
3520 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
3521
3522 SmallVector<SDValue, 8> Ops =
3523 getTargetIntrinsicOperands(I, HasChain, OnlyLoad);
3524 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
3525
3526 // Create the node.
3527 SDValue Result =
3528 getTargetNonMemIntrinsicNode(IntrinsicVT: *I.getType(), HasChain, Ops, VTs);
3529 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
3530
3531 setValue(V: &I, NewN: Result);
3532}
3533
3534void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3535 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3536
3537 if (I.isInlineAsm()) {
3538 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3539 // have to do anything here to lower funclet bundles.
3540 failForInvalidBundles(I, Name: "callbrs",
3541 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_funclet});
3542 visitInlineAsm(Call: I);
3543 } else {
3544 assert(!I.hasOperandBundles() &&
3545 "Can't have operand bundles for intrinsics");
3546 visitCallBrIntrinsic(I);
3547 }
3548 CopyToExportRegsIfNeeded(V: &I);
3549
3550 // Retrieve successors.
3551 SmallPtrSet<BasicBlock *, 8> Dests;
3552 Dests.insert(Ptr: I.getDefaultDest());
3553 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getDefaultDest());
3554
3555 // Update successor info.
3556 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3557 // TODO: For most of the cases where there is an intrinsic callbr, we're
3558 // having exactly one indirect target, which will be unreachable. As soon as
3559 // this changes, we might need to enhance
3560 // Target->setIsInlineAsmBrIndirectTarget or add something similar for
3561 // intrinsic indirect branches.
3562 if (I.isInlineAsm()) {
3563 for (BasicBlock *Dest : I.getIndirectDests()) {
3564 MachineBasicBlock *Target = FuncInfo.getMBB(BB: Dest);
3565 Target->setIsInlineAsmBrIndirectTarget();
3566 // If we introduce a type of asm goto statement that is permitted to use
3567 // an indirect call instruction to jump to its labels, then we should add
3568 // a call to Target->setMachineBlockAddressTaken() here, to mark the
3569 // target block as requiring a BTI.
3570
3571 Target->setLabelMustBeEmitted();
3572 // Don't add duplicate machine successors.
3573 if (Dests.insert(Ptr: Dest).second)
3574 addSuccessorWithProb(Src: CallBrMBB, Dst: Target, Prob: BranchProbability::getZero());
3575 }
3576 }
3577 CallBrMBB->normalizeSuccProbs();
3578
3579 // Drop into default successor.
3580 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(),
3581 VT: MVT::Other, N1: getControlRoot(),
3582 N2: DAG.getBasicBlock(MBB: Return)));
3583}
3584
3585void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3586 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3587}
3588
3589void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3590 assert(FuncInfo.MBB->isEHPad() &&
3591 "Call to landingpad not in landing pad!");
3592
3593 // If there aren't registers to copy the values into (e.g., during SjLj
3594 // exceptions), then don't bother to create these DAG nodes.
3595 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3596 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3597 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3598 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3599 return;
3600
3601 // If landingpad's return type is token type, we don't create DAG nodes
3602 // for its exception pointer and selector value. The extraction of exception
3603 // pointer or selector value from token type landingpads is not currently
3604 // supported.
3605 if (LP.getType()->isTokenTy())
3606 return;
3607
3608 SmallVector<EVT, 2> ValueVTs;
3609 SDLoc dl = getCurSDLoc();
3610 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: LP.getType(), ValueVTs);
3611 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3612
3613 // Get the two live-in registers as SDValues. The physregs have already been
3614 // copied into virtual registers.
3615 SDValue Ops[2];
3616 if (FuncInfo.ExceptionPointerVirtReg) {
3617 Ops[0] = DAG.getZExtOrTrunc(
3618 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3619 Reg: FuncInfo.ExceptionPointerVirtReg,
3620 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3621 DL: dl, VT: ValueVTs[0]);
3622 } else {
3623 Ops[0] = DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3624 }
3625 Ops[1] = DAG.getZExtOrTrunc(
3626 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3627 Reg: FuncInfo.ExceptionSelectorVirtReg,
3628 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3629 DL: dl, VT: ValueVTs[1]);
3630
3631 // Merge into one.
3632 SDValue Res = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
3633 VTList: DAG.getVTList(VTs: ValueVTs), Ops);
3634 setValue(V: &LP, NewN: Res);
3635}
3636
3637void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3638 MachineBasicBlock *Last) {
3639 // Update JTCases.
3640 for (JumpTableBlock &JTB : SL->JTCases)
3641 if (JTB.first.HeaderBB == First)
3642 JTB.first.HeaderBB = Last;
3643
3644 // Update BitTestCases.
3645 for (BitTestBlock &BTB : SL->BitTestCases)
3646 if (BTB.Parent == First)
3647 BTB.Parent = Last;
3648}
3649
3650void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3651 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3652
3653 // Update machine-CFG edges with unique successors.
3654 SmallPtrSet<BasicBlock *, 32> Done;
3655 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3656 BasicBlock *BB = I.getSuccessor(i);
3657 bool Inserted = Done.insert(Ptr: BB).second;
3658 if (!Inserted)
3659 continue;
3660
3661 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3662 addSuccessorWithProb(Src: IndirectBrMBB, Dst: Succ);
3663 }
3664 IndirectBrMBB->normalizeSuccProbs();
3665
3666 DAG.setRoot(DAG.getNode(Opcode: ISD::BRIND, DL: getCurSDLoc(),
3667 VT: MVT::Other, N1: getControlRoot(),
3668 N2: getValue(V: I.getAddress())));
3669}
3670
3671void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3672 if (!I.shouldLowerToTrap(TrapUnreachable: DAG.getTarget().Options.TrapUnreachable,
3673 NoTrapAfterNoreturn: DAG.getTarget().Options.NoTrapAfterNoreturn))
3674 return;
3675
3676 DAG.setRoot(DAG.getNode(Opcode: ISD::TRAP, DL: getCurSDLoc(), VT: MVT::Other, Operand: DAG.getRoot()));
3677}
3678
3679void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3680 SDNodeFlags Flags;
3681 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3682 Flags.copyFMF(FPMO: *FPOp);
3683
3684 SDValue Op = getValue(V: I.getOperand(i: 0));
3685 SDValue UnNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op.getValueType(),
3686 Operand: Op, Flags);
3687 setValue(V: &I, NewN: UnNodeValue);
3688}
3689
3690void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3691 SDNodeFlags Flags;
3692 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(Val: &I)) {
3693 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3694 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3695 }
3696 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(Val: &I))
3697 Flags.setExact(ExactOp->isExact());
3698 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(Val: &I))
3699 Flags.setDisjoint(DisjointOp->isDisjoint());
3700 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3701 Flags.copyFMF(FPMO: *FPOp);
3702
3703 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3704 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3705 SDValue BinNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(),
3706 N1: Op1, N2: Op2, Flags);
3707 setValue(V: &I, NewN: BinNodeValue);
3708}
3709
3710void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3711 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3712 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3713
3714 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3715 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
3716
3717 // Coerce the shift amount to the right type if we can. This exposes the
3718 // truncate or zext to optimization early.
3719 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3720 assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3721 "Unexpected shift type");
3722 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
3723 }
3724
3725 bool nuw = false;
3726 bool nsw = false;
3727 bool exact = false;
3728
3729 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3730
3731 if (const OverflowingBinaryOperator *OFBinOp =
3732 dyn_cast<const OverflowingBinaryOperator>(Val: &I)) {
3733 nuw = OFBinOp->hasNoUnsignedWrap();
3734 nsw = OFBinOp->hasNoSignedWrap();
3735 }
3736 if (const PossiblyExactOperator *ExactOp =
3737 dyn_cast<const PossiblyExactOperator>(Val: &I))
3738 exact = ExactOp->isExact();
3739 }
3740 SDNodeFlags Flags;
3741 Flags.setExact(exact);
3742 Flags.setNoSignedWrap(nsw);
3743 Flags.setNoUnsignedWrap(nuw);
3744 SDValue Res = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1, N2: Op2,
3745 Flags);
3746 setValue(V: &I, NewN: Res);
3747}
3748
3749void SelectionDAGBuilder::visitSDiv(const User &I) {
3750 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3751 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3752
3753 SDNodeFlags Flags;
3754 Flags.setExact(isa<PossiblyExactOperator>(Val: &I) &&
3755 cast<PossiblyExactOperator>(Val: &I)->isExact());
3756 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SDIV, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1,
3757 N2: Op2, Flags));
3758}
3759
3760void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3761 ICmpInst::Predicate predicate = I.getPredicate();
3762 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3763 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3764 ISD::CondCode Opcode = getICmpCondCode(Pred: predicate);
3765
3766 auto &TLI = DAG.getTargetLoweringInfo();
3767 EVT MemVT =
3768 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
3769
3770 // If a pointer's DAG type is larger than its memory type then the DAG values
3771 // are zero-extended. This breaks signed comparisons so truncate back to the
3772 // underlying type before doing the compare.
3773 if (Op1.getValueType() != MemVT) {
3774 Op1 = DAG.getPtrExtOrTrunc(Op: Op1, DL: getCurSDLoc(), VT: MemVT);
3775 Op2 = DAG.getPtrExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: MemVT);
3776 }
3777
3778 SDNodeFlags Flags;
3779 Flags.setSameSign(I.hasSameSign());
3780 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3781
3782 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3783 Ty: I.getType());
3784 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Opcode));
3785}
3786
3787void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3788 FCmpInst::Predicate predicate = I.getPredicate();
3789 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3790 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3791
3792 ISD::CondCode Condition = getFCmpCondCode(Pred: predicate);
3793 auto *FPMO = cast<FPMathOperator>(Val: &I);
3794 if (FPMO->hasNoNaNs() ||
3795 (DAG.isKnownNeverNaN(Op: Op1) && DAG.isKnownNeverNaN(Op: Op2)))
3796 Condition = getFCmpCodeWithoutNaN(CC: Condition);
3797
3798 SDNodeFlags Flags;
3799 Flags.copyFMF(FPMO: *FPMO);
3800 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3801
3802 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3803 Ty: I.getType());
3804 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Condition,
3805 /*Chian=*/Chain: {}, /*IsSignaling=*/false, Flags));
3806}
3807
3808// Check if the condition of the select has one use or two users that are both
3809// selects with the same condition.
3810static bool hasOnlySelectUsers(const Value *Cond) {
3811 return llvm::all_of(Range: Cond->users(), P: [](const Value *V) {
3812 return isa<SelectInst>(Val: V);
3813 });
3814}
3815
3816void SelectionDAGBuilder::visitSelect(const User &I) {
3817 SmallVector<EVT, 4> ValueVTs;
3818 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
3819 ValueVTs);
3820 unsigned NumValues = ValueVTs.size();
3821 if (NumValues == 0) return;
3822
3823 SmallVector<SDValue, 4> Values(NumValues);
3824 SDValue Cond = getValue(V: I.getOperand(i: 0));
3825 SDValue LHSVal = getValue(V: I.getOperand(i: 1));
3826 SDValue RHSVal = getValue(V: I.getOperand(i: 2));
3827 SmallVector<SDValue, 1> BaseOps(1, Cond);
3828 ISD::NodeType OpCode =
3829 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3830
3831 bool IsUnaryAbs = false;
3832 bool Negate = false;
3833
3834 SDNodeFlags Flags;
3835 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3836 Flags.copyFMF(FPMO: *FPOp);
3837
3838 Flags.setUnpredictable(
3839 cast<SelectInst>(Val: I).getMetadata(KindID: LLVMContext::MD_unpredictable));
3840
3841 // Min/max matching is only viable if all output VTs are the same.
3842 if (all_equal(Range&: ValueVTs)) {
3843 EVT VT = ValueVTs[0];
3844 LLVMContext &Ctx = *DAG.getContext();
3845 auto &TLI = DAG.getTargetLoweringInfo();
3846
3847 // We care about the legality of the operation after it has been type
3848 // legalized.
3849 while (TLI.getTypeAction(Context&: Ctx, VT) != TargetLoweringBase::TypeLegal)
3850 VT = TLI.getTypeToTransformTo(Context&: Ctx, VT);
3851
3852 // If the vselect is legal, assume we want to leave this as a vector setcc +
3853 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3854 // min/max is legal on the scalar type.
3855 bool UseScalarMinMax = VT.isVector() &&
3856 !TLI.isOperationLegalOrCustom(Op: ISD::VSELECT, VT);
3857
3858 // ValueTracking's select pattern matching does not account for -0.0,
3859 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3860 // -0.0 is less than +0.0.
3861 const Value *LHS, *RHS;
3862 auto SPR = matchSelectPattern(V: &I, LHS, RHS);
3863 ISD::NodeType Opc = ISD::DELETED_NODE;
3864 switch (SPR.Flavor) {
3865 case SPF_UMAX: Opc = ISD::UMAX; break;
3866 case SPF_UMIN: Opc = ISD::UMIN; break;
3867 case SPF_SMAX: Opc = ISD::SMAX; break;
3868 case SPF_SMIN: Opc = ISD::SMIN; break;
3869 case SPF_FMINNUM:
3870 if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
3871 break;
3872
3873 switch (SPR.NaNBehavior) {
3874 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3875 case SPNB_RETURNS_NAN: break;
3876 case SPNB_RETURNS_OTHER:
3877 Opc = ISD::FMINIMUMNUM;
3878 Flags.setNoSignedZeros(true);
3879 break;
3880 case SPNB_RETURNS_ANY:
3881 if (TLI.isOperationLegalOrCustom(Op: ISD::FMINNUM, VT) ||
3882 (UseScalarMinMax &&
3883 TLI.isOperationLegalOrCustom(Op: ISD::FMINNUM, VT: VT.getScalarType())))
3884 Opc = ISD::FMINNUM;
3885 break;
3886 }
3887 break;
3888 case SPF_FMAXNUM:
3889 if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
3890 break;
3891
3892 switch (SPR.NaNBehavior) {
3893 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3894 case SPNB_RETURNS_NAN: break;
3895 case SPNB_RETURNS_OTHER:
3896 Opc = ISD::FMAXIMUMNUM;
3897 Flags.setNoSignedZeros(true);
3898 break;
3899 case SPNB_RETURNS_ANY:
3900 if (TLI.isOperationLegalOrCustom(Op: ISD::FMAXNUM, VT) ||
3901 (UseScalarMinMax &&
3902 TLI.isOperationLegalOrCustom(Op: ISD::FMAXNUM, VT: VT.getScalarType())))
3903 Opc = ISD::FMAXNUM;
3904 break;
3905 }
3906 break;
3907 case SPF_NABS:
3908 Negate = true;
3909 [[fallthrough]];
3910 case SPF_ABS:
3911 IsUnaryAbs = true;
3912 Opc = ISD::ABS;
3913 break;
3914 default: break;
3915 }
3916
3917 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3918 (TLI.isOperationLegalOrCustom(Op: Opc, VT) ||
3919 (UseScalarMinMax &&
3920 TLI.isOperationLegalOrCustom(Op: Opc, VT: VT.getScalarType()))) &&
3921 // If the underlying comparison instruction is used by any other
3922 // instruction, the consumed instructions won't be destroyed, so it is
3923 // not profitable to convert to a min/max.
3924 hasOnlySelectUsers(Cond: cast<SelectInst>(Val: I).getCondition())) {
3925 OpCode = Opc;
3926 LHSVal = getValue(V: LHS);
3927 RHSVal = getValue(V: RHS);
3928 BaseOps.clear();
3929 }
3930
3931 if (IsUnaryAbs) {
3932 OpCode = Opc;
3933 LHSVal = getValue(V: LHS);
3934 BaseOps.clear();
3935 }
3936 }
3937
3938 if (IsUnaryAbs) {
3939 for (unsigned i = 0; i != NumValues; ++i) {
3940 SDLoc dl = getCurSDLoc();
3941 EVT VT = LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i);
3942 Values[i] =
3943 DAG.getNode(Opcode: OpCode, DL: dl, VT, Operand: LHSVal.getValue(R: LHSVal.getResNo() + i));
3944 if (Negate)
3945 Values[i] = DAG.getNegative(Val: Values[i], DL: dl, VT);
3946 }
3947 } else {
3948 for (unsigned i = 0; i != NumValues; ++i) {
3949 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3950 Ops.push_back(Elt: SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3951 Ops.push_back(Elt: SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3952 Values[i] = DAG.getNode(
3953 Opcode: OpCode, DL: getCurSDLoc(),
3954 VT: LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i), Ops, Flags);
3955 }
3956 }
3957
3958 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
3959 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
3960}
3961
3962void SelectionDAGBuilder::visitTrunc(const User &I) {
3963 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3964 SDValue N = getValue(V: I.getOperand(i: 0));
3965 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3966 Ty: I.getType());
3967 SDNodeFlags Flags;
3968 if (auto *Trunc = dyn_cast<TruncInst>(Val: &I)) {
3969 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
3970 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
3971 }
3972
3973 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
3974}
3975
3976void SelectionDAGBuilder::visitZExt(const User &I) {
3977 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3978 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3979 SDValue N = getValue(V: I.getOperand(i: 0));
3980 auto &TLI = DAG.getTargetLoweringInfo();
3981 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
3982
3983 SDNodeFlags Flags;
3984 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(Val: &I))
3985 Flags.setNonNeg(PNI->hasNonNeg());
3986
3987 // Eagerly use nonneg information to canonicalize towards sign_extend if
3988 // that is the target's preference.
3989 // TODO: Let the target do this later.
3990 if (Flags.hasNonNeg() &&
3991 TLI.isSExtCheaperThanZExt(FromTy: N.getValueType(), ToTy: DestVT)) {
3992 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
3993 return;
3994 }
3995
3996 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
3997}
3998
3999void SelectionDAGBuilder::visitSExt(const User &I) {
4000 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4001 // SExt also can't be a cast to bool for same reason. So, nothing much to do
4002 SDValue N = getValue(V: I.getOperand(i: 0));
4003 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4004 Ty: I.getType());
4005 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4006}
4007
4008void SelectionDAGBuilder::visitFPTrunc(const User &I) {
4009 // FPTrunc is never a no-op cast, no need to check
4010 SDValue N = getValue(V: I.getOperand(i: 0));
4011 SDLoc dl = getCurSDLoc();
4012 SDNodeFlags Flags;
4013 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
4014 Flags.copyFMF(FPMO: *FPOp);
4015 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4016 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4017 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: DestVT, N1: N,
4018 N2: DAG.getTargetConstant(
4019 Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4020 Flags));
4021}
4022
4023void SelectionDAGBuilder::visitFPExt(const User &I) {
4024 // FPExt is never a no-op cast, no need to check
4025 SDValue N = getValue(V: I.getOperand(i: 0));
4026 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4027 Ty: I.getType());
4028 SDNodeFlags Flags;
4029 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
4030 Flags.copyFMF(FPMO: *FPOp);
4031 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4032}
4033
4034void SelectionDAGBuilder::visitFPToUI(const User &I) {
4035 // FPToUI is never a no-op cast, no need to check
4036 SDValue N = getValue(V: I.getOperand(i: 0));
4037 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4038 Ty: I.getType());
4039 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4040}
4041
4042void SelectionDAGBuilder::visitFPToSI(const User &I) {
4043 // FPToSI is never a no-op cast, no need to check
4044 SDValue N = getValue(V: I.getOperand(i: 0));
4045 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4046 Ty: I.getType());
4047 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4048}
4049
4050void SelectionDAGBuilder::visitUIToFP(const User &I) {
4051 // UIToFP is never a no-op cast, no need to check
4052 SDValue N = getValue(V: I.getOperand(i: 0));
4053 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4054 Ty: I.getType());
4055 SDNodeFlags Flags;
4056 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(Val: &I))
4057 Flags.setNonNeg(PNI->hasNonNeg());
4058
4059 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4060}
4061
4062void SelectionDAGBuilder::visitSIToFP(const User &I) {
4063 // SIToFP is never a no-op cast, no need to check
4064 SDValue N = getValue(V: I.getOperand(i: 0));
4065 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4066 Ty: I.getType());
4067 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4068}
4069
4070void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
4071 SDValue N = getValue(V: I.getOperand(i: 0));
4072 // By definition the type of the ptrtoaddr must be equal to the address type.
4073 const auto &TLI = DAG.getTargetLoweringInfo();
4074 EVT AddrVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4075 // The address width must be smaller or equal to the pointer representation
4076 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).
4077 N = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: AddrVT, Operand: N);
4078 setValue(V: &I, NewN: N);
4079}
4080
4081void SelectionDAGBuilder::visitPtrToInt(const User &I) {
4082 // What to do depends on the size of the integer and the size of the pointer.
4083 // We can either truncate, zero extend, or no-op, accordingly.
4084 SDValue N = getValue(V: I.getOperand(i: 0));
4085 auto &TLI = DAG.getTargetLoweringInfo();
4086 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4087 Ty: I.getType());
4088 EVT PtrMemVT =
4089 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i: 0)->getType());
4090 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
4091 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
4092 setValue(V: &I, NewN: N);
4093}
4094
4095void SelectionDAGBuilder::visitIntToPtr(const User &I) {
4096 // What to do depends on the size of the integer and the size of the pointer.
4097 // We can either truncate, zero extend, or no-op, accordingly.
4098 SDValue N = getValue(V: I.getOperand(i: 0));
4099 auto &TLI = DAG.getTargetLoweringInfo();
4100 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4101 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4102 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
4103 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
4104 setValue(V: &I, NewN: N);
4105}
4106
4107void SelectionDAGBuilder::visitBitCast(const User &I) {
4108 SDValue N = getValue(V: I.getOperand(i: 0));
4109 SDLoc dl = getCurSDLoc();
4110 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4111 Ty: I.getType());
4112
4113 // BitCast assures us that source and destination are the same size so this is
4114 // either a BITCAST or a no-op.
4115 if (DestVT != N.getValueType())
4116 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITCAST, DL: dl,
4117 VT: DestVT, Operand: N)); // convert types.
4118 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
4119 // might fold any kind of constant expression to an integer constant and that
4120 // is not what we are looking for. Only recognize a bitcast of a genuine
4121 // constant integer as an opaque constant.
4122 else if(ConstantInt *C = dyn_cast<ConstantInt>(Val: I.getOperand(i: 0)))
4123 setValue(V: &I, NewN: DAG.getConstant(Val: C->getValue(), DL: dl, VT: DestVT, /*isTarget=*/false,
4124 /*isOpaque*/true));
4125 else
4126 setValue(V: &I, NewN: N); // noop cast.
4127}
4128
4129void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
4130 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4131 const Value *SV = I.getOperand(i: 0);
4132 SDValue N = getValue(V: SV);
4133 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4134
4135 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
4136 unsigned DestAS = I.getType()->getPointerAddressSpace();
4137
4138 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
4139 N = DAG.getAddrSpaceCast(dl: getCurSDLoc(), VT: DestVT, Ptr: N, SrcAS, DestAS);
4140
4141 setValue(V: &I, NewN: N);
4142}
4143
4144void SelectionDAGBuilder::visitInsertElement(const User &I) {
4145 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4146 SDValue InVec = getValue(V: I.getOperand(i: 0));
4147 SDValue InVal = getValue(V: I.getOperand(i: 1));
4148 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 2)), DL: getCurSDLoc(),
4149 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
4150 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: getCurSDLoc(),
4151 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
4152 N1: InVec, N2: InVal, N3: InIdx));
4153}
4154
4155void SelectionDAGBuilder::visitExtractElement(const User &I) {
4156 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4157 SDValue InVec = getValue(V: I.getOperand(i: 0));
4158 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 1)), DL: getCurSDLoc(),
4159 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
4160 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: getCurSDLoc(),
4161 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
4162 N1: InVec, N2: InIdx));
4163}
4164
4165void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4166 SDValue Src1 = getValue(V: I.getOperand(i: 0));
4167 SDValue Src2 = getValue(V: I.getOperand(i: 1));
4168 ArrayRef<int> Mask;
4169 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &I))
4170 Mask = SVI->getShuffleMask();
4171 else
4172 Mask = cast<ConstantExpr>(Val: I).getShuffleMask();
4173 SDLoc DL = getCurSDLoc();
4174 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4175 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4176 EVT SrcVT = Src1.getValueType();
4177
4178 if (all_of(Range&: Mask, P: equal_to(Arg: 0)) && VT.isScalableVector()) {
4179 // Canonical splat form of first element of first input vector.
4180 SDValue FirstElt =
4181 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SrcVT.getScalarType(), N1: Src1,
4182 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4183 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: FirstElt));
4184 return;
4185 }
4186
4187 // For now, we only handle splats for scalable vectors.
4188 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4189 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4190 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4191
4192 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4193 unsigned MaskNumElts = Mask.size();
4194
4195 if (SrcNumElts == MaskNumElts) {
4196 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask));
4197 return;
4198 }
4199
4200 // Normalize the shuffle vector since mask and vector length don't match.
4201 if (SrcNumElts < MaskNumElts) {
4202 // Mask is longer than the source vectors. We can use concatenate vector to
4203 // make the mask and vectors lengths match.
4204
4205 if (MaskNumElts % SrcNumElts == 0) {
4206 // Mask length is a multiple of the source vector length.
4207 // Check if the shuffle is some kind of concatenation of the input
4208 // vectors.
4209 unsigned NumConcat = MaskNumElts / SrcNumElts;
4210 bool IsConcat = true;
4211 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4212 for (unsigned i = 0; i != MaskNumElts; ++i) {
4213 int Idx = Mask[i];
4214 if (Idx < 0)
4215 continue;
4216 // Ensure the indices in each SrcVT sized piece are sequential and that
4217 // the same source is used for the whole piece.
4218 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4219 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4220 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4221 IsConcat = false;
4222 break;
4223 }
4224 // Remember which source this index came from.
4225 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4226 }
4227
4228 // The shuffle is concatenating multiple vectors together. Just emit
4229 // a CONCAT_VECTORS operation.
4230 if (IsConcat) {
4231 SmallVector<SDValue, 8> ConcatOps;
4232 for (auto Src : ConcatSrcs) {
4233 if (Src < 0)
4234 ConcatOps.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
4235 else if (Src == 0)
4236 ConcatOps.push_back(Elt: Src1);
4237 else
4238 ConcatOps.push_back(Elt: Src2);
4239 }
4240 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps));
4241 return;
4242 }
4243 }
4244
4245 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
4246 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4247 EVT PaddedVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(),
4248 NumElements: PaddedMaskNumElts);
4249
4250 // Pad both vectors with undefs to make them the same length as the mask.
4251 SDValue UndefVal = DAG.getUNDEF(VT: SrcVT);
4252
4253 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4254 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4255 MOps1[0] = Src1;
4256 MOps2[0] = Src2;
4257
4258 Src1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps1);
4259 Src2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps2);
4260
4261 // Readjust mask for new input vector length.
4262 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4263 for (unsigned i = 0; i != MaskNumElts; ++i) {
4264 int Idx = Mask[i];
4265 if (Idx >= (int)SrcNumElts)
4266 Idx -= SrcNumElts - PaddedMaskNumElts;
4267 MappedOps[i] = Idx;
4268 }
4269
4270 SDValue Result = DAG.getVectorShuffle(VT: PaddedVT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps);
4271
4272 // If the concatenated vector was padded, extract a subvector with the
4273 // correct number of elements.
4274 if (MaskNumElts != PaddedMaskNumElts)
4275 Result = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Result,
4276 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4277
4278 setValue(V: &I, NewN: Result);
4279 return;
4280 }
4281
4282 assert(SrcNumElts > MaskNumElts);
4283
4284 // Analyze the access pattern of the vector to see if we can extract
4285 // two subvectors and do the shuffle.
4286 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4287 bool CanExtract = true;
4288 for (int Idx : Mask) {
4289 unsigned Input = 0;
4290 if (Idx < 0)
4291 continue;
4292
4293 if (Idx >= (int)SrcNumElts) {
4294 Input = 1;
4295 Idx -= SrcNumElts;
4296 }
4297
4298 // If all the indices come from the same MaskNumElts sized portion of
4299 // the sources we can use extract. Also make sure the extract wouldn't
4300 // extract past the end of the source.
4301 int NewStartIdx = alignDown(Value: Idx, Align: MaskNumElts);
4302 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4303 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4304 CanExtract = false;
4305 // Make sure we always update StartIdx as we use it to track if all
4306 // elements are undef.
4307 StartIdx[Input] = NewStartIdx;
4308 }
4309
4310 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4311 setValue(V: &I, NewN: DAG.getUNDEF(VT)); // Vectors are not used.
4312 return;
4313 }
4314 if (CanExtract) {
4315 // Extract appropriate subvector and generate a vector shuffle
4316 for (unsigned Input = 0; Input < 2; ++Input) {
4317 SDValue &Src = Input == 0 ? Src1 : Src2;
4318 if (StartIdx[Input] < 0)
4319 Src = DAG.getUNDEF(VT);
4320 else {
4321 Src = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Src,
4322 N2: DAG.getVectorIdxConstant(Val: StartIdx[Input], DL));
4323 }
4324 }
4325
4326 // Calculate new mask.
4327 SmallVector<int, 8> MappedOps(Mask);
4328 for (int &Idx : MappedOps) {
4329 if (Idx >= (int)SrcNumElts)
4330 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4331 else if (Idx >= 0)
4332 Idx -= StartIdx[0];
4333 }
4334
4335 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps));
4336 return;
4337 }
4338
4339 // We can't use either concat vectors or extract subvectors so fall back to
4340 // replacing the shuffle with extract and build vector.
4341 // to insert and build vector.
4342 EVT EltVT = VT.getVectorElementType();
4343 SmallVector<SDValue,8> Ops;
4344 for (int Idx : Mask) {
4345 SDValue Res;
4346
4347 if (Idx < 0) {
4348 Res = DAG.getUNDEF(VT: EltVT);
4349 } else {
4350 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4351 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4352
4353 Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Src,
4354 N2: DAG.getVectorIdxConstant(Val: Idx, DL));
4355 }
4356
4357 Ops.push_back(Elt: Res);
4358 }
4359
4360 setValue(V: &I, NewN: DAG.getBuildVector(VT, DL, Ops));
4361}
4362
4363void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4364 ArrayRef<unsigned> Indices = I.getIndices();
4365 const Value *Op0 = I.getOperand(i_nocapture: 0);
4366 const Value *Op1 = I.getOperand(i_nocapture: 1);
4367 Type *AggTy = I.getType();
4368 Type *ValTy = Op1->getType();
4369 bool IntoUndef = isa<UndefValue>(Val: Op0);
4370 bool FromUndef = isa<UndefValue>(Val: Op1);
4371
4372 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4373
4374 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4375 SmallVector<EVT, 4> AggValueVTs;
4376 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: AggTy, ValueVTs&: AggValueVTs);
4377 SmallVector<EVT, 4> ValValueVTs;
4378 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4379
4380 unsigned NumAggValues = AggValueVTs.size();
4381 unsigned NumValValues = ValValueVTs.size();
4382 SmallVector<SDValue, 4> Values(NumAggValues);
4383
4384 // Ignore an insertvalue that produces an empty object
4385 if (!NumAggValues) {
4386 setValue(V: &I, NewN: DAG.getUNDEF(VT: MVT(MVT::Other)));
4387 return;
4388 }
4389
4390 SDValue Agg = getValue(V: Op0);
4391 unsigned i = 0;
4392 // Copy the beginning value(s) from the original aggregate.
4393 for (; i != LinearIndex; ++i)
4394 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4395 SDValue(Agg.getNode(), Agg.getResNo() + i);
4396 // Copy values from the inserted value(s).
4397 if (NumValValues) {
4398 SDValue Val = getValue(V: Op1);
4399 for (; i != LinearIndex + NumValValues; ++i)
4400 Values[i] = FromUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4401 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4402 }
4403 // Copy remaining value(s) from the original aggregate.
4404 for (; i != NumAggValues; ++i)
4405 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4406 SDValue(Agg.getNode(), Agg.getResNo() + i);
4407
4408 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4409 VTList: DAG.getVTList(VTs: AggValueVTs), Ops: Values));
4410}
4411
4412void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4413 ArrayRef<unsigned> Indices = I.getIndices();
4414 const Value *Op0 = I.getOperand(i_nocapture: 0);
4415 Type *AggTy = Op0->getType();
4416 Type *ValTy = I.getType();
4417 bool OutOfUndef = isa<UndefValue>(Val: Op0);
4418
4419 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4420
4421 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4422 SmallVector<EVT, 4> ValValueVTs;
4423 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4424
4425 unsigned NumValValues = ValValueVTs.size();
4426
4427 // Ignore a extractvalue that produces an empty object
4428 if (!NumValValues) {
4429 setValue(V: &I, NewN: DAG.getUNDEF(VT: MVT(MVT::Other)));
4430 return;
4431 }
4432
4433 SmallVector<SDValue, 4> Values(NumValValues);
4434
4435 SDValue Agg = getValue(V: Op0);
4436 // Copy out the selected value(s).
4437 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4438 Values[i - LinearIndex] =
4439 OutOfUndef ?
4440 DAG.getUNDEF(VT: Agg.getNode()->getValueType(ResNo: Agg.getResNo() + i)) :
4441 SDValue(Agg.getNode(), Agg.getResNo() + i);
4442
4443 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4444 VTList: DAG.getVTList(VTs: ValValueVTs), Ops: Values));
4445}
4446
4447void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4448 Value *Op0 = I.getOperand(i: 0);
4449 // Note that the pointer operand may be a vector of pointers. Take the scalar
4450 // element which holds a pointer.
4451 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4452 SDValue N = getValue(V: Op0);
4453 SDLoc dl = getCurSDLoc();
4454 auto &TLI = DAG.getTargetLoweringInfo();
4455 GEPNoWrapFlags NW = cast<GEPOperator>(Val: I).getNoWrapFlags();
4456
4457 // For a vector GEP, keep the prefix scalar as long as possible, then
4458 // convert any scalars encountered after the first vector operand to vectors.
4459 bool IsVectorGEP = I.getType()->isVectorTy();
4460 ElementCount VectorElementCount =
4461 IsVectorGEP ? cast<VectorType>(Val: I.getType())->getElementCount()
4462 : ElementCount::getFixed(MinVal: 0);
4463
4464 for (gep_type_iterator GTI = gep_type_begin(GEP: &I), E = gep_type_end(GEP: &I);
4465 GTI != E; ++GTI) {
4466 const Value *Idx = GTI.getOperand();
4467 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4468 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
4469 if (Field) {
4470 // N = N + Offset
4471 uint64_t Offset =
4472 DAG.getDataLayout().getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
4473
4474 // In an inbounds GEP with an offset that is nonnegative even when
4475 // interpreted as signed, assume there is no unsigned overflow.
4476 SDNodeFlags Flags;
4477 if (NW.hasNoUnsignedWrap() ||
4478 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4479 Flags |= SDNodeFlags::NoUnsignedWrap;
4480 Flags.setInBounds(NW.isInBounds());
4481
4482 N = DAG.getMemBasePlusOffset(
4483 Base: N, Offset: DAG.getConstant(Val: Offset, DL: dl, VT: N.getValueType()), DL: dl, Flags);
4484 }
4485 } else {
4486 // IdxSize is the width of the arithmetic according to IR semantics.
4487 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4488 // (and fix up the result later).
4489 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4490 MVT IdxTy = MVT::getIntegerVT(BitWidth: IdxSize);
4491 TypeSize ElementSize =
4492 GTI.getSequentialElementStride(DL: DAG.getDataLayout());
4493 // We intentionally mask away the high bits here; ElementSize may not
4494 // fit in IdxTy.
4495 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4496 /*isSigned=*/false, /*implicitTrunc=*/true);
4497 bool ElementScalable = ElementSize.isScalable();
4498
4499 // If this is a scalar constant or a splat vector of constants,
4500 // handle it quickly.
4501 const auto *C = dyn_cast<Constant>(Val: Idx);
4502 if (C && isa<VectorType>(Val: C->getType()))
4503 C = C->getSplatValue();
4504
4505 const auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
4506 if (CI && CI->isZero())
4507 continue;
4508 if (CI && !ElementScalable) {
4509 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(width: IdxSize);
4510 LLVMContext &Context = *DAG.getContext();
4511 SDValue OffsVal;
4512 if (N.getValueType().isVector())
4513 OffsVal = DAG.getConstant(
4514 Val: Offs, DL: dl, VT: EVT::getVectorVT(Context, VT: IdxTy, EC: VectorElementCount));
4515 else
4516 OffsVal = DAG.getConstant(Val: Offs, DL: dl, VT: IdxTy);
4517
4518 // In an inbounds GEP with an offset that is nonnegative even when
4519 // interpreted as signed, assume there is no unsigned overflow.
4520 SDNodeFlags Flags;
4521 if (NW.hasNoUnsignedWrap() ||
4522 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4523 Flags.setNoUnsignedWrap(true);
4524 Flags.setInBounds(NW.isInBounds());
4525
4526 OffsVal = DAG.getSExtOrTrunc(Op: OffsVal, DL: dl, VT: N.getValueType());
4527
4528 N = DAG.getMemBasePlusOffset(Base: N, Offset: OffsVal, DL: dl, Flags);
4529 continue;
4530 }
4531
4532 // N = N + Idx * ElementMul;
4533 SDValue IdxN = getValue(V: Idx);
4534
4535 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
4536 if (N.getValueType().isVector()) {
4537 EVT VT = EVT::getVectorVT(Context&: *Context, VT: IdxN.getValueType(),
4538 EC: VectorElementCount);
4539 IdxN = DAG.getSplat(VT, DL: dl, Op: IdxN);
4540 } else {
4541 EVT VT =
4542 EVT::getVectorVT(Context&: *Context, VT: N.getValueType(), EC: VectorElementCount);
4543 N = DAG.getSplat(VT, DL: dl, Op: N);
4544 }
4545 }
4546
4547 // If the index is smaller or larger than intptr_t, truncate or extend
4548 // it.
4549 IdxN = DAG.getSExtOrTrunc(Op: IdxN, DL: dl, VT: N.getValueType());
4550
4551 SDNodeFlags ScaleFlags;
4552 // The multiplication of an index by the type size does not wrap the
4553 // pointer index type in a signed sense (mul nsw).
4554 ScaleFlags.setNoSignedWrap(NW.hasNoUnsignedSignedWrap());
4555
4556 // The multiplication of an index by the type size does not wrap the
4557 // pointer index type in an unsigned sense (mul nuw).
4558 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4559
4560 if (ElementScalable) {
4561 EVT VScaleTy = N.getValueType().getScalarType();
4562 SDValue VScale = DAG.getNode(
4563 Opcode: ISD::VSCALE, DL: dl, VT: VScaleTy,
4564 Operand: DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl, VT: VScaleTy));
4565 if (N.getValueType().isVector())
4566 VScale = DAG.getSplatVector(VT: N.getValueType(), DL: dl, Op: VScale);
4567 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: VScale,
4568 Flags: ScaleFlags);
4569 } else {
4570 // If this is a multiply by a power of two, turn it into a shl
4571 // immediately. This is a very common case.
4572 if (ElementMul != 1) {
4573 if (ElementMul.isPowerOf2()) {
4574 unsigned Amt = ElementMul.logBase2();
4575 IdxN = DAG.getNode(
4576 Opcode: ISD::SHL, DL: dl, VT: N.getValueType(), N1: IdxN,
4577 N2: DAG.getShiftAmountConstant(Val: Amt, VT: N.getValueType(), DL: dl),
4578 Flags: ScaleFlags);
4579 } else {
4580 SDValue Scale = DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl,
4581 VT: IdxN.getValueType());
4582 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: Scale,
4583 Flags: ScaleFlags);
4584 }
4585 }
4586 }
4587
4588 // The successive addition of the current address, truncated to the
4589 // pointer index type and interpreted as an unsigned number, and each
4590 // offset, also interpreted as an unsigned number, does not wrap the
4591 // pointer index type (add nuw).
4592 SDNodeFlags AddFlags;
4593 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4594 AddFlags.setInBounds(NW.isInBounds());
4595
4596 N = DAG.getMemBasePlusOffset(Base: N, Offset: IdxN, DL: dl, Flags: AddFlags);
4597 }
4598 }
4599
4600 if (IsVectorGEP && !N.getValueType().isVector()) {
4601 EVT VT = EVT::getVectorVT(Context&: *Context, VT: N.getValueType(), EC: VectorElementCount);
4602 N = DAG.getSplat(VT, DL: dl, Op: N);
4603 }
4604
4605 MVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout(), AS);
4606 MVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout(), AS);
4607 if (IsVectorGEP) {
4608 PtrTy = MVT::getVectorVT(VT: PtrTy, EC: VectorElementCount);
4609 PtrMemTy = MVT::getVectorVT(VT: PtrMemTy, EC: VectorElementCount);
4610 }
4611
4612 if (PtrMemTy != PtrTy && !cast<GEPOperator>(Val: I).isInBounds())
4613 N = DAG.getPtrExtendInReg(Op: N, DL: dl, VT: PtrMemTy);
4614
4615 setValue(V: &I, NewN: N);
4616}
4617
4618void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4619 // If this is a fixed sized alloca in the entry block of the function,
4620 // allocate it statically on the stack.
4621 if (FuncInfo.StaticAllocaMap.count(Val: &I))
4622 return; // getValue will auto-populate this.
4623
4624 SDLoc dl = getCurSDLoc();
4625 Type *Ty = I.getAllocatedType();
4626 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4627 auto &DL = DAG.getDataLayout();
4628 TypeSize TySize = DL.getTypeAllocSize(Ty);
4629 MaybeAlign Alignment = I.getAlign();
4630
4631 SDValue AllocSize = getValue(V: I.getArraySize());
4632
4633 EVT IntPtr = TLI.getPointerTy(DL, AS: I.getAddressSpace());
4634 if (AllocSize.getValueType() != IntPtr)
4635 AllocSize = DAG.getZExtOrTrunc(Op: AllocSize, DL: dl, VT: IntPtr);
4636
4637 AllocSize = DAG.getNode(
4638 Opcode: ISD::MUL, DL: dl, VT: IntPtr, N1: AllocSize,
4639 N2: DAG.getZExtOrTrunc(Op: DAG.getTypeSize(DL: dl, VT: MVT::i64, TS: TySize), DL: dl, VT: IntPtr));
4640
4641 // Handle alignment. If the requested alignment is less than or equal to
4642 // the stack alignment, ignore it. If the size is greater than or equal to
4643 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4644 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4645 if (*Alignment <= StackAlign)
4646 Alignment = std::nullopt;
4647
4648 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4649 // Round the size of the allocation up to the stack alignment size
4650 // by add SA-1 to the size. This doesn't overflow because we're computing
4651 // an address inside an alloca.
4652 AllocSize = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4653 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT: IntPtr),
4654 Flags: SDNodeFlags::NoUnsignedWrap);
4655
4656 // Mask out the low bits for alignment purposes.
4657 AllocSize = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4658 N2: DAG.getSignedConstant(Val: ~StackAlignMask, DL: dl, VT: IntPtr));
4659
4660 SDValue Ops[] = {
4661 getRoot(), AllocSize,
4662 DAG.getConstant(Val: Alignment ? Alignment->value() : 0, DL: dl, VT: IntPtr)};
4663 SDVTList VTs = DAG.getVTList(VT1: AllocSize.getValueType(), VT2: MVT::Other);
4664 SDValue DSA = DAG.getNode(Opcode: ISD::DYNAMIC_STACKALLOC, DL: dl, VTList: VTs, Ops);
4665 setValue(V: &I, NewN: DSA);
4666 DAG.setRoot(DSA.getValue(R: 1));
4667
4668 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4669}
4670
4671static const MDNode *getRangeMetadata(const Instruction &I) {
4672 return I.getMetadata(KindID: LLVMContext::MD_range);
4673}
4674
4675static std::optional<ConstantRange> getRange(const Instruction &I) {
4676 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4677 if (std::optional<ConstantRange> CR = CB->getRange())
4678 return CR;
4679 if (const MDNode *Range = getRangeMetadata(I))
4680 return getConstantRangeFromMetadata(RangeMD: *Range);
4681 return std::nullopt;
4682}
4683
4684static FPClassTest getNoFPClass(const Instruction &I) {
4685 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4686 return CB->getRetNoFPClass();
4687 return fcNone;
4688}
4689
4690void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4691 if (I.isAtomic())
4692 return visitAtomicLoad(I);
4693
4694 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4695 const Value *SV = I.getOperand(i_nocapture: 0);
4696 if (TLI.supportSwiftError()) {
4697 // Swifterror values can come from either a function parameter with
4698 // swifterror attribute or an alloca with swifterror attribute.
4699 if (const Argument *Arg = dyn_cast<Argument>(Val: SV)) {
4700 if (Arg->hasSwiftErrorAttr())
4701 return visitLoadFromSwiftError(I);
4702 }
4703
4704 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: SV)) {
4705 if (Alloca->isSwiftError())
4706 return visitLoadFromSwiftError(I);
4707 }
4708 }
4709
4710 SDValue Ptr = getValue(V: SV);
4711
4712 Type *Ty = I.getType();
4713 SmallVector<EVT, 4> ValueVTs, MemVTs;
4714 SmallVector<TypeSize, 4> Offsets;
4715 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty, ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
4716 unsigned NumValues = ValueVTs.size();
4717 if (NumValues == 0)
4718 return;
4719
4720 Align Alignment = I.getAlign();
4721 AAMDNodes AAInfo = I.getAAMetadata();
4722 const MDNode *Ranges = getRangeMetadata(I);
4723 bool isVolatile = I.isVolatile();
4724 MachineMemOperand::Flags MMOFlags =
4725 TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
4726
4727 SDValue Root;
4728 bool ConstantMemory = false;
4729 if (isVolatile)
4730 // Serialize volatile loads with other side effects.
4731 Root = getRoot();
4732 else if (NumValues > MaxParallelChains)
4733 Root = getMemoryRoot();
4734 else if (BatchAA &&
4735 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(
4736 SV,
4737 LocationSize::precise(Value: DAG.getDataLayout().getTypeStoreSize(Ty)),
4738 AAInfo))) {
4739 // Do not serialize (non-volatile) loads of constant memory with anything.
4740 Root = DAG.getEntryNode();
4741 ConstantMemory = true;
4742 MMOFlags |= MachineMemOperand::MOInvariant;
4743 } else {
4744 // Do not serialize non-volatile loads against each other.
4745 Root = DAG.getRoot();
4746 }
4747
4748 SDLoc dl = getCurSDLoc();
4749
4750 if (isVolatile)
4751 Root = TLI.prepareVolatileOrAtomicLoad(Chain: Root, DL: dl, DAG);
4752
4753 SmallVector<SDValue, 4> Values(NumValues);
4754 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
4755
4756 unsigned ChainI = 0;
4757 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4758 // Serializing loads here may result in excessive register pressure, and
4759 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4760 // could recover a bit by hoisting nodes upward in the chain by recognizing
4761 // they are side-effect free or do not alias. The optimizer should really
4762 // avoid this case by converting large object/array copies to llvm.memcpy
4763 // (MaxParallelChains should always remain as failsafe).
4764 if (ChainI == MaxParallelChains) {
4765 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4766 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4767 Ops: ArrayRef(Chains.data(), ChainI));
4768 Root = Chain;
4769 ChainI = 0;
4770 }
4771
4772 // TODO: MachinePointerInfo only supports a fixed length offset.
4773 MachinePointerInfo PtrInfo =
4774 !Offsets[i].isScalable() || Offsets[i].isZero()
4775 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4776 : MachinePointerInfo();
4777
4778 SDValue A = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
4779 SDValue L = DAG.getLoad(VT: MemVTs[i], dl, Chain: Root, Ptr: A, PtrInfo, Alignment,
4780 MMOFlags, AAInfo, Ranges);
4781 Chains[ChainI] = L.getValue(R: 1);
4782
4783 if (MemVTs[i] != ValueVTs[i])
4784 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT: ValueVTs[i]);
4785
4786 if (MDNode *NoFPClassMD = I.getMetadata(KindID: LLVMContext::MD_nofpclass)) {
4787 uint64_t FPTestInt =
4788 cast<ConstantInt>(
4789 Val: cast<ConstantAsMetadata>(Val: NoFPClassMD->getOperand(I: 0))->getValue())
4790 ->getZExtValue();
4791 if (FPTestInt != fcNone) {
4792 SDValue FPTestConst =
4793 DAG.getTargetConstant(Val: FPTestInt, DL: SDLoc(), VT: MVT::i32);
4794 L = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: L.getValueType(), N1: L,
4795 N2: FPTestConst);
4796 }
4797 }
4798 Values[i] = L;
4799 }
4800
4801 if (!ConstantMemory) {
4802 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4803 Ops: ArrayRef(Chains.data(), ChainI));
4804 if (isVolatile)
4805 DAG.setRoot(Chain);
4806 else
4807 PendingLoads.push_back(Elt: Chain);
4808 }
4809
4810 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
4811 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
4812}
4813
4814void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4815 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4816 "call visitStoreToSwiftError when backend supports swifterror");
4817
4818 SmallVector<EVT, 4> ValueVTs;
4819 SmallVector<uint64_t, 4> Offsets;
4820 const Value *SrcV = I.getOperand(i_nocapture: 0);
4821 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
4822 Ty: SrcV->getType(), ValueVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
4823 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4824 "expect a single EVT for swifterror");
4825
4826 SDValue Src = getValue(V: SrcV);
4827 // Create a virtual register, then update the virtual register.
4828 Register VReg =
4829 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4830 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4831 // Chain can be getRoot or getControlRoot.
4832 SDValue CopyNode = DAG.getCopyToReg(Chain: getRoot(), dl: getCurSDLoc(), Reg: VReg,
4833 N: SDValue(Src.getNode(), Src.getResNo()));
4834 DAG.setRoot(CopyNode);
4835}
4836
4837void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4838 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4839 "call visitLoadFromSwiftError when backend supports swifterror");
4840
4841 assert(!I.isVolatile() &&
4842 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4843 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4844 "Support volatile, non temporal, invariant for load_from_swift_error");
4845
4846 const Value *SV = I.getOperand(i_nocapture: 0);
4847 Type *Ty = I.getType();
4848 assert(
4849 (!BatchAA ||
4850 !BatchAA->pointsToConstantMemory(MemoryLocation(
4851 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4852 I.getAAMetadata()))) &&
4853 "load_from_swift_error should not be constant memory");
4854
4855 SmallVector<EVT, 4> ValueVTs;
4856 SmallVector<uint64_t, 4> Offsets;
4857 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty,
4858 ValueVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
4859 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4860 "expect a single EVT for swifterror");
4861
4862 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4863 SDValue L = DAG.getCopyFromReg(
4864 Chain: getRoot(), dl: getCurSDLoc(),
4865 Reg: SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), VT: ValueVTs[0]);
4866
4867 setValue(V: &I, NewN: L);
4868}
4869
4870void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4871 if (I.isAtomic())
4872 return visitAtomicStore(I);
4873
4874 const Value *SrcV = I.getOperand(i_nocapture: 0);
4875 const Value *PtrV = I.getOperand(i_nocapture: 1);
4876
4877 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4878 if (TLI.supportSwiftError()) {
4879 // Swifterror values can come from either a function parameter with
4880 // swifterror attribute or an alloca with swifterror attribute.
4881 if (const Argument *Arg = dyn_cast<Argument>(Val: PtrV)) {
4882 if (Arg->hasSwiftErrorAttr())
4883 return visitStoreToSwiftError(I);
4884 }
4885
4886 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: PtrV)) {
4887 if (Alloca->isSwiftError())
4888 return visitStoreToSwiftError(I);
4889 }
4890 }
4891
4892 SmallVector<EVT, 4> ValueVTs, MemVTs;
4893 SmallVector<TypeSize, 4> Offsets;
4894 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
4895 Ty: SrcV->getType(), ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
4896 unsigned NumValues = ValueVTs.size();
4897 if (NumValues == 0)
4898 return;
4899
4900 // Get the lowered operands. Note that we do this after
4901 // checking if NumResults is zero, because with zero results
4902 // the operands won't have values in the map.
4903 SDValue Src = getValue(V: SrcV);
4904 SDValue Ptr = getValue(V: PtrV);
4905
4906 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4907 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
4908 SDLoc dl = getCurSDLoc();
4909 Align Alignment = I.getAlign();
4910 AAMDNodes AAInfo = I.getAAMetadata();
4911
4912 auto MMOFlags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
4913
4914 unsigned ChainI = 0;
4915 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4916 // See visitLoad comments.
4917 if (ChainI == MaxParallelChains) {
4918 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4919 Ops: ArrayRef(Chains.data(), ChainI));
4920 Root = Chain;
4921 ChainI = 0;
4922 }
4923
4924 // TODO: MachinePointerInfo only supports a fixed length offset.
4925 MachinePointerInfo PtrInfo =
4926 !Offsets[i].isScalable() || Offsets[i].isZero()
4927 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4928 : MachinePointerInfo();
4929
4930 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
4931 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4932 if (MemVTs[i] != ValueVTs[i])
4933 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVTs[i]);
4934 SDValue St =
4935 DAG.getStore(Chain: Root, dl, Val, Ptr: Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4936 Chains[ChainI] = St;
4937 }
4938
4939 SDValue StoreNode = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4940 Ops: ArrayRef(Chains.data(), ChainI));
4941 setValue(V: &I, NewN: StoreNode);
4942 DAG.setRoot(StoreNode);
4943}
4944
4945void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4946 bool IsCompressing) {
4947 SDLoc sdl = getCurSDLoc();
4948
4949 Value *Src0Operand = I.getArgOperand(i: 0);
4950 Value *PtrOperand = I.getArgOperand(i: 1);
4951 Value *MaskOperand = I.getArgOperand(i: 2);
4952 Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4953
4954 SDValue Ptr = getValue(V: PtrOperand);
4955 SDValue Src0 = getValue(V: Src0Operand);
4956 SDValue Mask = getValue(V: MaskOperand);
4957 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
4958
4959 EVT VT = Src0.getValueType();
4960
4961 auto MMOFlags = MachineMemOperand::MOStore;
4962 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
4963 MMOFlags |= MachineMemOperand::MONonTemporal;
4964
4965 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4966 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
4967 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: I.getAAMetadata());
4968
4969 const auto &TLI = DAG.getTargetLoweringInfo();
4970
4971 SDValue StoreNode =
4972 !IsCompressing && TTI->hasConditionalLoadStoreForType(
4973 Ty: I.getArgOperand(i: 0)->getType(), /*IsStore=*/true)
4974 ? TLI.visitMaskedStore(DAG, DL: sdl, Chain: getMemoryRoot(), MMO, Ptr, Val: Src0,
4975 Mask)
4976 : DAG.getMaskedStore(Chain: getMemoryRoot(), dl: sdl, Val: Src0, Base: Ptr, Offset, Mask,
4977 MemVT: VT, MMO, AM: ISD::UNINDEXED, /*Truncating=*/IsTruncating: false,
4978 IsCompressing);
4979 DAG.setRoot(StoreNode);
4980 setValue(V: &I, NewN: StoreNode);
4981}
4982
4983// Get a uniform base for the Gather/Scatter intrinsic.
4984// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4985// We try to represent it as a base pointer + vector of indices.
4986// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4987// The first operand of the GEP may be a single pointer or a vector of pointers
4988// Example:
4989// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4990// or
4991// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4992// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4993//
4994// When the first GEP operand is a single pointer - it is the uniform base we
4995// are looking for. If first operand of the GEP is a splat vector - we
4996// extract the splat value and use it as a uniform base.
4997// In all other cases the function returns 'false'.
4998static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4999 SDValue &Scale, SelectionDAGBuilder *SDB,
5000 const BasicBlock *CurBB, uint64_t ElemSize) {
5001 SelectionDAG& DAG = SDB->DAG;
5002 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5003 const DataLayout &DL = DAG.getDataLayout();
5004
5005 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
5006
5007 // Handle splat constant pointer.
5008 if (auto *C = dyn_cast<Constant>(Val: Ptr)) {
5009 C = C->getSplatValue();
5010 if (!C)
5011 return false;
5012
5013 Base = SDB->getValue(V: C);
5014
5015 ElementCount NumElts = cast<VectorType>(Val: Ptr->getType())->getElementCount();
5016 EVT VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TLI.getPointerTy(DL), EC: NumElts);
5017 Index = DAG.getConstant(Val: 0, DL: SDB->getCurSDLoc(), VT);
5018 Scale = DAG.getTargetConstant(Val: 1, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
5019 return true;
5020 }
5021
5022 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
5023 if (!GEP || GEP->getParent() != CurBB)
5024 return false;
5025
5026 if (GEP->getNumOperands() != 2)
5027 return false;
5028
5029 const Value *BasePtr = GEP->getPointerOperand();
5030 const Value *IndexVal = GEP->getOperand(i_nocapture: GEP->getNumOperands() - 1);
5031
5032 // Make sure the base is scalar and the index is a vector.
5033 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
5034 return false;
5035
5036 TypeSize ScaleVal = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
5037 if (ScaleVal.isScalable())
5038 return false;
5039
5040 // Target may not support the required addressing mode.
5041 if (ScaleVal != 1 &&
5042 !TLI.isLegalScaleForGatherScatter(Scale: ScaleVal.getFixedValue(), ElemSize))
5043 return false;
5044
5045 Base = SDB->getValue(V: BasePtr);
5046 Index = SDB->getValue(V: IndexVal);
5047
5048 Scale =
5049 DAG.getTargetConstant(Val: ScaleVal, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
5050 return true;
5051}
5052
5053void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
5054 SDLoc sdl = getCurSDLoc();
5055
5056 // llvm.masked.scatter.*(Src0, Ptrs, Mask)
5057 const Value *Ptr = I.getArgOperand(i: 1);
5058 SDValue Src0 = getValue(V: I.getArgOperand(i: 0));
5059 SDValue Mask = getValue(V: I.getArgOperand(i: 2));
5060 EVT VT = Src0.getValueType();
5061 Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
5062 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5063
5064 SDValue Base;
5065 SDValue Index;
5066 SDValue Scale;
5067 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
5068 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5069
5070 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5071 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5072 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOStore,
5073 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: I.getAAMetadata());
5074 if (!UniformBase) {
5075 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5076 Index = getValue(V: Ptr);
5077 Scale =
5078 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5079 }
5080
5081 EVT IdxVT = Index.getValueType();
5082 EVT EltTy = IdxVT.getVectorElementType();
5083 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5084 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
5085 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5086 }
5087
5088 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
5089 SDValue Scatter = DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: VT, dl: sdl,
5090 Ops, MMO, IndexType: ISD::SIGNED_SCALED, IsTruncating: false);
5091 DAG.setRoot(Scatter);
5092 setValue(V: &I, NewN: Scatter);
5093}
5094
5095void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
5096 SDLoc sdl = getCurSDLoc();
5097
5098 Value *PtrOperand = I.getArgOperand(i: 0);
5099 Value *MaskOperand = I.getArgOperand(i: 1);
5100 Value *Src0Operand = I.getArgOperand(i: 2);
5101 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5102
5103 SDValue Ptr = getValue(V: PtrOperand);
5104 SDValue Src0 = getValue(V: Src0Operand);
5105 SDValue Mask = getValue(V: MaskOperand);
5106 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
5107
5108 EVT VT = Src0.getValueType();
5109 AAMDNodes AAInfo = I.getAAMetadata();
5110 const MDNode *Ranges = getRangeMetadata(I);
5111
5112 // Do not serialize masked loads of constant memory with anything.
5113 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
5114 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
5115
5116 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5117
5118 auto MMOFlags = MachineMemOperand::MOLoad;
5119 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5120 MMOFlags |= MachineMemOperand::MONonTemporal;
5121 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
5122 MMOFlags |= MachineMemOperand::MOInvariant;
5123
5124 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5125 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5126 Size: VT.getStoreSize(), BaseAlignment: Alignment, AAInfo, Ranges);
5127
5128 const auto &TLI = DAG.getTargetLoweringInfo();
5129
5130 // The Load/Res may point to different values and both of them are output
5131 // variables.
5132 SDValue Load;
5133 SDValue Res;
5134 if (!IsExpanding &&
5135 TTI->hasConditionalLoadStoreForType(Ty: Src0Operand->getType(),
5136 /*IsStore=*/false))
5137 Res = TLI.visitMaskedLoad(DAG, DL: sdl, Chain: InChain, MMO, NewLoad&: Load, Ptr, PassThru: Src0, Mask);
5138 else
5139 Res = Load =
5140 DAG.getMaskedLoad(VT, dl: sdl, Chain: InChain, Base: Ptr, Offset, Mask, Src0, MemVT: VT, MMO,
5141 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5142 if (AddToChain)
5143 PendingLoads.push_back(Elt: Load.getValue(R: 1));
5144 setValue(V: &I, NewN: Res);
5145}
5146
5147void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5148 SDLoc sdl = getCurSDLoc();
5149
5150 // @llvm.masked.gather.*(Ptrs, Mask, Src0)
5151 const Value *Ptr = I.getArgOperand(i: 0);
5152 SDValue Src0 = getValue(V: I.getArgOperand(i: 2));
5153 SDValue Mask = getValue(V: I.getArgOperand(i: 1));
5154
5155 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5156 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5157 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5158
5159 const MDNode *Ranges = getRangeMetadata(I);
5160
5161 SDValue Root = DAG.getRoot();
5162 SDValue Base;
5163 SDValue Index;
5164 SDValue Scale;
5165 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
5166 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5167 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5168 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5169 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOLoad,
5170 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: I.getAAMetadata(),
5171 Ranges);
5172
5173 if (!UniformBase) {
5174 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5175 Index = getValue(V: Ptr);
5176 Scale =
5177 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5178 }
5179
5180 EVT IdxVT = Index.getValueType();
5181 EVT EltTy = IdxVT.getVectorElementType();
5182 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5183 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
5184 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5185 }
5186
5187 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5188 SDValue Gather =
5189 DAG.getMaskedGather(VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), MemVT: VT, dl: sdl, Ops, MMO,
5190 IndexType: ISD::SIGNED_SCALED, ExtTy: ISD::NON_EXTLOAD);
5191
5192 PendingLoads.push_back(Elt: Gather.getValue(R: 1));
5193 setValue(V: &I, NewN: Gather);
5194}
5195
5196void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5197 SDLoc dl = getCurSDLoc();
5198 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5199 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5200 SyncScope::ID SSID = I.getSyncScopeID();
5201
5202 SDValue InChain = getRoot();
5203
5204 MVT MemVT = getValue(V: I.getCompareOperand()).getSimpleValueType();
5205 SDVTList VTs = DAG.getVTList(VT1: MemVT, VT2: MVT::i1, VT3: MVT::Other);
5206
5207 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5208 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5209
5210 MachineFunction &MF = DAG.getMachineFunction();
5211 MachineMemOperand *MMO = MF.getMachineMemOperand(
5212 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5213 BaseAlignment: DAG.getEVTAlign(MemoryVT: MemVT), AAInfo: AAMDNodes(), Ranges: nullptr, SSID, Ordering: SuccessOrdering,
5214 FailureOrdering);
5215
5216 SDValue L = DAG.getAtomicCmpSwap(Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
5217 dl, MemVT, VTs, Chain: InChain,
5218 Ptr: getValue(V: I.getPointerOperand()),
5219 Cmp: getValue(V: I.getCompareOperand()),
5220 Swp: getValue(V: I.getNewValOperand()), MMO);
5221
5222 SDValue OutChain = L.getValue(R: 2);
5223
5224 setValue(V: &I, NewN: L);
5225 DAG.setRoot(OutChain);
5226}
5227
5228void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5229 SDLoc dl = getCurSDLoc();
5230 ISD::NodeType NT;
5231 switch (I.getOperation()) {
5232 default: llvm_unreachable("Unknown atomicrmw operation");
5233 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
5234 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
5235 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
5236 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
5237 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
5238 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
5239 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
5240 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
5241 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
5242 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
5243 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
5244 case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
5245 case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
5246 case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
5247 case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
5248 case AtomicRMWInst::FMaximum:
5249 NT = ISD::ATOMIC_LOAD_FMAXIMUM;
5250 break;
5251 case AtomicRMWInst::FMinimum:
5252 NT = ISD::ATOMIC_LOAD_FMINIMUM;
5253 break;
5254 case AtomicRMWInst::UIncWrap:
5255 NT = ISD::ATOMIC_LOAD_UINC_WRAP;
5256 break;
5257 case AtomicRMWInst::UDecWrap:
5258 NT = ISD::ATOMIC_LOAD_UDEC_WRAP;
5259 break;
5260 case AtomicRMWInst::USubCond:
5261 NT = ISD::ATOMIC_LOAD_USUB_COND;
5262 break;
5263 case AtomicRMWInst::USubSat:
5264 NT = ISD::ATOMIC_LOAD_USUB_SAT;
5265 break;
5266 }
5267 AtomicOrdering Ordering = I.getOrdering();
5268 SyncScope::ID SSID = I.getSyncScopeID();
5269
5270 SDValue InChain = getRoot();
5271
5272 auto MemVT = getValue(V: I.getValOperand()).getSimpleValueType();
5273 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5274 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5275
5276 MachineFunction &MF = DAG.getMachineFunction();
5277 MachineMemOperand *MMO = MF.getMachineMemOperand(
5278 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5279 BaseAlignment: DAG.getEVTAlign(MemoryVT: MemVT), AAInfo: AAMDNodes(), Ranges: nullptr, SSID, Ordering);
5280
5281 SDValue L =
5282 DAG.getAtomic(Opcode: NT, dl, MemVT, Chain: InChain,
5283 Ptr: getValue(V: I.getPointerOperand()), Val: getValue(V: I.getValOperand()),
5284 MMO);
5285
5286 SDValue OutChain = L.getValue(R: 1);
5287
5288 setValue(V: &I, NewN: L);
5289 DAG.setRoot(OutChain);
5290}
5291
5292void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5293 SDLoc dl = getCurSDLoc();
5294 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5295 SDValue Ops[3];
5296 Ops[0] = getRoot();
5297 Ops[1] = DAG.getTargetConstant(Val: (unsigned)I.getOrdering(), DL: dl,
5298 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5299 Ops[2] = DAG.getTargetConstant(Val: I.getSyncScopeID(), DL: dl,
5300 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5301 SDValue N = DAG.getNode(Opcode: ISD::ATOMIC_FENCE, DL: dl, VT: MVT::Other, Ops);
5302 setValue(V: &I, NewN: N);
5303 DAG.setRoot(N);
5304}
5305
5306void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5307 SDLoc dl = getCurSDLoc();
5308 AtomicOrdering Order = I.getOrdering();
5309 SyncScope::ID SSID = I.getSyncScopeID();
5310
5311 SDValue InChain = getRoot();
5312
5313 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5314 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5315 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5316
5317 if (!TLI.supportsUnalignedAtomics() &&
5318 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5319 report_fatal_error(reason: "Cannot generate unaligned atomic load");
5320
5321 auto Flags = TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
5322
5323 const MDNode *Ranges = getRangeMetadata(I);
5324 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5325 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5326 BaseAlignment: I.getAlign(), AAInfo: AAMDNodes(), Ranges, SSID, Ordering: Order);
5327
5328 InChain = TLI.prepareVolatileOrAtomicLoad(Chain: InChain, DL: dl, DAG);
5329
5330 SDValue Ptr = getValue(V: I.getPointerOperand());
5331 SDValue L =
5332 DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT, VT: MemVT, Chain: InChain, Ptr, MMO);
5333
5334 SDValue OutChain = L.getValue(R: 1);
5335 if (MemVT != VT)
5336 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT);
5337
5338 setValue(V: &I, NewN: L);
5339 DAG.setRoot(OutChain);
5340}
5341
5342void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5343 SDLoc dl = getCurSDLoc();
5344
5345 AtomicOrdering Ordering = I.getOrdering();
5346 SyncScope::ID SSID = I.getSyncScopeID();
5347
5348 SDValue InChain = getRoot();
5349
5350 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5351 EVT MemVT =
5352 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getValueOperand()->getType());
5353
5354 if (!TLI.supportsUnalignedAtomics() &&
5355 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5356 report_fatal_error(reason: "Cannot generate unaligned atomic store");
5357
5358 auto Flags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
5359
5360 MachineFunction &MF = DAG.getMachineFunction();
5361 MachineMemOperand *MMO = MF.getMachineMemOperand(
5362 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5363 BaseAlignment: I.getAlign(), AAInfo: AAMDNodes(), Ranges: nullptr, SSID, Ordering);
5364
5365 SDValue Val = getValue(V: I.getValueOperand());
5366 if (Val.getValueType() != MemVT)
5367 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVT);
5368 SDValue Ptr = getValue(V: I.getPointerOperand());
5369
5370 SDValue OutChain =
5371 DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT, Chain: InChain, Ptr: Val, Val: Ptr, MMO);
5372
5373 setValue(V: &I, NewN: OutChain);
5374 DAG.setRoot(OutChain);
5375}
5376
5377/// Check if this intrinsic call depends on the chain (1st return value)
5378/// and if it only *loads* memory.
5379/// Ignore the callsite's attributes. A specific call site may be marked with
5380/// readnone, but the lowering code will expect the chain based on the
5381/// definition.
5382std::pair<bool, bool>
5383SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {
5384 const Function *F = I.getCalledFunction();
5385 bool HasChain = !F->doesNotAccessMemory();
5386 bool OnlyLoad =
5387 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5388
5389 return {HasChain, OnlyLoad};
5390}
5391
5392SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
5393 const CallBase &I, bool HasChain, bool OnlyLoad,
5394 TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
5395 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5396
5397 // Build the operand list.
5398 SmallVector<SDValue, 8> Ops;
5399 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5400 if (OnlyLoad) {
5401 // We don't need to serialize loads against other loads.
5402 Ops.push_back(Elt: DAG.getRoot());
5403 } else {
5404 Ops.push_back(Elt: getRoot());
5405 }
5406 }
5407
5408 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5409 if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||
5410 TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)
5411 Ops.push_back(Elt: DAG.getTargetConstant(Val: I.getIntrinsicID(), DL: getCurSDLoc(),
5412 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
5413
5414 // Add all operands of the call to the operand list.
5415 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5416 const Value *Arg = I.getArgOperand(i);
5417 if (!I.paramHasAttr(ArgNo: i, Kind: Attribute::ImmArg)) {
5418 Ops.push_back(Elt: getValue(V: Arg));
5419 continue;
5420 }
5421
5422 // Use TargetConstant instead of a regular constant for immarg.
5423 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: Arg->getType(), AllowUnknown: true);
5424 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg)) {
5425 assert(CI->getBitWidth() <= 64 &&
5426 "large intrinsic immediates not handled");
5427 Ops.push_back(Elt: DAG.getTargetConstant(Val: *CI, DL: SDLoc(), VT));
5428 } else {
5429 Ops.push_back(
5430 Elt: DAG.getTargetConstantFP(Val: *cast<ConstantFP>(Val: Arg), DL: SDLoc(), VT));
5431 }
5432 }
5433
5434 if (std::optional<OperandBundleUse> Bundle =
5435 I.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
5436 auto *Sym = Bundle->Inputs[0].get();
5437 SDValue SDSym = getValue(V: Sym);
5438 SDSym = DAG.getDeactivationSymbol(GV: cast<GlobalValue>(Val: Sym));
5439 Ops.push_back(Elt: SDSym);
5440 }
5441
5442 if (std::optional<OperandBundleUse> Bundle =
5443 I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
5444 Value *Token = Bundle->Inputs[0].get();
5445 SDValue ConvControlToken = getValue(V: Token);
5446 assert(Ops.back().getValueType() != MVT::Glue &&
5447 "Did not expect another glue node here.");
5448 ConvControlToken =
5449 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: {}, VT: MVT::Glue, Operand: ConvControlToken);
5450 Ops.push_back(Elt: ConvControlToken);
5451 }
5452
5453 return Ops;
5454}
5455
5456SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
5457 bool HasChain) {
5458 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5459
5460 SmallVector<EVT, 4> ValueVTs;
5461 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
5462
5463 if (HasChain)
5464 ValueVTs.push_back(Elt: MVT::Other);
5465
5466 return DAG.getVTList(VTs: ValueVTs);
5467}
5468
5469/// Get an INTRINSIC node for a target intrinsic which does not touch memory.
5470SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5471 const Type &IntrinsicVT, bool HasChain, ArrayRef<SDValue> Ops,
5472 const SDVTList &VTs) {
5473 if (!HasChain)
5474 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5475 if (!IntrinsicVT.isVoidTy())
5476 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5477 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops);
5478}
5479
5480/// Set root, convert return type if necessary and check alignment.
5481SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
5482 bool HasChain,
5483 bool OnlyLoad,
5484 SDValue Result) {
5485 if (HasChain) {
5486 SDValue Chain = Result.getValue(R: Result.getNode()->getNumValues() - 1);
5487 if (OnlyLoad)
5488 PendingLoads.push_back(Elt: Chain);
5489 else
5490 DAG.setRoot(Chain);
5491 }
5492
5493 if (I.getType()->isVoidTy())
5494 return Result;
5495
5496 if (MaybeAlign Alignment = I.getRetAlign(); InsertAssertAlign && Alignment) {
5497 // Insert `assertalign` node if there's an alignment.
5498 Result = DAG.getAssertAlign(DL: getCurSDLoc(), V: Result, A: Alignment.valueOrOne());
5499 } else if (!isa<VectorType>(Val: I.getType())) {
5500 Result = lowerRangeToAssertZExt(DAG, I, Op: Result);
5501 }
5502
5503 return Result;
5504}
5505
5506/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5507/// node.
5508void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5509 unsigned Intrinsic) {
5510 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
5511
5512 // Infos is set by getTgtMemIntrinsic.
5513 SmallVector<TargetLowering::IntrinsicInfo> Infos;
5514 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5515 TLI.getTgtMemIntrinsic(Infos, I, MF&: DAG.getMachineFunction(), Intrinsic);
5516 // Use the first (primary) info determines the node opcode.
5517 TargetLowering::IntrinsicInfo *Info = !Infos.empty() ? &Infos[0] : nullptr;
5518
5519 SmallVector<SDValue, 8> Ops =
5520 getTargetIntrinsicOperands(I, HasChain, OnlyLoad, TgtMemIntrinsicInfo: Info);
5521 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
5522
5523 // Propagate fast-math-flags from IR to node(s).
5524 SDNodeFlags Flags;
5525 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
5526 Flags.copyFMF(FPMO: *FPMO);
5527 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5528
5529 // Create the node.
5530 SDValue Result;
5531
5532 // In some cases, custom collection of operands from CallInst I may be needed.
5533 TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);
5534 if (!Infos.empty()) {
5535 // This is target intrinsic that touches memory
5536 // Create MachineMemOperands for each memory access described by the target.
5537 MachineFunction &MF = DAG.getMachineFunction();
5538 SmallVector<MachineMemOperand *> MMOs;
5539 for (const auto &Info : Infos) {
5540 // TODO: We currently just fallback to address space 0 if
5541 // getTgtMemIntrinsic didn't yield anything useful.
5542 MachinePointerInfo MPI;
5543 if (Info.ptrVal)
5544 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5545 else if (Info.fallbackAddressSpace)
5546 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5547 EVT MemVT = Info.memVT;
5548 LocationSize Size = LocationSize::precise(Value: Info.size);
5549 if (Size.hasValue() && !Size.getValue())
5550 Size = LocationSize::precise(Value: MemVT.getStoreSize());
5551 Align Alignment = Info.align.value_or(u: DAG.getEVTAlign(MemoryVT: MemVT));
5552 MachineMemOperand *MMO = MF.getMachineMemOperand(
5553 PtrInfo: MPI, F: Info.flags, Size, BaseAlignment: Alignment, AAInfo: I.getAAMetadata(),
5554 /*Ranges=*/nullptr, SSID: Info.ssid, Ordering: Info.order, FailureOrdering: Info.failureOrder);
5555 MMOs.push_back(Elt: MMO);
5556 }
5557
5558 Result = DAG.getMemIntrinsicNode(Opcode: Info->opc, dl: getCurSDLoc(), VTList: VTs, Ops,
5559 MemVT: Info->memVT, MMOs);
5560 } else {
5561 Result = getTargetNonMemIntrinsicNode(IntrinsicVT: *I.getType(), HasChain, Ops, VTs);
5562 }
5563
5564 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
5565
5566 setValue(V: &I, NewN: Result);
5567}
5568
5569/// GetSignificand - Get the significand and build it into a floating-point
5570/// number with exponent of 1:
5571///
5572/// Op = (Op & 0x007fffff) | 0x3f800000;
5573///
5574/// where Op is the hexadecimal representation of floating point value.
5575static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
5576 SDValue t1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Op,
5577 N2: DAG.getConstant(Val: 0x007fffff, DL: dl, VT: MVT::i32));
5578 SDValue t2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: t1,
5579 N2: DAG.getConstant(Val: 0x3f800000, DL: dl, VT: MVT::i32));
5580 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: t2);
5581}
5582
5583/// GetExponent - Get the exponent:
5584///
5585/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5586///
5587/// where Op is the hexadecimal representation of floating point value.
5588static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
5589 const TargetLowering &TLI, const SDLoc &dl) {
5590 SDValue t0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Op,
5591 N2: DAG.getConstant(Val: 0x7f800000, DL: dl, VT: MVT::i32));
5592 SDValue t1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: t0,
5593 N2: DAG.getShiftAmountConstant(Val: 23, VT: MVT::i32, DL: dl));
5594 SDValue t2 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: t1,
5595 N2: DAG.getConstant(Val: 127, DL: dl, VT: MVT::i32));
5596 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::f32, Operand: t2);
5597}
5598
5599/// getF32Constant - Get 32-bit floating point constant.
5600static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
5601 const SDLoc &dl) {
5602 return DAG.getConstantFP(Val: APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), DL: dl,
5603 VT: MVT::f32);
5604}
5605
5606static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
5607 SelectionDAG &DAG) {
5608 // TODO: What fast-math-flags should be set on the floating-point nodes?
5609
5610 // IntegerPartOfX = ((int32_t)(t0);
5611 SDValue IntegerPartOfX = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: t0);
5612
5613 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5614 SDValue t1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::f32, Operand: IntegerPartOfX);
5615 SDValue X = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0, N2: t1);
5616
5617 // IntegerPartOfX <<= 23;
5618 IntegerPartOfX = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: IntegerPartOfX,
5619 N2: DAG.getShiftAmountConstant(Val: 23, VT: MVT::i32, DL: dl));
5620
5621 SDValue TwoToFractionalPartOfX;
5622 if (LimitFloatPrecision <= 6) {
5623 // For floating-point precision of 6:
5624 //
5625 // TwoToFractionalPartOfX =
5626 // 0.997535578f +
5627 // (0.735607626f + 0.252464424f * x) * x;
5628 //
5629 // error 0.0144103317, which is 6 bits
5630 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5631 N2: getF32Constant(DAG, Flt: 0x3e814304, dl));
5632 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5633 N2: getF32Constant(DAG, Flt: 0x3f3c50c8, dl));
5634 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5635 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5636 N2: getF32Constant(DAG, Flt: 0x3f7f5e7e, dl));
5637 } else if (LimitFloatPrecision <= 12) {
5638 // For floating-point precision of 12:
5639 //
5640 // TwoToFractionalPartOfX =
5641 // 0.999892986f +
5642 // (0.696457318f +
5643 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5644 //
5645 // error 0.000107046256, which is 13 to 14 bits
5646 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5647 N2: getF32Constant(DAG, Flt: 0x3da235e3, dl));
5648 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5649 N2: getF32Constant(DAG, Flt: 0x3e65b8f3, dl));
5650 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5651 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5652 N2: getF32Constant(DAG, Flt: 0x3f324b07, dl));
5653 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5654 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5655 N2: getF32Constant(DAG, Flt: 0x3f7ff8fd, dl));
5656 } else { // LimitFloatPrecision <= 18
5657 // For floating-point precision of 18:
5658 //
5659 // TwoToFractionalPartOfX =
5660 // 0.999999982f +
5661 // (0.693148872f +
5662 // (0.240227044f +
5663 // (0.554906021e-1f +
5664 // (0.961591928e-2f +
5665 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5666 // error 2.47208000*10^(-7), which is better than 18 bits
5667 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5668 N2: getF32Constant(DAG, Flt: 0x3924b03e, dl));
5669 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5670 N2: getF32Constant(DAG, Flt: 0x3ab24b87, dl));
5671 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5672 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5673 N2: getF32Constant(DAG, Flt: 0x3c1d8c17, dl));
5674 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5675 SDValue t7 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5676 N2: getF32Constant(DAG, Flt: 0x3d634a1d, dl));
5677 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5678 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
5679 N2: getF32Constant(DAG, Flt: 0x3e75fe14, dl));
5680 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
5681 SDValue t11 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t10,
5682 N2: getF32Constant(DAG, Flt: 0x3f317234, dl));
5683 SDValue t12 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t11, N2: X);
5684 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t12,
5685 N2: getF32Constant(DAG, Flt: 0x3f800000, dl));
5686 }
5687
5688 // Add the exponent into the result in integer domain.
5689 SDValue t13 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: TwoToFractionalPartOfX);
5690 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32,
5691 Operand: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: t13, N2: IntegerPartOfX));
5692}
5693
5694/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5695/// limited-precision mode.
5696static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5697 const TargetLowering &TLI, SDNodeFlags Flags) {
5698 if (Op.getValueType() == MVT::f32 &&
5699 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5700
5701 // Put the exponent in the right bit position for later addition to the
5702 // final result:
5703 //
5704 // t0 = Op * log2(e)
5705
5706 // TODO: What fast-math-flags should be set here?
5707 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Op,
5708 N2: DAG.getConstantFP(Val: numbers::log2ef, DL: dl, VT: MVT::f32));
5709 return getLimitedPrecisionExp2(t0, dl, DAG);
5710 }
5711
5712 // No special expansion.
5713 return DAG.getNode(Opcode: ISD::FEXP, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5714}
5715
5716/// expandLog - Lower a log intrinsic. Handles the special sequences for
5717/// limited-precision mode.
5718static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5719 const TargetLowering &TLI, SDNodeFlags Flags) {
5720 // TODO: What fast-math-flags should be set on the floating-point nodes?
5721
5722 if (Op.getValueType() == MVT::f32 &&
5723 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5724 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
5725
5726 // Scale the exponent by log(2).
5727 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
5728 SDValue LogOfExponent =
5729 DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Exp,
5730 N2: DAG.getConstantFP(Val: numbers::ln2f, DL: dl, VT: MVT::f32));
5731
5732 // Get the significand and build it into a floating-point number with
5733 // exponent of 1.
5734 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5735
5736 SDValue LogOfMantissa;
5737 if (LimitFloatPrecision <= 6) {
5738 // For floating-point precision of 6:
5739 //
5740 // LogofMantissa =
5741 // -1.1609546f +
5742 // (1.4034025f - 0.23903021f * x) * x;
5743 //
5744 // error 0.0034276066, which is better than 8 bits
5745 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5746 N2: getF32Constant(DAG, Flt: 0xbe74c456, dl));
5747 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5748 N2: getF32Constant(DAG, Flt: 0x3fb3a2b1, dl));
5749 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5750 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5751 N2: getF32Constant(DAG, Flt: 0x3f949a29, dl));
5752 } else if (LimitFloatPrecision <= 12) {
5753 // For floating-point precision of 12:
5754 //
5755 // LogOfMantissa =
5756 // -1.7417939f +
5757 // (2.8212026f +
5758 // (-1.4699568f +
5759 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5760 //
5761 // error 0.000061011436, which is 14 bits
5762 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5763 N2: getF32Constant(DAG, Flt: 0xbd67b6d6, dl));
5764 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5765 N2: getF32Constant(DAG, Flt: 0x3ee4f4b8, dl));
5766 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5767 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5768 N2: getF32Constant(DAG, Flt: 0x3fbc278b, dl));
5769 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5770 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5771 N2: getF32Constant(DAG, Flt: 0x40348e95, dl));
5772 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5773 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
5774 N2: getF32Constant(DAG, Flt: 0x3fdef31a, dl));
5775 } else { // LimitFloatPrecision <= 18
5776 // For floating-point precision of 18:
5777 //
5778 // LogOfMantissa =
5779 // -2.1072184f +
5780 // (4.2372794f +
5781 // (-3.7029485f +
5782 // (2.2781945f +
5783 // (-0.87823314f +
5784 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5785 //
5786 // error 0.0000023660568, which is better than 18 bits
5787 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5788 N2: getF32Constant(DAG, Flt: 0xbc91e5ac, dl));
5789 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5790 N2: getF32Constant(DAG, Flt: 0x3e4350aa, dl));
5791 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5792 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5793 N2: getF32Constant(DAG, Flt: 0x3f60d3e3, dl));
5794 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5795 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5796 N2: getF32Constant(DAG, Flt: 0x4011cdf0, dl));
5797 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5798 SDValue t7 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
5799 N2: getF32Constant(DAG, Flt: 0x406cfd1c, dl));
5800 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5801 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
5802 N2: getF32Constant(DAG, Flt: 0x408797cb, dl));
5803 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
5804 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t10,
5805 N2: getF32Constant(DAG, Flt: 0x4006dcab, dl));
5806 }
5807
5808 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: LogOfMantissa);
5809 }
5810
5811 // No special expansion.
5812 return DAG.getNode(Opcode: ISD::FLOG, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5813}
5814
5815/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5816/// limited-precision mode.
5817static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5818 const TargetLowering &TLI, SDNodeFlags Flags) {
5819 // TODO: What fast-math-flags should be set on the floating-point nodes?
5820
5821 if (Op.getValueType() == MVT::f32 &&
5822 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5823 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
5824
5825 // Get the exponent.
5826 SDValue LogOfExponent = GetExponent(DAG, Op: Op1, TLI, dl);
5827
5828 // Get the significand and build it into a floating-point number with
5829 // exponent of 1.
5830 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5831
5832 // Different possible minimax approximations of significand in
5833 // floating-point for various degrees of accuracy over [1,2].
5834 SDValue Log2ofMantissa;
5835 if (LimitFloatPrecision <= 6) {
5836 // For floating-point precision of 6:
5837 //
5838 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5839 //
5840 // error 0.0049451742, which is more than 7 bits
5841 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5842 N2: getF32Constant(DAG, Flt: 0xbeb08fe0, dl));
5843 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5844 N2: getF32Constant(DAG, Flt: 0x40019463, dl));
5845 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5846 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5847 N2: getF32Constant(DAG, Flt: 0x3fd6633d, dl));
5848 } else if (LimitFloatPrecision <= 12) {
5849 // For floating-point precision of 12:
5850 //
5851 // Log2ofMantissa =
5852 // -2.51285454f +
5853 // (4.07009056f +
5854 // (-2.12067489f +
5855 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5856 //
5857 // error 0.0000876136000, which is better than 13 bits
5858 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5859 N2: getF32Constant(DAG, Flt: 0xbda7262e, dl));
5860 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5861 N2: getF32Constant(DAG, Flt: 0x3f25280b, dl));
5862 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5863 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5864 N2: getF32Constant(DAG, Flt: 0x4007b923, dl));
5865 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5866 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5867 N2: getF32Constant(DAG, Flt: 0x40823e2f, dl));
5868 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5869 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
5870 N2: getF32Constant(DAG, Flt: 0x4020d29c, dl));
5871 } else { // LimitFloatPrecision <= 18
5872 // For floating-point precision of 18:
5873 //
5874 // Log2ofMantissa =
5875 // -3.0400495f +
5876 // (6.1129976f +
5877 // (-5.3420409f +
5878 // (3.2865683f +
5879 // (-1.2669343f +
5880 // (0.27515199f -
5881 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5882 //
5883 // error 0.0000018516, which is better than 18 bits
5884 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5885 N2: getF32Constant(DAG, Flt: 0xbcd2769e, dl));
5886 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5887 N2: getF32Constant(DAG, Flt: 0x3e8ce0b9, dl));
5888 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5889 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5890 N2: getF32Constant(DAG, Flt: 0x3fa22ae7, dl));
5891 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5892 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5893 N2: getF32Constant(DAG, Flt: 0x40525723, dl));
5894 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5895 SDValue t7 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
5896 N2: getF32Constant(DAG, Flt: 0x40aaf200, dl));
5897 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5898 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
5899 N2: getF32Constant(DAG, Flt: 0x40c39dad, dl));
5900 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
5901 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t10,
5902 N2: getF32Constant(DAG, Flt: 0x4042902c, dl));
5903 }
5904
5905 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: Log2ofMantissa);
5906 }
5907
5908 // No special expansion.
5909 return DAG.getNode(Opcode: ISD::FLOG2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5910}
5911
5912/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5913/// limited-precision mode.
5914static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5915 const TargetLowering &TLI, SDNodeFlags Flags) {
5916 // TODO: What fast-math-flags should be set on the floating-point nodes?
5917
5918 if (Op.getValueType() == MVT::f32 &&
5919 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5920 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
5921
5922 // Scale the exponent by log10(2) [0.30102999f].
5923 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
5924 SDValue LogOfExponent = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Exp,
5925 N2: getF32Constant(DAG, Flt: 0x3e9a209a, dl));
5926
5927 // Get the significand and build it into a floating-point number with
5928 // exponent of 1.
5929 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5930
5931 SDValue Log10ofMantissa;
5932 if (LimitFloatPrecision <= 6) {
5933 // For floating-point precision of 6:
5934 //
5935 // Log10ofMantissa =
5936 // -0.50419619f +
5937 // (0.60948995f - 0.10380950f * x) * x;
5938 //
5939 // error 0.0014886165, which is 6 bits
5940 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5941 N2: getF32Constant(DAG, Flt: 0xbdd49a13, dl));
5942 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5943 N2: getF32Constant(DAG, Flt: 0x3f1c0789, dl));
5944 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5945 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5946 N2: getF32Constant(DAG, Flt: 0x3f011300, dl));
5947 } else if (LimitFloatPrecision <= 12) {
5948 // For floating-point precision of 12:
5949 //
5950 // Log10ofMantissa =
5951 // -0.64831180f +
5952 // (0.91751397f +
5953 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5954 //
5955 // error 0.00019228036, which is better than 12 bits
5956 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5957 N2: getF32Constant(DAG, Flt: 0x3d431f31, dl));
5958 SDValue t1 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0,
5959 N2: getF32Constant(DAG, Flt: 0x3ea21fb2, dl));
5960 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5961 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5962 N2: getF32Constant(DAG, Flt: 0x3f6ae232, dl));
5963 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5964 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t4,
5965 N2: getF32Constant(DAG, Flt: 0x3f25f7c3, dl));
5966 } else { // LimitFloatPrecision <= 18
5967 // For floating-point precision of 18:
5968 //
5969 // Log10ofMantissa =
5970 // -0.84299375f +
5971 // (1.5327582f +
5972 // (-1.0688956f +
5973 // (0.49102474f +
5974 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5975 //
5976 // error 0.0000037995730, which is better than 18 bits
5977 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5978 N2: getF32Constant(DAG, Flt: 0x3c5d51ce, dl));
5979 SDValue t1 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0,
5980 N2: getF32Constant(DAG, Flt: 0x3e00685a, dl));
5981 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5982 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5983 N2: getF32Constant(DAG, Flt: 0x3efb6798, dl));
5984 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5985 SDValue t5 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t4,
5986 N2: getF32Constant(DAG, Flt: 0x3f88d192, dl));
5987 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5988 SDValue t7 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5989 N2: getF32Constant(DAG, Flt: 0x3fc4316c, dl));
5990 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5991 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t8,
5992 N2: getF32Constant(DAG, Flt: 0x3f57ce70, dl));
5993 }
5994
5995 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: Log10ofMantissa);
5996 }
5997
5998 // No special expansion.
5999 return DAG.getNode(Opcode: ISD::FLOG10, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6000}
6001
6002/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
6003/// limited-precision mode.
6004static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
6005 const TargetLowering &TLI, SDNodeFlags Flags) {
6006 if (Op.getValueType() == MVT::f32 &&
6007 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
6008 return getLimitedPrecisionExp2(t0: Op, dl, DAG);
6009
6010 // No special expansion.
6011 return DAG.getNode(Opcode: ISD::FEXP2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6012}
6013
6014/// visitPow - Lower a pow intrinsic. Handles the special sequences for
6015/// limited-precision mode with x == 10.0f.
6016static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
6017 SelectionDAG &DAG, const TargetLowering &TLI,
6018 SDNodeFlags Flags) {
6019 bool IsExp10 = false;
6020 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
6021 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
6022 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
6023 APFloat Ten(10.0f);
6024 IsExp10 = LHSC->isExactlyValue(V: Ten);
6025 }
6026 }
6027
6028 // TODO: What fast-math-flags should be set on the FMUL node?
6029 if (IsExp10) {
6030 // Put the exponent in the right bit position for later addition to the
6031 // final result:
6032 //
6033 // #define LOG2OF10 3.3219281f
6034 // t0 = Op * LOG2OF10;
6035 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: RHS,
6036 N2: getF32Constant(DAG, Flt: 0x40549a78, dl));
6037 return getLimitedPrecisionExp2(t0, dl, DAG);
6038 }
6039
6040 // No special expansion.
6041 return DAG.getNode(Opcode: ISD::FPOW, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags);
6042}
6043
6044/// ExpandPowI - Expand a llvm.powi intrinsic.
6045static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
6046 SelectionDAG &DAG) {
6047 // If RHS is a constant, we can expand this out to a multiplication tree if
6048 // it's beneficial on the target, otherwise we end up lowering to a call to
6049 // __powidf2 (for example).
6050 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
6051 unsigned Val = RHSC->getSExtValue();
6052
6053 // powi(x, 0) -> 1.0
6054 if (Val == 0)
6055 return DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType());
6056
6057 if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
6058 Exponent: Val, OptForSize: DAG.shouldOptForSize())) {
6059 // Get the exponent as a positive value.
6060 if ((int)Val < 0)
6061 Val = -Val;
6062 // We use the simple binary decomposition method to generate the multiply
6063 // sequence. There are more optimal ways to do this (for example,
6064 // powi(x,15) generates one more multiply than it should), but this has
6065 // the benefit of being both really simple and much better than a libcall.
6066 SDValue Res; // Logically starts equal to 1.0
6067 SDValue CurSquare = LHS;
6068 // TODO: Intrinsics should have fast-math-flags that propagate to these
6069 // nodes.
6070 while (Val) {
6071 if (Val & 1) {
6072 if (Res.getNode())
6073 Res =
6074 DAG.getNode(Opcode: ISD::FMUL, DL, VT: Res.getValueType(), N1: Res, N2: CurSquare);
6075 else
6076 Res = CurSquare; // 1.0*CurSquare.
6077 }
6078
6079 CurSquare = DAG.getNode(Opcode: ISD::FMUL, DL, VT: CurSquare.getValueType(),
6080 N1: CurSquare, N2: CurSquare);
6081 Val >>= 1;
6082 }
6083
6084 // If the original was negative, invert the result, producing 1/(x*x*x).
6085 if (RHSC->getSExtValue() < 0)
6086 Res = DAG.getNode(Opcode: ISD::FDIV, DL, VT: LHS.getValueType(),
6087 N1: DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType()), N2: Res);
6088 return Res;
6089 }
6090 }
6091
6092 // Otherwise, expand to a libcall.
6093 return DAG.getNode(Opcode: ISD::FPOWI, DL, VT: LHS.getValueType(), N1: LHS, N2: RHS);
6094}
6095
6096static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
6097 SDValue LHS, SDValue RHS, SDValue Scale,
6098 SelectionDAG &DAG, const TargetLowering &TLI) {
6099 EVT VT = LHS.getValueType();
6100 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
6101 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
6102 LLVMContext &Ctx = *DAG.getContext();
6103
6104 // If the type is legal but the operation isn't, this node might survive all
6105 // the way to operation legalization. If we end up there and we do not have
6106 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
6107 // node.
6108
6109 // Coax the legalizer into expanding the node during type legalization instead
6110 // by bumping the size by one bit. This will force it to Promote, enabling the
6111 // early expansion and avoiding the need to expand later.
6112
6113 // We don't have to do this if Scale is 0; that can always be expanded, unless
6114 // it's a saturating signed operation. Those can experience true integer
6115 // division overflow, a case which we must avoid.
6116
6117 // FIXME: We wouldn't have to do this (or any of the early
6118 // expansion/promotion) if it was possible to expand a libcall of an
6119 // illegal type during operation legalization. But it's not, so things
6120 // get a bit hacky.
6121 unsigned ScaleInt = Scale->getAsZExtVal();
6122 if ((ScaleInt > 0 || (Saturating && Signed)) &&
6123 (TLI.isTypeLegal(VT) ||
6124 (VT.isVector() && TLI.isTypeLegal(VT: VT.getVectorElementType())))) {
6125 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
6126 Op: Opcode, VT, Scale: ScaleInt);
6127 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
6128 EVT PromVT;
6129 if (VT.isScalarInteger())
6130 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits() + 1);
6131 else if (VT.isVector()) {
6132 PromVT = VT.getVectorElementType();
6133 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PromVT.getSizeInBits() + 1);
6134 PromVT = EVT::getVectorVT(Context&: Ctx, VT: PromVT, EC: VT.getVectorElementCount());
6135 } else
6136 llvm_unreachable("Wrong VT for DIVFIX?");
6137 LHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL, VT: PromVT);
6138 RHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL, VT: PromVT);
6139 EVT ShiftTy = TLI.getShiftAmountTy(LHSTy: PromVT, DL: DAG.getDataLayout());
6140 // For saturating operations, we need to shift up the LHS to get the
6141 // proper saturation width, and then shift down again afterwards.
6142 if (Saturating)
6143 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: PromVT, N1: LHS,
6144 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
6145 SDValue Res = DAG.getNode(Opcode, DL, VT: PromVT, N1: LHS, N2: RHS, N3: Scale);
6146 if (Saturating)
6147 Res = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: PromVT, N1: Res,
6148 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
6149 return DAG.getZExtOrTrunc(Op: Res, DL, VT);
6150 }
6151 }
6152
6153 return DAG.getNode(Opcode, DL, VT, N1: LHS, N2: RHS, N3: Scale);
6154}
6155
6156// getUnderlyingArgRegs - Find underlying registers used for a truncated,
6157// bitcasted, or split argument. Returns a list of <Register, size in bits>
6158static void
6159getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
6160 const SDValue &N) {
6161 switch (N.getOpcode()) {
6162 case ISD::CopyFromReg: {
6163 SDValue Op = N.getOperand(i: 1);
6164 Regs.emplace_back(Args: cast<RegisterSDNode>(Val&: Op)->getReg(),
6165 Args: Op.getValueType().getSizeInBits());
6166 return;
6167 }
6168 case ISD::BITCAST:
6169 case ISD::AssertZext:
6170 case ISD::AssertSext:
6171 case ISD::TRUNCATE:
6172 getUnderlyingArgRegs(Regs, N: N.getOperand(i: 0));
6173 return;
6174 case ISD::BUILD_PAIR:
6175 case ISD::BUILD_VECTOR:
6176 case ISD::CONCAT_VECTORS:
6177 for (SDValue Op : N->op_values())
6178 getUnderlyingArgRegs(Regs, N: Op);
6179 return;
6180 default:
6181 return;
6182 }
6183}
6184
6185/// If the DbgValueInst is a dbg_value of a function argument, create the
6186/// corresponding DBG_VALUE machine instruction for it now. At the end of
6187/// instruction selection, they will be inserted to the entry BB.
6188/// We don't currently support this for variadic dbg_values, as they shouldn't
6189/// appear for function arguments or in the prologue.
6190bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6191 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6192 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6193 const Argument *Arg = dyn_cast<Argument>(Val: V);
6194 if (!Arg)
6195 return false;
6196
6197 MachineFunction &MF = DAG.getMachineFunction();
6198 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6199
6200 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6201 // we've been asked to pursue.
6202 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6203 bool Indirect) {
6204 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6205 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6206 // pointing at the VReg, which will be patched up later.
6207 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_INSTR_REF);
6208 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
6209 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6210 /* isKill */ false, /* isDead */ false,
6211 /* isUndef */ false, /* isEarlyClobber */ false,
6212 /* SubReg */ 0, /* isDebug */ true)});
6213
6214 auto *NewDIExpr = FragExpr;
6215 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6216 // the DIExpression.
6217 if (Indirect)
6218 NewDIExpr = DIExpression::prepend(Expr: FragExpr, Flags: DIExpression::DerefBefore);
6219 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
6220 NewDIExpr = DIExpression::prependOpcodes(Expr: NewDIExpr, Ops);
6221 return BuildMI(MF, DL, MCID: Inst, IsIndirect: false, MOs, Variable, Expr: NewDIExpr);
6222 } else {
6223 // Create a completely standard DBG_VALUE.
6224 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_VALUE);
6225 return BuildMI(MF, DL, MCID: Inst, IsIndirect: Indirect, Reg, Variable, Expr: FragExpr);
6226 }
6227 };
6228
6229 if (Kind == FuncArgumentDbgValueKind::Value) {
6230 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6231 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6232 // the entry block.
6233 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6234 if (!IsInEntryBlock)
6235 return false;
6236
6237 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6238 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6239 // variable that also is a param.
6240 //
6241 // Although, if we are at the top of the entry block already, we can still
6242 // emit using ArgDbgValue. This might catch some situations when the
6243 // dbg.value refers to an argument that isn't used in the entry block, so
6244 // any CopyToReg node would be optimized out and the only way to express
6245 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6246 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6247 // we should only emit as ArgDbgValue if the Variable is an argument to the
6248 // current function, and the dbg.value intrinsic is found in the entry
6249 // block.
6250 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6251 !DL->getInlinedAt();
6252 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6253 if (!IsInPrologue && !VariableIsFunctionInputArg)
6254 return false;
6255
6256 // Here we assume that a function argument on IR level only can be used to
6257 // describe one input parameter on source level. If we for example have
6258 // source code like this
6259 //
6260 // struct A { long x, y; };
6261 // void foo(struct A a, long b) {
6262 // ...
6263 // b = a.x;
6264 // ...
6265 // }
6266 //
6267 // and IR like this
6268 //
6269 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6270 // entry:
6271 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6272 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6273 // call void @llvm.dbg.value(metadata i32 %b, "b",
6274 // ...
6275 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6276 // ...
6277 //
6278 // then the last dbg.value is describing a parameter "b" using a value that
6279 // is an argument. But since we already has used %a1 to describe a parameter
6280 // we should not handle that last dbg.value here (that would result in an
6281 // incorrect hoisting of the DBG_VALUE to the function entry).
6282 // Notice that we allow one dbg.value per IR level argument, to accommodate
6283 // for the situation with fragments above.
6284 // If there is no node for the value being handled, we return true to skip
6285 // the normal generation of debug info, as it would kill existing debug
6286 // info for the parameter in case of duplicates.
6287 if (VariableIsFunctionInputArg) {
6288 unsigned ArgNo = Arg->getArgNo();
6289 if (ArgNo >= FuncInfo.DescribedArgs.size())
6290 FuncInfo.DescribedArgs.resize(N: ArgNo + 1, t: false);
6291 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(Idx: ArgNo))
6292 return !NodeMap[V].getNode();
6293 FuncInfo.DescribedArgs.set(ArgNo);
6294 }
6295 }
6296
6297 bool IsIndirect = false;
6298 std::optional<MachineOperand> Op;
6299 // Some arguments' frame index is recorded during argument lowering.
6300 int FI = FuncInfo.getArgumentFrameIndex(A: Arg);
6301 if (FI != std::numeric_limits<int>::max())
6302 Op = MachineOperand::CreateFI(Idx: FI);
6303
6304 SmallVector<std::pair<Register, TypeSize>, 8> ArgRegsAndSizes;
6305 if (!Op && N.getNode()) {
6306 getUnderlyingArgRegs(Regs&: ArgRegsAndSizes, N);
6307 Register Reg;
6308 if (ArgRegsAndSizes.size() == 1)
6309 Reg = ArgRegsAndSizes.front().first;
6310
6311 if (Reg && Reg.isVirtual()) {
6312 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6313 Register PR = RegInfo.getLiveInPhysReg(VReg: Reg);
6314 if (PR)
6315 Reg = PR;
6316 }
6317 if (Reg) {
6318 Op = MachineOperand::CreateReg(Reg, isDef: false);
6319 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6320 }
6321 }
6322
6323 if (!Op && N.getNode()) {
6324 // Check if frame index is available.
6325 SDValue LCandidate = peekThroughBitcasts(V: N);
6326 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(Val: LCandidate.getNode()))
6327 if (FrameIndexSDNode *FINode =
6328 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
6329 Op = MachineOperand::CreateFI(Idx: FINode->getIndex());
6330 }
6331
6332 if (!Op) {
6333 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6334 auto splitMultiRegDbgValue =
6335 [&](ArrayRef<std::pair<Register, TypeSize>> SplitRegs) -> bool {
6336 unsigned Offset = 0;
6337 for (const auto &[Reg, RegSizeInBits] : SplitRegs) {
6338 // FIXME: Scalable sizes are not supported in fragment expressions.
6339 if (RegSizeInBits.isScalable())
6340 return false;
6341
6342 // If the expression is already a fragment, the current register
6343 // offset+size might extend beyond the fragment. In this case, only
6344 // the register bits that are inside the fragment are relevant.
6345 int RegFragmentSizeInBits = RegSizeInBits.getFixedValue();
6346 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6347 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6348 // The register is entirely outside the expression fragment,
6349 // so is irrelevant for debug info.
6350 if (Offset >= ExprFragmentSizeInBits)
6351 break;
6352 // The register is partially outside the expression fragment, only
6353 // the low bits within the fragment are relevant for debug info.
6354 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6355 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6356 }
6357 }
6358
6359 auto FragmentExpr = DIExpression::createFragmentExpression(
6360 Expr, OffsetInBits: Offset, SizeInBits: RegFragmentSizeInBits);
6361 Offset += RegSizeInBits.getFixedValue();
6362 // If a valid fragment expression cannot be created, the variable's
6363 // correct value cannot be determined and so it is set as poison.
6364 if (!FragmentExpr) {
6365 SDDbgValue *SDV = DAG.getConstantDbgValue(
6366 Var: Variable, Expr, C: PoisonValue::get(T: V->getType()), DL, O: SDNodeOrder);
6367 DAG.AddDbgValue(DB: SDV, isParameter: false);
6368 continue;
6369 }
6370 MachineInstr *NewMI = MakeVRegDbgValue(
6371 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6372 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6373 }
6374
6375 return true;
6376 };
6377
6378 // Check if ValueMap has reg number.
6379 DenseMap<const Value *, Register>::const_iterator
6380 VMI = FuncInfo.ValueMap.find(Val: V);
6381 if (VMI != FuncInfo.ValueMap.end()) {
6382 const auto &TLI = DAG.getTargetLoweringInfo();
6383 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6384 V->getType(), std::nullopt);
6385 if (RFV.occupiesMultipleRegs())
6386 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6387
6388 Op = MachineOperand::CreateReg(Reg: VMI->second, isDef: false);
6389 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6390 } else if (ArgRegsAndSizes.size() > 1) {
6391 // This was split due to the calling convention, and no virtual register
6392 // mapping exists for the value.
6393 return splitMultiRegDbgValue(ArgRegsAndSizes);
6394 }
6395 }
6396
6397 if (!Op)
6398 return false;
6399
6400 assert(Variable->isValidLocationForIntrinsic(DL) &&
6401 "Expected inlined-at fields to agree");
6402 MachineInstr *NewMI = nullptr;
6403
6404 if (Op->isReg())
6405 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6406 else
6407 NewMI = BuildMI(MF, DL, MCID: TII->get(Opcode: TargetOpcode::DBG_VALUE), IsIndirect: true, MOs: *Op,
6408 Variable, Expr);
6409
6410 // Otherwise, use ArgDbgValues.
6411 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6412 return true;
6413}
6414
6415/// Return the appropriate SDDbgValue based on N.
6416SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6417 DILocalVariable *Variable,
6418 DIExpression *Expr,
6419 const DebugLoc &dl,
6420 unsigned DbgSDNodeOrder) {
6421 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
6422 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6423 // stack slot locations.
6424 //
6425 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6426 // debug values here after optimization:
6427 //
6428 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6429 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6430 //
6431 // Both describe the direct values of their associated variables.
6432 return DAG.getFrameIndexDbgValue(Var: Variable, Expr, FI: FISDN->getIndex(),
6433 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6434 }
6435 return DAG.getDbgValue(Var: Variable, Expr, N: N.getNode(), R: N.getResNo(),
6436 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6437}
6438
6439static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6440 switch (Intrinsic) {
6441 case Intrinsic::smul_fix:
6442 return ISD::SMULFIX;
6443 case Intrinsic::umul_fix:
6444 return ISD::UMULFIX;
6445 case Intrinsic::smul_fix_sat:
6446 return ISD::SMULFIXSAT;
6447 case Intrinsic::umul_fix_sat:
6448 return ISD::UMULFIXSAT;
6449 case Intrinsic::sdiv_fix:
6450 return ISD::SDIVFIX;
6451 case Intrinsic::udiv_fix:
6452 return ISD::UDIVFIX;
6453 case Intrinsic::sdiv_fix_sat:
6454 return ISD::SDIVFIXSAT;
6455 case Intrinsic::udiv_fix_sat:
6456 return ISD::UDIVFIXSAT;
6457 default:
6458 llvm_unreachable("Unhandled fixed point intrinsic");
6459 }
6460}
6461
6462/// Given a @llvm.call.preallocated.setup, return the corresponding
6463/// preallocated call.
6464static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6465 assert(cast<CallBase>(PreallocatedSetup)
6466 ->getCalledFunction()
6467 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6468 "expected call_preallocated_setup Value");
6469 for (const auto *U : PreallocatedSetup->users()) {
6470 auto *UseCall = cast<CallBase>(Val: U);
6471 const Function *Fn = UseCall->getCalledFunction();
6472 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6473 return UseCall;
6474 }
6475 }
6476 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6477}
6478
6479/// If DI is a debug value with an EntryValue expression, lower it using the
6480/// corresponding physical register of the associated Argument value
6481/// (guaranteed to exist by the verifier).
6482bool SelectionDAGBuilder::visitEntryValueDbgValue(
6483 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6484 DIExpression *Expr, DebugLoc DbgLoc) {
6485 if (!Expr->isEntryValue() || !hasSingleElement(C&: Values))
6486 return false;
6487
6488 // These properties are guaranteed by the verifier.
6489 const Argument *Arg = cast<Argument>(Val: Values[0]);
6490 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6491
6492 auto ArgIt = FuncInfo.ValueMap.find(Val: Arg);
6493 if (ArgIt == FuncInfo.ValueMap.end()) {
6494 LLVM_DEBUG(
6495 dbgs() << "Dropping dbg.value: expression is entry_value but "
6496 "couldn't find an associated register for the Argument\n");
6497 return true;
6498 }
6499 Register ArgVReg = ArgIt->getSecond();
6500
6501 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6502 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6503 SDDbgValue *SDV = DAG.getVRegDbgValue(
6504 Var: Variable, Expr, VReg: PhysReg, IsIndirect: false /*IsIndidrect*/, DL: DbgLoc, O: SDNodeOrder);
6505 DAG.AddDbgValue(DB: SDV, isParameter: false /*treat as dbg.declare byval parameter*/);
6506 return true;
6507 }
6508 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6509 "couldn't find a physical register\n");
6510 return true;
6511}
6512
6513/// Lower the call to the specified intrinsic function.
6514void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6515 unsigned Intrinsic) {
6516 SDLoc sdl = getCurSDLoc();
6517 switch (Intrinsic) {
6518 case Intrinsic::experimental_convergence_anchor:
6519 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_ANCHOR, DL: sdl, VT: MVT::Untyped));
6520 break;
6521 case Intrinsic::experimental_convergence_entry:
6522 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_ENTRY, DL: sdl, VT: MVT::Untyped));
6523 break;
6524 case Intrinsic::experimental_convergence_loop: {
6525 auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
6526 auto *Token = Bundle->Inputs[0].get();
6527 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_LOOP, DL: sdl, VT: MVT::Untyped,
6528 Operand: getValue(V: Token)));
6529 break;
6530 }
6531 }
6532}
6533
6534void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6535 unsigned IntrinsicID) {
6536 // For now, we're only lowering an 'add' histogram.
6537 // We can add others later, e.g. saturating adds, min/max.
6538 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6539 "Tried to lower unsupported histogram type");
6540 SDLoc sdl = getCurSDLoc();
6541 Value *Ptr = I.getOperand(i_nocapture: 0);
6542 SDValue Inc = getValue(V: I.getOperand(i_nocapture: 1));
6543 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
6544
6545 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6546 DataLayout TargetDL = DAG.getDataLayout();
6547 EVT VT = Inc.getValueType();
6548 Align Alignment = DAG.getEVTAlign(MemoryVT: VT);
6549
6550 const MDNode *Ranges = getRangeMetadata(I);
6551
6552 SDValue Root = DAG.getRoot();
6553 SDValue Base;
6554 SDValue Index;
6555 SDValue Scale;
6556 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
6557 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
6558
6559 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6560
6561 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6562 PtrInfo: MachinePointerInfo(AS),
6563 F: MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
6564 Size: MemoryLocation::UnknownSize, BaseAlignment: Alignment, AAInfo: I.getAAMetadata(), Ranges);
6565
6566 if (!UniformBase) {
6567 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6568 Index = getValue(V: Ptr);
6569 Scale =
6570 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6571 }
6572
6573 EVT IdxVT = Index.getValueType();
6574
6575 // Avoid using e.g. i32 as index type when the increment must be performed
6576 // on i64's.
6577 bool MustExtendIndex = VT.getScalarSizeInBits() > IdxVT.getScalarSizeInBits();
6578 EVT EltTy = MustExtendIndex ? VT : IdxVT.getVectorElementType();
6579 if (MustExtendIndex || TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
6580 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
6581 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
6582 }
6583
6584 SDValue ID = DAG.getTargetConstant(Val: IntrinsicID, DL: sdl, VT: MVT::i32);
6585
6586 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6587 SDValue Histogram = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT: VT, dl: sdl,
6588 Ops, MMO, IndexType: ISD::SIGNED_SCALED);
6589
6590 setValue(V: &I, NewN: Histogram);
6591 DAG.setRoot(Histogram);
6592}
6593
6594void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6595 unsigned Intrinsic) {
6596 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6597 "Tried lowering invalid vector extract last");
6598 SDLoc sdl = getCurSDLoc();
6599 const DataLayout &Layout = DAG.getDataLayout();
6600 SDValue Data = getValue(V: I.getOperand(i_nocapture: 0));
6601 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
6602
6603 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6604 EVT ResVT = TLI.getValueType(DL: Layout, Ty: I.getType());
6605
6606 EVT ExtVT = TLI.getVectorIdxTy(DL: Layout);
6607 SDValue Idx = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL: sdl, VT: ExtVT, Operand: Mask);
6608 SDValue Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: sdl, VT: ResVT, N1: Data, N2: Idx);
6609
6610 Value *Default = I.getOperand(i_nocapture: 2);
6611 if (!isa<PoisonValue>(Val: Default) && !isa<UndefValue>(Val: Default)) {
6612 SDValue PassThru = getValue(V: Default);
6613 EVT BoolVT = Mask.getValueType().getScalarType();
6614 SDValue AnyActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: sdl, VT: BoolVT, Operand: Mask);
6615 Result = DAG.getSelect(DL: sdl, VT: ResVT, Cond: AnyActive, LHS: Result, RHS: PassThru);
6616 }
6617
6618 setValue(V: &I, NewN: Result);
6619}
6620
6621/// Lower the call to the specified intrinsic function.
6622void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6623 unsigned Intrinsic) {
6624 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6625 SDLoc sdl = getCurSDLoc();
6626 DebugLoc dl = getCurDebugLoc();
6627 SDValue Res;
6628
6629 SDNodeFlags Flags;
6630 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
6631 Flags.copyFMF(FPMO: *FPOp);
6632
6633 switch (Intrinsic) {
6634 default:
6635 // By default, turn this into a target intrinsic node.
6636 visitTargetIntrinsic(I, Intrinsic);
6637 return;
6638 case Intrinsic::vscale: {
6639 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6640 setValue(V: &I, NewN: DAG.getVScale(DL: sdl, VT, MulImm: APInt(VT.getSizeInBits(), 1)));
6641 return;
6642 }
6643 case Intrinsic::vastart: visitVAStart(I); return;
6644 case Intrinsic::vaend: visitVAEnd(I); return;
6645 case Intrinsic::vacopy: visitVACopy(I); return;
6646 case Intrinsic::returnaddress:
6647 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::RETURNADDR, DL: sdl,
6648 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
6649 Operand: getValue(V: I.getArgOperand(i: 0))));
6650 return;
6651 case Intrinsic::addressofreturnaddress:
6652 setValue(V: &I,
6653 NewN: DAG.getNode(Opcode: ISD::ADDROFRETURNADDR, DL: sdl,
6654 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6655 return;
6656 case Intrinsic::sponentry:
6657 setValue(V: &I,
6658 NewN: DAG.getNode(Opcode: ISD::SPONENTRY, DL: sdl,
6659 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6660 return;
6661 case Intrinsic::frameaddress:
6662 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FRAMEADDR, DL: sdl,
6663 VT: TLI.getFrameIndexTy(DL: DAG.getDataLayout()),
6664 Operand: getValue(V: I.getArgOperand(i: 0))));
6665 return;
6666 case Intrinsic::read_volatile_register:
6667 case Intrinsic::read_register: {
6668 Value *Reg = I.getArgOperand(i: 0);
6669 SDValue Chain = getRoot();
6670 SDValue RegName =
6671 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6672 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6673 Res = DAG.getNode(Opcode: ISD::READ_REGISTER, DL: sdl,
6674 VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), N1: Chain, N2: RegName);
6675 setValue(V: &I, NewN: Res);
6676 DAG.setRoot(Res.getValue(R: 1));
6677 return;
6678 }
6679 case Intrinsic::write_register: {
6680 Value *Reg = I.getArgOperand(i: 0);
6681 Value *RegValue = I.getArgOperand(i: 1);
6682 SDValue Chain = getRoot();
6683 SDValue RegName =
6684 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6685 DAG.setRoot(DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL: sdl, VT: MVT::Other, N1: Chain,
6686 N2: RegName, N3: getValue(V: RegValue)));
6687 return;
6688 }
6689 case Intrinsic::memcpy:
6690 case Intrinsic::memcpy_inline: {
6691 const auto &MCI = cast<MemCpyInst>(Val: I);
6692 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6693 SDValue Src = getValue(V: I.getArgOperand(i: 1));
6694 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6695 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6696 "memcpy_inline needs constant size");
6697 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6698 Align DstAlign = MCI.getDestAlign().valueOrOne();
6699 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6700 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
6701 bool isVol = MCI.isVolatile();
6702 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6703 // node.
6704 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6705 SDValue MC = DAG.getMemcpy(Chain: Root, dl: sdl, Dst, Src, Size, Alignment, isVol,
6706 AlwaysInline: MCI.isForceInlined(), CI: &I, OverrideTailCall: std::nullopt,
6707 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
6708 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)),
6709 AAInfo: I.getAAMetadata(), BatchAA);
6710 updateDAGForMaybeTailCall(MaybeTC: MC);
6711 return;
6712 }
6713 case Intrinsic::memset:
6714 case Intrinsic::memset_inline: {
6715 const auto &MSII = cast<MemSetInst>(Val: I);
6716 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6717 SDValue Value = getValue(V: I.getArgOperand(i: 1));
6718 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6719 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6720 "memset_inline needs constant size");
6721 // @llvm.memset defines 0 and 1 to both mean no alignment.
6722 Align DstAlign = MSII.getDestAlign().valueOrOne();
6723 bool isVol = MSII.isVolatile();
6724 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6725 SDValue MC = DAG.getMemset(
6726 Chain: Root, dl: sdl, Dst, Src: Value, Size, Alignment: DstAlign, isVol, AlwaysInline: MSII.isForceInlined(),
6727 CI: &I, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)), AAInfo: I.getAAMetadata());
6728 updateDAGForMaybeTailCall(MaybeTC: MC);
6729 return;
6730 }
6731 case Intrinsic::memmove: {
6732 const auto &MMI = cast<MemMoveInst>(Val: I);
6733 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
6734 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
6735 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
6736 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6737 Align DstAlign = MMI.getDestAlign().valueOrOne();
6738 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6739 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
6740 bool isVol = MMI.isVolatile();
6741 // FIXME: Support passing different dest/src alignments to the memmove DAG
6742 // node.
6743 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6744 SDValue MM = DAG.getMemmove(Chain: Root, dl: sdl, Dst: Op1, Src: Op2, Size: Op3, Alignment, isVol, CI: &I,
6745 /* OverrideTailCall */ std::nullopt,
6746 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
6747 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)),
6748 AAInfo: I.getAAMetadata(), BatchAA);
6749 updateDAGForMaybeTailCall(MaybeTC: MM);
6750 return;
6751 }
6752 case Intrinsic::memcpy_element_unordered_atomic: {
6753 auto &MI = cast<AnyMemCpyInst>(Val: I);
6754 SDValue Dst = getValue(V: MI.getRawDest());
6755 SDValue Src = getValue(V: MI.getRawSource());
6756 SDValue Length = getValue(V: MI.getLength());
6757
6758 Type *LengthTy = MI.getLength()->getType();
6759 unsigned ElemSz = MI.getElementSizeInBytes();
6760 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6761 SDValue MC =
6762 DAG.getAtomicMemcpy(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
6763 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
6764 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
6765 updateDAGForMaybeTailCall(MaybeTC: MC);
6766 return;
6767 }
6768 case Intrinsic::memmove_element_unordered_atomic: {
6769 auto &MI = cast<AnyMemMoveInst>(Val: I);
6770 SDValue Dst = getValue(V: MI.getRawDest());
6771 SDValue Src = getValue(V: MI.getRawSource());
6772 SDValue Length = getValue(V: MI.getLength());
6773
6774 Type *LengthTy = MI.getLength()->getType();
6775 unsigned ElemSz = MI.getElementSizeInBytes();
6776 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6777 SDValue MC =
6778 DAG.getAtomicMemmove(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
6779 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
6780 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
6781 updateDAGForMaybeTailCall(MaybeTC: MC);
6782 return;
6783 }
6784 case Intrinsic::memset_element_unordered_atomic: {
6785 auto &MI = cast<AnyMemSetInst>(Val: I);
6786 SDValue Dst = getValue(V: MI.getRawDest());
6787 SDValue Val = getValue(V: MI.getValue());
6788 SDValue Length = getValue(V: MI.getLength());
6789
6790 Type *LengthTy = MI.getLength()->getType();
6791 unsigned ElemSz = MI.getElementSizeInBytes();
6792 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6793 SDValue MC =
6794 DAG.getAtomicMemset(Chain: getRoot(), dl: sdl, Dst, Value: Val, Size: Length, SizeTy: LengthTy, ElemSz,
6795 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()));
6796 updateDAGForMaybeTailCall(MaybeTC: MC);
6797 return;
6798 }
6799 case Intrinsic::call_preallocated_setup: {
6800 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: &I);
6801 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
6802 SDValue Res = DAG.getNode(Opcode: ISD::PREALLOCATED_SETUP, DL: sdl, VT: MVT::Other,
6803 N1: getRoot(), N2: SrcValue);
6804 setValue(V: &I, NewN: Res);
6805 DAG.setRoot(Res);
6806 return;
6807 }
6808 case Intrinsic::call_preallocated_arg: {
6809 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: I.getOperand(i_nocapture: 0));
6810 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
6811 SDValue Ops[3];
6812 Ops[0] = getRoot();
6813 Ops[1] = SrcValue;
6814 Ops[2] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 1)), DL: sdl,
6815 VT: MVT::i32); // arg index
6816 SDValue Res = DAG.getNode(
6817 Opcode: ISD::PREALLOCATED_ARG, DL: sdl,
6818 VTList: DAG.getVTList(VT1: TLI.getPointerTy(DL: DAG.getDataLayout()), VT2: MVT::Other), Ops);
6819 setValue(V: &I, NewN: Res);
6820 DAG.setRoot(Res.getValue(R: 1));
6821 return;
6822 }
6823
6824 case Intrinsic::eh_typeid_for: {
6825 // Find the type id for the given typeinfo.
6826 GlobalValue *GV = ExtractTypeInfo(V: I.getArgOperand(i: 0));
6827 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(TI: GV);
6828 Res = DAG.getConstant(Val: TypeID, DL: sdl, VT: MVT::i32);
6829 setValue(V: &I, NewN: Res);
6830 return;
6831 }
6832
6833 case Intrinsic::eh_return_i32:
6834 case Intrinsic::eh_return_i64:
6835 DAG.getMachineFunction().setCallsEHReturn(true);
6836 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_RETURN, DL: sdl,
6837 VT: MVT::Other,
6838 N1: getControlRoot(),
6839 N2: getValue(V: I.getArgOperand(i: 0)),
6840 N3: getValue(V: I.getArgOperand(i: 1))));
6841 return;
6842 case Intrinsic::eh_unwind_init:
6843 DAG.getMachineFunction().setCallsUnwindInit(true);
6844 return;
6845 case Intrinsic::eh_dwarf_cfa:
6846 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EH_DWARF_CFA, DL: sdl,
6847 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
6848 Operand: getValue(V: I.getArgOperand(i: 0))));
6849 return;
6850 case Intrinsic::eh_sjlj_callsite: {
6851 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 0));
6852 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
6853
6854 FuncInfo.setCurrentCallSite(CI->getZExtValue());
6855 return;
6856 }
6857 case Intrinsic::eh_sjlj_functioncontext: {
6858 // Get and store the index of the function context.
6859 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6860 AllocaInst *FnCtx =
6861 cast<AllocaInst>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
6862 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6863 MFI.setFunctionContextIndex(FI);
6864 return;
6865 }
6866 case Intrinsic::eh_sjlj_setjmp: {
6867 SDValue Ops[2];
6868 Ops[0] = getRoot();
6869 Ops[1] = getValue(V: I.getArgOperand(i: 0));
6870 SDValue Op = DAG.getNode(Opcode: ISD::EH_SJLJ_SETJMP, DL: sdl,
6871 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
6872 setValue(V: &I, NewN: Op.getValue(R: 0));
6873 DAG.setRoot(Op.getValue(R: 1));
6874 return;
6875 }
6876 case Intrinsic::eh_sjlj_longjmp:
6877 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_SJLJ_LONGJMP, DL: sdl, VT: MVT::Other,
6878 N1: getRoot(), N2: getValue(V: I.getArgOperand(i: 0))));
6879 return;
6880 case Intrinsic::eh_sjlj_setup_dispatch:
6881 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_SJLJ_SETUP_DISPATCH, DL: sdl, VT: MVT::Other,
6882 Operand: getRoot()));
6883 return;
6884 case Intrinsic::masked_gather:
6885 visitMaskedGather(I);
6886 return;
6887 case Intrinsic::masked_load:
6888 visitMaskedLoad(I);
6889 return;
6890 case Intrinsic::masked_scatter:
6891 visitMaskedScatter(I);
6892 return;
6893 case Intrinsic::masked_store:
6894 visitMaskedStore(I);
6895 return;
6896 case Intrinsic::masked_expandload:
6897 visitMaskedLoad(I, IsExpanding: true /* IsExpanding */);
6898 return;
6899 case Intrinsic::masked_compressstore:
6900 visitMaskedStore(I, IsCompressing: true /* IsCompressing */);
6901 return;
6902 case Intrinsic::powi:
6903 setValue(V: &I, NewN: ExpandPowI(DL: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
6904 RHS: getValue(V: I.getArgOperand(i: 1)), DAG));
6905 return;
6906 case Intrinsic::log:
6907 setValue(V: &I, NewN: expandLog(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6908 return;
6909 case Intrinsic::log2:
6910 setValue(V: &I,
6911 NewN: expandLog2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6912 return;
6913 case Intrinsic::log10:
6914 setValue(V: &I,
6915 NewN: expandLog10(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6916 return;
6917 case Intrinsic::exp:
6918 setValue(V: &I, NewN: expandExp(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6919 return;
6920 case Intrinsic::exp2:
6921 setValue(V: &I,
6922 NewN: expandExp2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6923 return;
6924 case Intrinsic::pow:
6925 setValue(V: &I, NewN: expandPow(dl: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
6926 RHS: getValue(V: I.getArgOperand(i: 1)), DAG, TLI, Flags));
6927 return;
6928 case Intrinsic::sqrt:
6929 case Intrinsic::fabs:
6930 case Intrinsic::sin:
6931 case Intrinsic::cos:
6932 case Intrinsic::tan:
6933 case Intrinsic::asin:
6934 case Intrinsic::acos:
6935 case Intrinsic::atan:
6936 case Intrinsic::sinh:
6937 case Intrinsic::cosh:
6938 case Intrinsic::tanh:
6939 case Intrinsic::exp10:
6940 case Intrinsic::floor:
6941 case Intrinsic::ceil:
6942 case Intrinsic::trunc:
6943 case Intrinsic::rint:
6944 case Intrinsic::nearbyint:
6945 case Intrinsic::round:
6946 case Intrinsic::roundeven:
6947 case Intrinsic::canonicalize: {
6948 unsigned Opcode;
6949 // clang-format off
6950 switch (Intrinsic) {
6951 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6952 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6953 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6954 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6955 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6956 case Intrinsic::tan: Opcode = ISD::FTAN; break;
6957 case Intrinsic::asin: Opcode = ISD::FASIN; break;
6958 case Intrinsic::acos: Opcode = ISD::FACOS; break;
6959 case Intrinsic::atan: Opcode = ISD::FATAN; break;
6960 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
6961 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
6962 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
6963 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6964 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6965 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6966 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6967 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6968 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6969 case Intrinsic::round: Opcode = ISD::FROUND; break;
6970 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6971 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6972 }
6973 // clang-format on
6974
6975 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl,
6976 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6977 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
6978 return;
6979 }
6980 case Intrinsic::atan2:
6981 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FATAN2, DL: sdl,
6982 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6983 N1: getValue(V: I.getArgOperand(i: 0)),
6984 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6985 return;
6986 case Intrinsic::lround:
6987 case Intrinsic::llround:
6988 case Intrinsic::lrint:
6989 case Intrinsic::llrint: {
6990 unsigned Opcode;
6991 // clang-format off
6992 switch (Intrinsic) {
6993 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6994 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6995 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6996 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6997 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6998 }
6999 // clang-format on
7000
7001 EVT RetVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7002 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VT: RetVT,
7003 Operand: getValue(V: I.getArgOperand(i: 0))));
7004 return;
7005 }
7006 case Intrinsic::minnum:
7007 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINNUM, DL: sdl,
7008 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7009 N1: getValue(V: I.getArgOperand(i: 0)),
7010 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7011 return;
7012 case Intrinsic::maxnum:
7013 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXNUM, DL: sdl,
7014 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7015 N1: getValue(V: I.getArgOperand(i: 0)),
7016 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7017 return;
7018 case Intrinsic::minimum:
7019 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUM, DL: sdl,
7020 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7021 N1: getValue(V: I.getArgOperand(i: 0)),
7022 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7023 return;
7024 case Intrinsic::maximum:
7025 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUM, DL: sdl,
7026 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7027 N1: getValue(V: I.getArgOperand(i: 0)),
7028 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7029 return;
7030 case Intrinsic::minimumnum:
7031 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUMNUM, DL: sdl,
7032 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7033 N1: getValue(V: I.getArgOperand(i: 0)),
7034 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7035 return;
7036 case Intrinsic::maximumnum:
7037 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUMNUM, DL: sdl,
7038 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7039 N1: getValue(V: I.getArgOperand(i: 0)),
7040 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7041 return;
7042 case Intrinsic::copysign:
7043 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: sdl,
7044 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7045 N1: getValue(V: I.getArgOperand(i: 0)),
7046 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7047 return;
7048 case Intrinsic::ldexp:
7049 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FLDEXP, DL: sdl,
7050 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7051 N1: getValue(V: I.getArgOperand(i: 0)),
7052 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7053 return;
7054 case Intrinsic::modf:
7055 case Intrinsic::sincos:
7056 case Intrinsic::sincospi:
7057 case Intrinsic::frexp: {
7058 unsigned Opcode;
7059 switch (Intrinsic) {
7060 default:
7061 llvm_unreachable("unexpected intrinsic");
7062 case Intrinsic::sincos:
7063 Opcode = ISD::FSINCOS;
7064 break;
7065 case Intrinsic::sincospi:
7066 Opcode = ISD::FSINCOSPI;
7067 break;
7068 case Intrinsic::modf:
7069 Opcode = ISD::FMODF;
7070 break;
7071 case Intrinsic::frexp:
7072 Opcode = ISD::FFREXP;
7073 break;
7074 }
7075 SmallVector<EVT, 2> ValueVTs;
7076 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
7077 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
7078 setValue(
7079 V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: getValue(V: I.getArgOperand(i: 0)), Flags));
7080 return;
7081 }
7082 case Intrinsic::arithmetic_fence: {
7083 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: sdl,
7084 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7085 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
7086 return;
7087 }
7088 case Intrinsic::fma:
7089 setValue(V: &I, NewN: DAG.getNode(
7090 Opcode: ISD::FMA, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7091 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)),
7092 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7093 return;
7094#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7095 case Intrinsic::INTRINSIC:
7096#include "llvm/IR/ConstrainedOps.def"
7097 visitConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: I));
7098 return;
7099#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7100#include "llvm/IR/VPIntrinsics.def"
7101 visitVectorPredicationIntrinsic(VPIntrin: cast<VPIntrinsic>(Val: I));
7102 return;
7103 case Intrinsic::fptrunc_round: {
7104 // Get the last argument, the metadata and convert it to an integer in the
7105 // call
7106 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7107 std::optional<RoundingMode> RoundMode =
7108 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
7109
7110 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7111
7112 // Propagate fast-math-flags from IR to node(s).
7113 SDNodeFlags Flags;
7114 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
7115 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
7116
7117 SDValue Result;
7118 Result = DAG.getNode(
7119 Opcode: ISD::FPTRUNC_ROUND, DL: sdl, VT, N1: getValue(V: I.getArgOperand(i: 0)),
7120 N2: DAG.getTargetConstant(Val: (int)*RoundMode, DL: sdl, VT: MVT::i32));
7121 setValue(V: &I, NewN: Result);
7122
7123 return;
7124 }
7125 case Intrinsic::fmuladd: {
7126 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7127 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
7128 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
7129 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMA, DL: sdl,
7130 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7131 N1: getValue(V: I.getArgOperand(i: 0)),
7132 N2: getValue(V: I.getArgOperand(i: 1)),
7133 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7134 } else if (TLI.isOperationLegalOrCustom(Op: ISD::FMULADD, VT)) {
7135 // TODO: Support splitting the vector.
7136 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMULADD, DL: sdl,
7137 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7138 N1: getValue(V: I.getArgOperand(i: 0)),
7139 N2: getValue(V: I.getArgOperand(i: 1)),
7140 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7141 } else {
7142 // TODO: Intrinsic calls should have fast-math-flags.
7143 SDValue Mul = DAG.getNode(
7144 Opcode: ISD::FMUL, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7145 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)), Flags);
7146 SDValue Add = DAG.getNode(Opcode: ISD::FADD, DL: sdl,
7147 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7148 N1: Mul, N2: getValue(V: I.getArgOperand(i: 2)), Flags);
7149 setValue(V: &I, NewN: Add);
7150 }
7151 return;
7152 }
7153 case Intrinsic::fptosi_sat: {
7154 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7155 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT_SAT, DL: sdl, VT,
7156 N1: getValue(V: I.getArgOperand(i: 0)),
7157 N2: DAG.getValueType(VT.getScalarType())));
7158 return;
7159 }
7160 case Intrinsic::fptoui_sat: {
7161 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7162 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT_SAT, DL: sdl, VT,
7163 N1: getValue(V: I.getArgOperand(i: 0)),
7164 N2: DAG.getValueType(VT.getScalarType())));
7165 return;
7166 }
7167 case Intrinsic::convert_from_arbitrary_fp: {
7168 // Extract format metadata and convert to semantics enum.
7169 EVT DstVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7170 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7171 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
7172 const fltSemantics *SrcSem =
7173 APFloatBase::getArbitraryFPSemantics(Format: FormatStr);
7174 if (!SrcSem) {
7175 DAG.getContext()->emitError(
7176 ErrorStr: "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7177 "'");
7178 setValue(V: &I, NewN: DAG.getPOISON(VT: DstVT));
7179 return;
7180 }
7181 APFloatBase::Semantics SemEnum = APFloatBase::SemanticsToEnum(Sem: *SrcSem);
7182
7183 SDValue IntVal = getValue(V: I.getArgOperand(i: 0));
7184
7185 // Emit ISD::CONVERT_FROM_ARBITRARY_FP node.
7186 SDValue SemConst =
7187 DAG.getTargetConstant(Val: static_cast<int>(SemEnum), DL: sdl, VT: MVT::i32);
7188 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: sdl, VT: DstVT, N1: IntVal,
7189 N2: SemConst));
7190 return;
7191 }
7192 case Intrinsic::set_rounding:
7193 Res = DAG.getNode(Opcode: ISD::SET_ROUNDING, DL: sdl, VT: MVT::Other,
7194 Ops: {getRoot(), getValue(V: I.getArgOperand(i: 0))});
7195 setValue(V: &I, NewN: Res);
7196 DAG.setRoot(Res.getValue(R: 0));
7197 return;
7198 case Intrinsic::is_fpclass: {
7199 const DataLayout DLayout = DAG.getDataLayout();
7200 EVT DestVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7201 EVT ArgVT = TLI.getValueType(DL: DLayout, Ty: I.getArgOperand(i: 0)->getType());
7202 FPClassTest Test = static_cast<FPClassTest>(
7203 cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue());
7204 MachineFunction &MF = DAG.getMachineFunction();
7205 const Function &F = MF.getFunction();
7206 SDValue Op = getValue(V: I.getArgOperand(i: 0));
7207 SDNodeFlags Flags;
7208 Flags.setNoFPExcept(
7209 !F.getAttributes().hasFnAttr(Kind: llvm::Attribute::StrictFP));
7210 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7211 // expansion can use illegal types. Making expansion early allows
7212 // legalizing these types prior to selection.
7213 if (!TLI.isOperationLegal(Op: ISD::IS_FPCLASS, VT: ArgVT) &&
7214 !TLI.isOperationCustom(Op: ISD::IS_FPCLASS, VT: ArgVT)) {
7215 SDValue Result = TLI.expandIS_FPCLASS(ResultVT: DestVT, Op, Test, Flags, DL: sdl, DAG);
7216 setValue(V: &I, NewN: Result);
7217 return;
7218 }
7219
7220 SDValue Check = DAG.getTargetConstant(Val: Test, DL: sdl, VT: MVT::i32);
7221 SDValue V = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: sdl, VT: DestVT, Ops: {Op, Check}, Flags);
7222 setValue(V: &I, NewN: V);
7223 return;
7224 }
7225 case Intrinsic::get_fpenv: {
7226 const DataLayout DLayout = DAG.getDataLayout();
7227 EVT EnvVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7228 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7229 SDValue Chain = getRoot();
7230 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7231 // and temporary storage in stack.
7232 if (TLI.isOperationLegalOrCustom(Op: ISD::GET_FPENV, VT: EnvVT)) {
7233 Res = DAG.getNode(
7234 Opcode: ISD::GET_FPENV, DL: sdl,
7235 VTList: DAG.getVTList(VT1: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
7236 VT2: MVT::Other),
7237 N: Chain);
7238 } else {
7239 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7240 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7241 auto MPI =
7242 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7243 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7244 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
7245 BaseAlignment: TempAlign);
7246 Chain = DAG.getGetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7247 Res = DAG.getLoad(VT: EnvVT, dl: sdl, Chain, Ptr: Temp, PtrInfo: MPI);
7248 }
7249 setValue(V: &I, NewN: Res);
7250 DAG.setRoot(Res.getValue(R: 1));
7251 return;
7252 }
7253 case Intrinsic::set_fpenv: {
7254 const DataLayout DLayout = DAG.getDataLayout();
7255 SDValue Env = getValue(V: I.getArgOperand(i: 0));
7256 EVT EnvVT = Env.getValueType();
7257 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7258 SDValue Chain = getRoot();
7259 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7260 // environment from memory.
7261 if (TLI.isOperationLegalOrCustom(Op: ISD::SET_FPENV, VT: EnvVT)) {
7262 Chain = DAG.getNode(Opcode: ISD::SET_FPENV, DL: sdl, VT: MVT::Other, N1: Chain, N2: Env);
7263 } else {
7264 // Allocate space in stack, copy environment bits into it and use this
7265 // memory in SET_FPENV_MEM.
7266 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7267 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7268 auto MPI =
7269 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7270 Chain = DAG.getStore(Chain, dl: sdl, Val: Env, Ptr: Temp, PtrInfo: MPI, Alignment: TempAlign,
7271 MMOFlags: MachineMemOperand::MOStore);
7272 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7273 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
7274 BaseAlignment: TempAlign);
7275 Chain = DAG.getSetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7276 }
7277 DAG.setRoot(Chain);
7278 return;
7279 }
7280 case Intrinsic::reset_fpenv:
7281 DAG.setRoot(DAG.getNode(Opcode: ISD::RESET_FPENV, DL: sdl, VT: MVT::Other, Operand: getRoot()));
7282 return;
7283 case Intrinsic::get_fpmode:
7284 Res = DAG.getNode(
7285 Opcode: ISD::GET_FPMODE, DL: sdl,
7286 VTList: DAG.getVTList(VT1: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
7287 VT2: MVT::Other),
7288 N: DAG.getRoot());
7289 setValue(V: &I, NewN: Res);
7290 DAG.setRoot(Res.getValue(R: 1));
7291 return;
7292 case Intrinsic::set_fpmode:
7293 Res = DAG.getNode(Opcode: ISD::SET_FPMODE, DL: sdl, VT: MVT::Other, N1: {DAG.getRoot()},
7294 N2: getValue(V: I.getArgOperand(i: 0)));
7295 DAG.setRoot(Res);
7296 return;
7297 case Intrinsic::reset_fpmode: {
7298 Res = DAG.getNode(Opcode: ISD::RESET_FPMODE, DL: sdl, VT: MVT::Other, Operand: getRoot());
7299 DAG.setRoot(Res);
7300 return;
7301 }
7302 case Intrinsic::pcmarker: {
7303 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
7304 DAG.setRoot(DAG.getNode(Opcode: ISD::PCMARKER, DL: sdl, VT: MVT::Other, N1: getRoot(), N2: Tmp));
7305 return;
7306 }
7307 case Intrinsic::readcyclecounter: {
7308 SDValue Op = getRoot();
7309 Res = DAG.getNode(Opcode: ISD::READCYCLECOUNTER, DL: sdl,
7310 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N: Op);
7311 setValue(V: &I, NewN: Res);
7312 DAG.setRoot(Res.getValue(R: 1));
7313 return;
7314 }
7315 case Intrinsic::readsteadycounter: {
7316 SDValue Op = getRoot();
7317 Res = DAG.getNode(Opcode: ISD::READSTEADYCOUNTER, DL: sdl,
7318 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N: Op);
7319 setValue(V: &I, NewN: Res);
7320 DAG.setRoot(Res.getValue(R: 1));
7321 return;
7322 }
7323 case Intrinsic::bitreverse:
7324 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITREVERSE, DL: sdl,
7325 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7326 Operand: getValue(V: I.getArgOperand(i: 0))));
7327 return;
7328 case Intrinsic::bswap:
7329 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BSWAP, DL: sdl,
7330 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7331 Operand: getValue(V: I.getArgOperand(i: 0))));
7332 return;
7333 case Intrinsic::cttz: {
7334 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7335 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7336 EVT Ty = Arg.getValueType();
7337 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
7338 DL: sdl, VT: Ty, Operand: Arg));
7339 return;
7340 }
7341 case Intrinsic::ctlz: {
7342 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7343 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7344 EVT Ty = Arg.getValueType();
7345 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
7346 DL: sdl, VT: Ty, Operand: Arg));
7347 return;
7348 }
7349 case Intrinsic::ctpop: {
7350 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7351 EVT Ty = Arg.getValueType();
7352 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CTPOP, DL: sdl, VT: Ty, Operand: Arg));
7353 return;
7354 }
7355 case Intrinsic::fshl:
7356 case Intrinsic::fshr: {
7357 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7358 SDValue X = getValue(V: I.getArgOperand(i: 0));
7359 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7360 SDValue Z = getValue(V: I.getArgOperand(i: 2));
7361 EVT VT = X.getValueType();
7362
7363 if (X == Y) {
7364 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7365 setValue(V: &I, NewN: DAG.getNode(Opcode: RotateOpcode, DL: sdl, VT, N1: X, N2: Z));
7366 } else {
7367 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7368 setValue(V: &I, NewN: DAG.getNode(Opcode: FunnelOpcode, DL: sdl, VT, N1: X, N2: Y, N3: Z));
7369 }
7370 return;
7371 }
7372 case Intrinsic::clmul: {
7373 SDValue X = getValue(V: I.getArgOperand(i: 0));
7374 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7375 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CLMUL, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7376 return;
7377 }
7378 case Intrinsic::sadd_sat: {
7379 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7380 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7381 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7382 return;
7383 }
7384 case Intrinsic::uadd_sat: {
7385 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7386 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7387 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7388 return;
7389 }
7390 case Intrinsic::ssub_sat: {
7391 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7392 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7393 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SSUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7394 return;
7395 }
7396 case Intrinsic::usub_sat: {
7397 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7398 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7399 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::USUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7400 return;
7401 }
7402 case Intrinsic::sshl_sat:
7403 case Intrinsic::ushl_sat: {
7404 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7405 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7406
7407 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
7408 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
7409
7410 // Coerce the shift amount to the right type if we can. This exposes the
7411 // truncate or zext to optimization early.
7412 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
7413 assert(ShiftTy.getSizeInBits() >=
7414 Log2_32_Ceil(Op1.getValueSizeInBits()) &&
7415 "Unexpected shift type");
7416 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
7417 }
7418
7419 unsigned Opc =
7420 Intrinsic == Intrinsic::sshl_sat ? ISD::SSHLSAT : ISD::USHLSAT;
7421 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7422 return;
7423 }
7424 case Intrinsic::smul_fix:
7425 case Intrinsic::umul_fix:
7426 case Intrinsic::smul_fix_sat:
7427 case Intrinsic::umul_fix_sat: {
7428 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7429 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7430 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7431 setValue(V: &I, NewN: DAG.getNode(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7432 VT: Op1.getValueType(), N1: Op1, N2: Op2, N3: Op3));
7433 return;
7434 }
7435 case Intrinsic::sdiv_fix:
7436 case Intrinsic::udiv_fix:
7437 case Intrinsic::sdiv_fix_sat:
7438 case Intrinsic::udiv_fix_sat: {
7439 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7440 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7441 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7442 setValue(V: &I, NewN: expandDivFix(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7443 LHS: Op1, RHS: Op2, Scale: Op3, DAG, TLI));
7444 return;
7445 }
7446 case Intrinsic::smax: {
7447 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7448 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7449 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7450 return;
7451 }
7452 case Intrinsic::smin: {
7453 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7454 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7455 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7456 return;
7457 }
7458 case Intrinsic::umax: {
7459 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7460 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7461 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7462 return;
7463 }
7464 case Intrinsic::umin: {
7465 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7466 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7467 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7468 return;
7469 }
7470 case Intrinsic::abs: {
7471 // TODO: Preserve "int min is poison" arg in SDAG?
7472 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7473 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ABS, DL: sdl, VT: Op1.getValueType(), Operand: Op1));
7474 return;
7475 }
7476 case Intrinsic::scmp: {
7477 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7478 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7479 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7480 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7481 break;
7482 }
7483 case Intrinsic::ucmp: {
7484 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7485 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7486 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7487 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7488 break;
7489 }
7490 case Intrinsic::stackaddress:
7491 case Intrinsic::stacksave: {
7492 unsigned SDOpcode = Intrinsic == Intrinsic::stackaddress ? ISD::STACKADDRESS
7493 : ISD::STACKSAVE;
7494 SDValue Op = getRoot();
7495 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7496 Res = DAG.getNode(Opcode: SDOpcode, DL: sdl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), N: Op);
7497 setValue(V: &I, NewN: Res);
7498 DAG.setRoot(Res.getValue(R: 1));
7499 return;
7500 }
7501 case Intrinsic::stackrestore:
7502 Res = getValue(V: I.getArgOperand(i: 0));
7503 DAG.setRoot(DAG.getNode(Opcode: ISD::STACKRESTORE, DL: sdl, VT: MVT::Other, N1: getRoot(), N2: Res));
7504 return;
7505 case Intrinsic::get_dynamic_area_offset: {
7506 SDValue Op = getRoot();
7507 EVT ResTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7508 Res = DAG.getNode(Opcode: ISD::GET_DYNAMIC_AREA_OFFSET, DL: sdl, VTList: DAG.getVTList(VT: ResTy),
7509 N: Op);
7510 DAG.setRoot(Op);
7511 setValue(V: &I, NewN: Res);
7512 return;
7513 }
7514 case Intrinsic::stackguard: {
7515 MachineFunction &MF = DAG.getMachineFunction();
7516 const Module &M = *MF.getFunction().getParent();
7517 EVT PtrTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7518 SDValue Chain = getRoot();
7519 if (TLI.useLoadStackGuardNode(M)) {
7520 Res = getLoadStackGuard(DAG, DL: sdl, Chain);
7521 Res = DAG.getPtrExtOrTrunc(Op: Res, DL: sdl, VT: PtrTy);
7522 } else {
7523 const Value *Global = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls());
7524 if (!Global) {
7525 LLVMContext &Ctx = *DAG.getContext();
7526 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
7527 setValue(V: &I, NewN: DAG.getPOISON(VT: PtrTy));
7528 return;
7529 }
7530
7531 Align Align = DAG.getDataLayout().getPrefTypeAlign(Ty: Global->getType());
7532 Res = DAG.getLoad(VT: PtrTy, dl: sdl, Chain, Ptr: getValue(V: Global),
7533 PtrInfo: MachinePointerInfo(Global, 0), Alignment: Align,
7534 MMOFlags: MachineMemOperand::MOVolatile);
7535 }
7536 if (TLI.useStackGuardXorFP())
7537 Res = TLI.emitStackGuardXorFP(DAG, Val: Res, DL: sdl);
7538 DAG.setRoot(Chain);
7539 setValue(V: &I, NewN: Res);
7540 return;
7541 }
7542 case Intrinsic::stackprotector: {
7543 // Emit code into the DAG to store the stack guard onto the stack.
7544 MachineFunction &MF = DAG.getMachineFunction();
7545 MachineFrameInfo &MFI = MF.getFrameInfo();
7546 const Module &M = *MF.getFunction().getParent();
7547 SDValue Src, Chain = getRoot();
7548
7549 if (TLI.useLoadStackGuardNode(M))
7550 Src = getLoadStackGuard(DAG, DL: sdl, Chain);
7551 else
7552 Src = getValue(V: I.getArgOperand(i: 0)); // The guard's value.
7553
7554 AllocaInst *Slot = cast<AllocaInst>(Val: I.getArgOperand(i: 1));
7555
7556 int FI = FuncInfo.StaticAllocaMap[Slot];
7557 MFI.setStackProtectorIndex(FI);
7558 EVT PtrTy = TLI.getFrameIndexTy(DL: DAG.getDataLayout());
7559
7560 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrTy);
7561
7562 // Store the stack protector onto the stack.
7563 Res = DAG.getStore(
7564 Chain, dl: sdl, Val: Src, Ptr: FIN,
7565 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI),
7566 Alignment: MaybeAlign(), MMOFlags: MachineMemOperand::MOVolatile);
7567 setValue(V: &I, NewN: Res);
7568 DAG.setRoot(Res);
7569 return;
7570 }
7571 case Intrinsic::objectsize:
7572 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7573
7574 case Intrinsic::is_constant:
7575 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7576
7577 case Intrinsic::annotation:
7578 case Intrinsic::ptr_annotation:
7579 case Intrinsic::launder_invariant_group:
7580 case Intrinsic::strip_invariant_group:
7581 // Drop the intrinsic, but forward the value
7582 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
7583 return;
7584
7585 case Intrinsic::type_test:
7586 case Intrinsic::public_type_test:
7587 reportFatalUsageError(reason: "llvm.type.test intrinsic must be lowered by the "
7588 "LowerTypeTests pass before code generation");
7589 return;
7590
7591 case Intrinsic::assume:
7592 case Intrinsic::experimental_noalias_scope_decl:
7593 case Intrinsic::var_annotation:
7594 case Intrinsic::sideeffect:
7595 // Discard annotate attributes, noalias scope declarations, assumptions, and
7596 // artificial side-effects.
7597 return;
7598
7599 case Intrinsic::codeview_annotation: {
7600 // Emit a label associated with this metadata.
7601 MachineFunction &MF = DAG.getMachineFunction();
7602 MCSymbol *Label = MF.getContext().createTempSymbol(Name: "annotation", AlwaysAddSuffix: true);
7603 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
7604 MF.addCodeViewAnnotation(Label, MD: cast<MDNode>(Val: MD));
7605 Res = DAG.getLabelNode(Opcode: ISD::ANNOTATION_LABEL, dl: sdl, Root: getRoot(), Label);
7606 DAG.setRoot(Res);
7607 return;
7608 }
7609
7610 case Intrinsic::init_trampoline: {
7611 const Function *F = cast<Function>(Val: I.getArgOperand(i: 1)->stripPointerCasts());
7612
7613 SDValue Ops[6];
7614 Ops[0] = getRoot();
7615 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7616 Ops[2] = getValue(V: I.getArgOperand(i: 1));
7617 Ops[3] = getValue(V: I.getArgOperand(i: 2));
7618 Ops[4] = DAG.getSrcValue(v: I.getArgOperand(i: 0));
7619 Ops[5] = DAG.getSrcValue(v: F);
7620
7621 Res = DAG.getNode(Opcode: ISD::INIT_TRAMPOLINE, DL: sdl, VT: MVT::Other, Ops);
7622
7623 DAG.setRoot(Res);
7624 return;
7625 }
7626 case Intrinsic::adjust_trampoline:
7627 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ADJUST_TRAMPOLINE, DL: sdl,
7628 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
7629 Operand: getValue(V: I.getArgOperand(i: 0))));
7630 return;
7631 case Intrinsic::gcroot: {
7632 assert(DAG.getMachineFunction().getFunction().hasGC() &&
7633 "only valid in functions with gc specified, enforced by Verifier");
7634 assert(GFI && "implied by previous");
7635 const Value *Alloca = I.getArgOperand(i: 0)->stripPointerCasts();
7636 const Constant *TypeMap = cast<Constant>(Val: I.getArgOperand(i: 1));
7637
7638 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(Val: getValue(V: Alloca).getNode());
7639 GFI->addStackRoot(Num: FI->getIndex(), Metadata: TypeMap);
7640 return;
7641 }
7642 case Intrinsic::gcread:
7643 case Intrinsic::gcwrite:
7644 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7645 case Intrinsic::get_rounding:
7646 Res = DAG.getNode(Opcode: ISD::GET_ROUNDING, DL: sdl, ResultTys: {MVT::i32, MVT::Other}, Ops: getRoot());
7647 setValue(V: &I, NewN: Res);
7648 DAG.setRoot(Res.getValue(R: 1));
7649 return;
7650
7651 case Intrinsic::expect:
7652 case Intrinsic::expect_with_probability:
7653 // Just replace __builtin_expect(exp, c) and
7654 // __builtin_expect_with_probability(exp, c, p) with EXP.
7655 setValue(V: &I, NewN: getValue(V: I.getArgOperand(i: 0)));
7656 return;
7657
7658 case Intrinsic::ubsantrap:
7659 case Intrinsic::debugtrap:
7660 case Intrinsic::trap: {
7661 StringRef TrapFuncName =
7662 I.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
7663 if (TrapFuncName.empty()) {
7664 switch (Intrinsic) {
7665 case Intrinsic::trap:
7666 DAG.setRoot(DAG.getNode(Opcode: ISD::TRAP, DL: sdl, VT: MVT::Other, Operand: getRoot()));
7667 break;
7668 case Intrinsic::debugtrap:
7669 DAG.setRoot(DAG.getNode(Opcode: ISD::DEBUGTRAP, DL: sdl, VT: MVT::Other, Operand: getRoot()));
7670 break;
7671 case Intrinsic::ubsantrap:
7672 DAG.setRoot(DAG.getNode(
7673 Opcode: ISD::UBSANTRAP, DL: sdl, VT: MVT::Other, N1: getRoot(),
7674 N2: DAG.getTargetConstant(
7675 Val: cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue(), DL: sdl,
7676 VT: MVT::i32)));
7677 break;
7678 default: llvm_unreachable("unknown trap intrinsic");
7679 }
7680 DAG.addNoMergeSiteInfo(Node: DAG.getRoot().getNode(),
7681 NoMerge: I.hasFnAttr(Kind: Attribute::NoMerge));
7682 return;
7683 }
7684 TargetLowering::ArgListTy Args;
7685 if (Intrinsic == Intrinsic::ubsantrap) {
7686 Value *Arg = I.getArgOperand(i: 0);
7687 Args.emplace_back(args&: Arg, args: getValue(V: Arg));
7688 }
7689
7690 TargetLowering::CallLoweringInfo CLI(DAG);
7691 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7692 CC: CallingConv::C, ResultType: I.getType(),
7693 Target: DAG.getExternalSymbol(Sym: TrapFuncName.data(),
7694 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
7695 ArgsList: std::move(Args));
7696 CLI.NoMerge = I.hasFnAttr(Kind: Attribute::NoMerge);
7697 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7698 DAG.setRoot(Result.second);
7699 return;
7700 }
7701
7702 case Intrinsic::allow_runtime_check:
7703 case Intrinsic::allow_ubsan_check:
7704 setValue(V: &I, NewN: getValue(V: ConstantInt::getTrue(Ty: I.getType())));
7705 return;
7706
7707 case Intrinsic::uadd_with_overflow:
7708 case Intrinsic::sadd_with_overflow:
7709 case Intrinsic::usub_with_overflow:
7710 case Intrinsic::ssub_with_overflow:
7711 case Intrinsic::umul_with_overflow:
7712 case Intrinsic::smul_with_overflow: {
7713 ISD::NodeType Op;
7714 switch (Intrinsic) {
7715 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7716 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7717 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7718 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7719 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7720 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7721 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7722 }
7723 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7724 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7725
7726 EVT ResultVT = Op1.getValueType();
7727 EVT OverflowVT = ResultVT.changeElementType(Context&: *Context, EltVT: MVT::i1);
7728
7729 SDVTList VTs = DAG.getVTList(VT1: ResultVT, VT2: OverflowVT);
7730 setValue(V: &I, NewN: DAG.getNode(Opcode: Op, DL: sdl, VTList: VTs, N1: Op1, N2: Op2));
7731 return;
7732 }
7733 case Intrinsic::prefetch: {
7734 SDValue Ops[5];
7735 unsigned rw = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
7736 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
7737 Ops[0] = DAG.getRoot();
7738 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7739 Ops[2] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 1)), DL: sdl,
7740 VT: MVT::i32);
7741 Ops[3] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 2)), DL: sdl,
7742 VT: MVT::i32);
7743 Ops[4] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 3)), DL: sdl,
7744 VT: MVT::i32);
7745 SDValue Result = DAG.getMemIntrinsicNode(
7746 Opcode: ISD::PREFETCH, dl: sdl, VTList: DAG.getVTList(VT: MVT::Other), Ops,
7747 MemVT: EVT::getIntegerVT(Context&: *Context, BitWidth: 8), PtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
7748 /* align */ Alignment: std::nullopt, Flags);
7749
7750 // Chain the prefetch in parallel with any pending loads, to stay out of
7751 // the way of later optimizations.
7752 PendingLoads.push_back(Elt: Result);
7753 Result = getRoot();
7754 DAG.setRoot(Result);
7755 return;
7756 }
7757 case Intrinsic::lifetime_start:
7758 case Intrinsic::lifetime_end: {
7759 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7760 // Stack coloring is not enabled in O0, discard region information.
7761 if (TM.getOptLevel() == CodeGenOptLevel::None)
7762 return;
7763
7764 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(Val: I.getArgOperand(i: 0));
7765 if (!LifetimeObject)
7766 return;
7767
7768 // First check that the Alloca is static, otherwise it won't have a
7769 // valid frame index.
7770 auto SI = FuncInfo.StaticAllocaMap.find(Val: LifetimeObject);
7771 if (SI == FuncInfo.StaticAllocaMap.end())
7772 return;
7773
7774 const int FrameIndex = SI->second;
7775 Res = DAG.getLifetimeNode(IsStart, dl: sdl, Chain: getRoot(), FrameIndex);
7776 DAG.setRoot(Res);
7777 return;
7778 }
7779 case Intrinsic::pseudoprobe: {
7780 auto Guid = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue();
7781 auto Index = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
7782 auto Attr = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
7783 Res = DAG.getPseudoProbeNode(Dl: sdl, Chain: getRoot(), Guid, Index, Attr);
7784 DAG.setRoot(Res);
7785 return;
7786 }
7787 case Intrinsic::invariant_start:
7788 // Discard region information.
7789 setValue(V: &I,
7790 NewN: DAG.getUNDEF(VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
7791 return;
7792 case Intrinsic::invariant_end:
7793 // Discard region information.
7794 return;
7795 case Intrinsic::clear_cache: {
7796 SDValue InputChain = DAG.getRoot();
7797 SDValue StartVal = getValue(V: I.getArgOperand(i: 0));
7798 SDValue EndVal = getValue(V: I.getArgOperand(i: 1));
7799 Res = DAG.getNode(Opcode: ISD::CLEAR_CACHE, DL: sdl, VTList: DAG.getVTList(VT: MVT::Other),
7800 Ops: {InputChain, StartVal, EndVal});
7801 setValue(V: &I, NewN: Res);
7802 DAG.setRoot(Res);
7803 return;
7804 }
7805 case Intrinsic::donothing:
7806 case Intrinsic::seh_try_begin:
7807 case Intrinsic::seh_scope_begin:
7808 case Intrinsic::seh_try_end:
7809 case Intrinsic::seh_scope_end:
7810 // ignore
7811 return;
7812 case Intrinsic::experimental_stackmap:
7813 visitStackmap(I);
7814 return;
7815 case Intrinsic::experimental_patchpoint_void:
7816 case Intrinsic::experimental_patchpoint:
7817 visitPatchpoint(CB: I);
7818 return;
7819 case Intrinsic::experimental_gc_statepoint:
7820 LowerStatepoint(I: cast<GCStatepointInst>(Val: I));
7821 return;
7822 case Intrinsic::experimental_gc_result:
7823 visitGCResult(I: cast<GCResultInst>(Val: I));
7824 return;
7825 case Intrinsic::experimental_gc_relocate:
7826 visitGCRelocate(Relocate: cast<GCRelocateInst>(Val: I));
7827 return;
7828 case Intrinsic::instrprof_cover:
7829 llvm_unreachable("instrprof failed to lower a cover");
7830 case Intrinsic::instrprof_increment:
7831 llvm_unreachable("instrprof failed to lower an increment");
7832 case Intrinsic::instrprof_timestamp:
7833 llvm_unreachable("instrprof failed to lower a timestamp");
7834 case Intrinsic::instrprof_value_profile:
7835 llvm_unreachable("instrprof failed to lower a value profiling call");
7836 case Intrinsic::instrprof_mcdc_parameters:
7837 llvm_unreachable("instrprof failed to lower mcdc parameters");
7838 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7839 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7840 case Intrinsic::localescape: {
7841 MachineFunction &MF = DAG.getMachineFunction();
7842 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
7843
7844 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7845 // is the same on all targets.
7846 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7847 Value *Arg = I.getArgOperand(i: Idx)->stripPointerCasts();
7848 if (isa<ConstantPointerNull>(Val: Arg))
7849 continue; // Skip null pointers. They represent a hole in index space.
7850 AllocaInst *Slot = cast<AllocaInst>(Val: Arg);
7851 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7852 "can only escape static allocas");
7853 int FI = FuncInfo.StaticAllocaMap[Slot];
7854 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7855 FuncName: GlobalValue::dropLLVMManglingEscape(Name: MF.getName()), Idx);
7856 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: dl,
7857 MCID: TII->get(Opcode: TargetOpcode::LOCAL_ESCAPE))
7858 .addSym(Sym: FrameAllocSym)
7859 .addFrameIndex(Idx: FI);
7860 }
7861
7862 return;
7863 }
7864
7865 case Intrinsic::localrecover: {
7866 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7867 MachineFunction &MF = DAG.getMachineFunction();
7868
7869 // Get the symbol that defines the frame offset.
7870 auto *Fn = cast<Function>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
7871 auto *Idx = cast<ConstantInt>(Val: I.getArgOperand(i: 2));
7872 unsigned IdxVal =
7873 unsigned(Idx->getLimitedValue(Limit: std::numeric_limits<int>::max()));
7874 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7875 FuncName: GlobalValue::dropLLVMManglingEscape(Name: Fn->getName()), Idx: IdxVal);
7876
7877 Value *FP = I.getArgOperand(i: 1);
7878 SDValue FPVal = getValue(V: FP);
7879 EVT PtrVT = FPVal.getValueType();
7880
7881 // Create a MCSymbol for the label to avoid any target lowering
7882 // that would make this PC relative.
7883 SDValue OffsetSym = DAG.getMCSymbol(Sym: FrameAllocSym, VT: PtrVT);
7884 SDValue OffsetVal =
7885 DAG.getNode(Opcode: ISD::LOCAL_RECOVER, DL: sdl, VT: PtrVT, Operand: OffsetSym);
7886
7887 // Add the offset to the FP.
7888 SDValue Add = DAG.getMemBasePlusOffset(Base: FPVal, Offset: OffsetVal, DL: sdl);
7889 setValue(V: &I, NewN: Add);
7890
7891 return;
7892 }
7893
7894 case Intrinsic::fake_use: {
7895 Value *V = I.getArgOperand(i: 0);
7896 SDValue Ops[2];
7897 // For Values not declared or previously used in this basic block, the
7898 // NodeMap will not have an entry, and `getValue` will assert if V has no
7899 // valid register value.
7900 auto FakeUseValue = [&]() -> SDValue {
7901 SDValue &N = NodeMap[V];
7902 if (N.getNode())
7903 return N;
7904
7905 // If there's a virtual register allocated and initialized for this
7906 // value, use it.
7907 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
7908 return copyFromReg;
7909 // FIXME: Do we want to preserve constants? It seems pointless.
7910 if (isa<Constant>(Val: V))
7911 return getValue(V);
7912 return SDValue();
7913 }();
7914 if (!FakeUseValue || FakeUseValue.isUndef())
7915 return;
7916 Ops[0] = getRoot();
7917 Ops[1] = FakeUseValue;
7918 // Also, do not translate a fake use with an undef operand, or any other
7919 // empty SDValues.
7920 if (!Ops[1] || Ops[1].isUndef())
7921 return;
7922 DAG.setRoot(DAG.getNode(Opcode: ISD::FAKE_USE, DL: sdl, VT: MVT::Other, Ops));
7923 return;
7924 }
7925
7926 case Intrinsic::reloc_none: {
7927 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
7928 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
7929 SDValue Ops[2] = {
7930 getRoot(),
7931 DAG.getTargetExternalSymbol(
7932 Sym: SymbolName.data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout()))};
7933 DAG.setRoot(DAG.getNode(Opcode: ISD::RELOC_NONE, DL: sdl, VT: MVT::Other, Ops));
7934 return;
7935 }
7936
7937 case Intrinsic::cond_loop: {
7938 SDValue InputChain = DAG.getRoot();
7939 SDValue P = getValue(V: I.getArgOperand(i: 0));
7940 Res = DAG.getNode(Opcode: ISD::COND_LOOP, DL: sdl, VTList: DAG.getVTList(VT: MVT::Other),
7941 Ops: {InputChain, P});
7942 setValue(V: &I, NewN: Res);
7943 DAG.setRoot(Res);
7944 return;
7945 }
7946
7947 case Intrinsic::eh_exceptionpointer:
7948 case Intrinsic::eh_exceptioncode: {
7949 // Get the exception pointer vreg, copy from it, and resize it to fit.
7950 const auto *CPI = cast<CatchPadInst>(Val: I.getArgOperand(i: 0));
7951 MVT PtrVT = TLI.getPointerTy(DL: DAG.getDataLayout());
7952 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(VT: PtrVT);
7953 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, RC: PtrRC);
7954 SDValue N = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: sdl, Reg: VReg, VT: PtrVT);
7955 if (Intrinsic == Intrinsic::eh_exceptioncode)
7956 N = DAG.getZExtOrTrunc(Op: N, DL: sdl, VT: MVT::i32);
7957 setValue(V: &I, NewN: N);
7958 return;
7959 }
7960 case Intrinsic::xray_customevent: {
7961 // Here we want to make sure that the intrinsic behaves as if it has a
7962 // specific calling convention.
7963 const auto &Triple = DAG.getTarget().getTargetTriple();
7964 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
7965 return;
7966
7967 SmallVector<SDValue, 8> Ops;
7968
7969 // We want to say that we always want the arguments in registers.
7970 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 0));
7971 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 1));
7972 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
7973 SDValue Chain = getRoot();
7974 Ops.push_back(Elt: LogEntryVal);
7975 Ops.push_back(Elt: StrSizeVal);
7976 Ops.push_back(Elt: Chain);
7977
7978 // We need to enforce the calling convention for the callsite, so that
7979 // argument ordering is enforced correctly, and that register allocation can
7980 // see that some registers may be assumed clobbered and have to preserve
7981 // them across calls to the intrinsic.
7982 MachineSDNode *MN = DAG.getMachineNode(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL,
7983 dl: sdl, VTs: NodeTys, Ops);
7984 SDValue patchableNode = SDValue(MN, 0);
7985 DAG.setRoot(patchableNode);
7986 setValue(V: &I, NewN: patchableNode);
7987 return;
7988 }
7989 case Intrinsic::xray_typedevent: {
7990 // Here we want to make sure that the intrinsic behaves as if it has a
7991 // specific calling convention.
7992 const auto &Triple = DAG.getTarget().getTargetTriple();
7993 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
7994 return;
7995
7996 SmallVector<SDValue, 8> Ops;
7997
7998 // We want to say that we always want the arguments in registers.
7999 // It's unclear to me how manipulating the selection DAG here forces callers
8000 // to provide arguments in registers instead of on the stack.
8001 SDValue LogTypeId = getValue(V: I.getArgOperand(i: 0));
8002 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 1));
8003 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 2));
8004 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
8005 SDValue Chain = getRoot();
8006 Ops.push_back(Elt: LogTypeId);
8007 Ops.push_back(Elt: LogEntryVal);
8008 Ops.push_back(Elt: StrSizeVal);
8009 Ops.push_back(Elt: Chain);
8010
8011 // We need to enforce the calling convention for the callsite, so that
8012 // argument ordering is enforced correctly, and that register allocation can
8013 // see that some registers may be assumed clobbered and have to preserve
8014 // them across calls to the intrinsic.
8015 MachineSDNode *MN = DAG.getMachineNode(
8016 Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, dl: sdl, VTs: NodeTys, Ops);
8017 SDValue patchableNode = SDValue(MN, 0);
8018 DAG.setRoot(patchableNode);
8019 setValue(V: &I, NewN: patchableNode);
8020 return;
8021 }
8022 case Intrinsic::experimental_deoptimize:
8023 LowerDeoptimizeCall(CI: &I);
8024 return;
8025 case Intrinsic::stepvector:
8026 visitStepVector(I);
8027 return;
8028 case Intrinsic::vector_reduce_fadd:
8029 case Intrinsic::vector_reduce_fmul:
8030 case Intrinsic::vector_reduce_add:
8031 case Intrinsic::vector_reduce_mul:
8032 case Intrinsic::vector_reduce_and:
8033 case Intrinsic::vector_reduce_or:
8034 case Intrinsic::vector_reduce_xor:
8035 case Intrinsic::vector_reduce_smax:
8036 case Intrinsic::vector_reduce_smin:
8037 case Intrinsic::vector_reduce_umax:
8038 case Intrinsic::vector_reduce_umin:
8039 case Intrinsic::vector_reduce_fmax:
8040 case Intrinsic::vector_reduce_fmin:
8041 case Intrinsic::vector_reduce_fmaximum:
8042 case Intrinsic::vector_reduce_fminimum:
8043 visitVectorReduce(I, Intrinsic);
8044 return;
8045
8046 case Intrinsic::icall_branch_funnel: {
8047 SmallVector<SDValue, 16> Ops;
8048 Ops.push_back(Elt: getValue(V: I.getArgOperand(i: 0)));
8049
8050 int64_t Offset;
8051 auto *Base = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
8052 Ptr: I.getArgOperand(i: 1), Offset, DL: DAG.getDataLayout()));
8053 if (!Base)
8054 report_fatal_error(
8055 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
8056 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Base, DL: sdl, VT: MVT::i64, offset: 0));
8057
8058 struct BranchFunnelTarget {
8059 int64_t Offset;
8060 SDValue Target;
8061 };
8062 SmallVector<BranchFunnelTarget, 8> Targets;
8063
8064 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
8065 auto *ElemBase = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
8066 Ptr: I.getArgOperand(i: Op), Offset, DL: DAG.getDataLayout()));
8067 if (ElemBase != Base)
8068 report_fatal_error(reason: "all llvm.icall.branch.funnel operands must refer "
8069 "to the same GlobalValue");
8070
8071 SDValue Val = getValue(V: I.getArgOperand(i: Op + 1));
8072 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
8073 if (!GA)
8074 report_fatal_error(
8075 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
8076 Targets.push_back(Elt: {.Offset: Offset, .Target: DAG.getTargetGlobalAddress(
8077 GV: GA->getGlobal(), DL: sdl, VT: Val.getValueType(),
8078 offset: GA->getOffset())});
8079 }
8080 llvm::sort(C&: Targets,
8081 Comp: [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
8082 return T1.Offset < T2.Offset;
8083 });
8084
8085 for (auto &T : Targets) {
8086 Ops.push_back(Elt: DAG.getTargetConstant(Val: T.Offset, DL: sdl, VT: MVT::i32));
8087 Ops.push_back(Elt: T.Target);
8088 }
8089
8090 Ops.push_back(Elt: DAG.getRoot()); // Chain
8091 SDValue N(DAG.getMachineNode(Opcode: TargetOpcode::ICALL_BRANCH_FUNNEL, dl: sdl,
8092 VT: MVT::Other, Ops),
8093 0);
8094 DAG.setRoot(N);
8095 setValue(V: &I, NewN: N);
8096 HasTailCall = true;
8097 return;
8098 }
8099
8100 case Intrinsic::wasm_landingpad_index:
8101 // Information this intrinsic contained has been transferred to
8102 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
8103 // delete it now.
8104 return;
8105
8106 case Intrinsic::aarch64_settag:
8107 case Intrinsic::aarch64_settag_zero: {
8108 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8109 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
8110 SDValue Val = TSI.EmitTargetCodeForSetTag(
8111 DAG, dl: sdl, Chain: getRoot(), Addr: getValue(V: I.getArgOperand(i: 0)),
8112 Size: getValue(V: I.getArgOperand(i: 1)), DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
8113 ZeroData: ZeroMemory);
8114 DAG.setRoot(Val);
8115 setValue(V: &I, NewN: Val);
8116 return;
8117 }
8118 case Intrinsic::amdgcn_cs_chain: {
8119 // At this point we don't care if it's amdgpu_cs_chain or
8120 // amdgpu_cs_chain_preserve.
8121 CallingConv::ID CC = CallingConv::AMDGPU_CS_Chain;
8122
8123 Type *RetTy = I.getType();
8124 assert(RetTy->isVoidTy() && "Should not return");
8125
8126 SDValue Callee = getValue(V: I.getOperand(i_nocapture: 0));
8127
8128 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
8129 // We'll also tack the value of the EXEC mask at the end.
8130 TargetLowering::ArgListTy Args;
8131 Args.reserve(n: 3);
8132
8133 for (unsigned Idx : {2, 3, 1}) {
8134 TargetLowering::ArgListEntry Arg(getValue(V: I.getOperand(i_nocapture: Idx)),
8135 I.getOperand(i_nocapture: Idx)->getType());
8136 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8137 Args.push_back(x: Arg);
8138 }
8139
8140 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
8141 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
8142 Args[2].IsInReg = true; // EXEC should be inreg
8143
8144 // Forward the flags and any additional arguments.
8145 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
8146 TargetLowering::ArgListEntry Arg(getValue(V: I.getOperand(i_nocapture: Idx)),
8147 I.getOperand(i_nocapture: Idx)->getType());
8148 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8149 Args.push_back(x: Arg);
8150 }
8151
8152 TargetLowering::CallLoweringInfo CLI(DAG);
8153 CLI.setDebugLoc(getCurSDLoc())
8154 .setChain(getRoot())
8155 .setCallee(CC, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
8156 .setNoReturn(true)
8157 .setTailCall(true)
8158 .setConvergent(I.isConvergent());
8159 CLI.CB = &I;
8160 std::pair<SDValue, SDValue> Result =
8161 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
8162 (void)Result;
8163 assert(!Result.first.getNode() && !Result.second.getNode() &&
8164 "Should've lowered as tail call");
8165
8166 HasTailCall = true;
8167 return;
8168 }
8169 case Intrinsic::amdgcn_call_whole_wave: {
8170 TargetLowering::ArgListTy Args;
8171 bool isTailCall = I.isTailCall();
8172
8173 // The first argument is the callee. Skip it when assembling the call args.
8174 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
8175 TargetLowering::ArgListEntry Arg(getValue(V: I.getArgOperand(i: Idx)),
8176 I.getArgOperand(i: Idx)->getType());
8177 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8178
8179 // If we have an explicit sret argument that is an Instruction, (i.e., it
8180 // might point to function-local memory), we can't meaningfully tail-call.
8181 if (Arg.IsSRet && isa<Instruction>(Val: I.getArgOperand(i: Idx)))
8182 isTailCall = false;
8183
8184 Args.push_back(x: Arg);
8185 }
8186
8187 SDValue ConvControlToken;
8188 if (auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
8189 auto *Token = Bundle->Inputs[0].get();
8190 ConvControlToken = getValue(V: Token);
8191 }
8192
8193 TargetLowering::CallLoweringInfo CLI(DAG);
8194 CLI.setDebugLoc(getCurSDLoc())
8195 .setChain(getRoot())
8196 .setCallee(CC: CallingConv::AMDGPU_Gfx_WholeWave, ResultType: I.getType(),
8197 Target: getValue(V: I.getArgOperand(i: 0)), ArgsList: std::move(Args))
8198 .setTailCall(isTailCall && canTailCall(CB: I))
8199 .setIsPreallocated(
8200 I.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
8201 .setConvergent(I.isConvergent())
8202 .setConvergenceControlToken(ConvControlToken);
8203 CLI.CB = &I;
8204
8205 std::pair<SDValue, SDValue> Result =
8206 lowerInvokable(CLI, /*EHPadBB=*/nullptr);
8207
8208 if (Result.first.getNode())
8209 setValue(V: &I, NewN: Result.first);
8210 return;
8211 }
8212 case Intrinsic::ptrmask: {
8213 SDValue Ptr = getValue(V: I.getOperand(i_nocapture: 0));
8214 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
8215
8216 // On arm64_32, pointers are 32 bits when stored in memory, but
8217 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
8218 // match the index type, but the pointer is 64 bits, so the mask must be
8219 // zero-extended up to 64 bits to match the pointer.
8220 EVT PtrVT =
8221 TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
8222 EVT MemVT =
8223 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
8224 assert(PtrVT == Ptr.getValueType());
8225 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
8226 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
8227 // 128-bit, so we have to pad the mask with ones for unused bits.
8228 auto HighOnes = DAG.getNode(
8229 Opcode: ISD::SHL, DL: sdl, VT: PtrVT, N1: DAG.getAllOnesConstant(DL: sdl, VT: PtrVT),
8230 N2: DAG.getShiftAmountConstant(Val: Mask.getValueType().getFixedSizeInBits(),
8231 VT: PtrVT, DL: sdl));
8232 Mask = DAG.getNode(Opcode: ISD::OR, DL: sdl, VT: PtrVT,
8233 N1: DAG.getZExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT), N2: HighOnes);
8234 } else if (Mask.getValueType() != PtrVT)
8235 Mask = DAG.getPtrExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT);
8236
8237 assert(Mask.getValueType() == PtrVT);
8238 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::AND, DL: sdl, VT: PtrVT, N1: Ptr, N2: Mask));
8239 return;
8240 }
8241 case Intrinsic::threadlocal_address: {
8242 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
8243 return;
8244 }
8245 case Intrinsic::get_active_lane_mask: {
8246 EVT CCVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8247 SDValue Index = getValue(V: I.getOperand(i_nocapture: 0));
8248 SDValue TripCount = getValue(V: I.getOperand(i_nocapture: 1));
8249 EVT ElementVT = Index.getValueType();
8250
8251 if (!TLI.shouldExpandGetActiveLaneMask(VT: CCVT, OpVT: ElementVT)) {
8252 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: sdl, VT: CCVT, N1: Index,
8253 N2: TripCount));
8254 return;
8255 }
8256
8257 EVT VecTy = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElementVT,
8258 EC: CCVT.getVectorElementCount());
8259
8260 SDValue VectorIndex = DAG.getSplat(VT: VecTy, DL: sdl, Op: Index);
8261 SDValue VectorTripCount = DAG.getSplat(VT: VecTy, DL: sdl, Op: TripCount);
8262 SDValue VectorStep = DAG.getStepVector(DL: sdl, ResVT: VecTy);
8263 SDValue VectorInduction = DAG.getNode(
8264 Opcode: ISD::UADDSAT, DL: sdl, VT: VecTy, N1: VectorIndex, N2: VectorStep);
8265 SDValue SetCC = DAG.getSetCC(DL: sdl, VT: CCVT, LHS: VectorInduction,
8266 RHS: VectorTripCount, Cond: ISD::CondCode::SETULT);
8267 setValue(V: &I, NewN: SetCC);
8268 return;
8269 }
8270 case Intrinsic::experimental_get_vector_length: {
8271 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8272 "Expected positive VF");
8273 unsigned VF = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 1))->getZExtValue();
8274 bool IsScalable = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->isOne();
8275
8276 SDValue Count = getValue(V: I.getOperand(i_nocapture: 0));
8277 EVT CountVT = Count.getValueType();
8278
8279 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8280 visitTargetIntrinsic(I, Intrinsic);
8281 return;
8282 }
8283
8284 // Expand to a umin between the trip count and the maximum elements the type
8285 // can hold.
8286 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8287
8288 // Extend the trip count to at least the result VT.
8289 if (CountVT.bitsLT(VT)) {
8290 Count = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: sdl, VT, Operand: Count);
8291 CountVT = VT;
8292 }
8293
8294 SDValue MaxEVL = DAG.getElementCount(DL: sdl, VT: CountVT,
8295 EC: ElementCount::get(MinVal: VF, Scalable: IsScalable));
8296
8297 SDValue UMin = DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: CountVT, N1: Count, N2: MaxEVL);
8298 // Clip to the result type if needed.
8299 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: sdl, VT, Operand: UMin);
8300
8301 setValue(V: &I, NewN: Trunc);
8302 return;
8303 }
8304 case Intrinsic::vector_partial_reduce_add: {
8305 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8306 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8307 setValue(V: &I,
8308 NewN: DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_UMLA, DL: sdl, VT: Acc.getValueType(), N1: Acc,
8309 N2: Input, N3: DAG.getConstant(Val: 1, DL: sdl, VT: Input.getValueType())));
8310 return;
8311 }
8312 case Intrinsic::vector_partial_reduce_fadd: {
8313 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8314 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8315 setValue(V: &I, NewN: DAG.getNode(
8316 Opcode: ISD::PARTIAL_REDUCE_FMLA, DL: sdl, VT: Acc.getValueType(), N1: Acc,
8317 N2: Input, N3: DAG.getConstantFP(Val: 1.0, DL: sdl, VT: Input.getValueType())));
8318 return;
8319 }
8320 case Intrinsic::experimental_cttz_elts: {
8321 auto DL = getCurSDLoc();
8322 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
8323 EVT OpVT = Op.getValueType();
8324 EVT RetTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8325 bool ZeroIsPoison =
8326 !cast<ConstantSDNode>(Val: getValue(V: I.getOperand(i_nocapture: 1)))->isZero();
8327
8328 if (!TLI.shouldExpandCttzElements(VT: OpVT)) {
8329 SDValue Ret = DAG.getNode(Opcode: ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
8330 : ISD::CTTZ_ELTS,
8331 DL: sdl, VT: RetTy, Operand: Op);
8332 setValue(V: &I, NewN: Ret);
8333 return;
8334 }
8335
8336 if (OpVT.getScalarType() != MVT::i1) {
8337 // Compare the input vector elements to zero & use to count trailing zeros
8338 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: OpVT);
8339 OpVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8340 EC: OpVT.getVectorElementCount());
8341 Op = DAG.getSetCC(DL, VT: OpVT, LHS: Op, RHS: AllZero, Cond: ISD::SETNE);
8342 }
8343
8344 // If the zero-is-poison flag is set, we can assume the upper limit
8345 // of the result is VF-1.
8346 ConstantRange VScaleRange(1, true); // Dummy value.
8347 if (isa<ScalableVectorType>(Val: I.getOperand(i_nocapture: 0)->getType()))
8348 VScaleRange = getVScaleRange(F: I.getCaller(), BitWidth: 64);
8349 unsigned EltWidth = TLI.getBitWidthForCttzElements(
8350 RetTy: I.getType(), EC: OpVT.getVectorElementCount(), ZeroIsPoison, VScaleRange: &VScaleRange);
8351
8352 MVT NewEltTy = MVT::getIntegerVT(BitWidth: EltWidth);
8353
8354 // Create the new vector type & get the vector length
8355 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltTy,
8356 EC: OpVT.getVectorElementCount());
8357
8358 SDValue VL =
8359 DAG.getElementCount(DL, VT: NewEltTy, EC: OpVT.getVectorElementCount());
8360
8361 SDValue StepVec = DAG.getStepVector(DL, ResVT: NewVT);
8362 SDValue SplatVL = DAG.getSplat(VT: NewVT, DL, Op: VL);
8363 SDValue StepVL = DAG.getNode(Opcode: ISD::SUB, DL, VT: NewVT, N1: SplatVL, N2: StepVec);
8364 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewVT, Operand: Op);
8365 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: NewVT, N1: StepVL, N2: Ext);
8366 SDValue Max = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL, VT: NewEltTy, Operand: And);
8367 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: NewEltTy, N1: VL, N2: Max);
8368
8369 SDValue Ret = DAG.getZExtOrTrunc(Op: Sub, DL, VT: RetTy);
8370
8371 setValue(V: &I, NewN: Ret);
8372 return;
8373 }
8374 case Intrinsic::vector_insert: {
8375 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8376 SDValue SubVec = getValue(V: I.getOperand(i_nocapture: 1));
8377 SDValue Index = getValue(V: I.getOperand(i_nocapture: 2));
8378
8379 // The intrinsic's index type is i64, but the SDNode requires an index type
8380 // suitable for the target. Convert the index as required.
8381 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8382 if (Index.getValueType() != VectorIdxTy)
8383 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8384
8385 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8386 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: SubVec,
8387 N3: Index));
8388 return;
8389 }
8390 case Intrinsic::vector_extract: {
8391 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8392 SDValue Index = getValue(V: I.getOperand(i_nocapture: 1));
8393 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8394
8395 // The intrinsic's index type is i64, but the SDNode requires an index type
8396 // suitable for the target. Convert the index as required.
8397 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8398 if (Index.getValueType() != VectorIdxTy)
8399 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8400
8401 setValue(V: &I,
8402 NewN: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: Index));
8403 return;
8404 }
8405 case Intrinsic::experimental_vector_match: {
8406 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
8407 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
8408 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
8409 EVT Op1VT = Op1.getValueType();
8410 EVT Op2VT = Op2.getValueType();
8411 EVT ResVT = Mask.getValueType();
8412 unsigned SearchSize = Op2VT.getVectorNumElements();
8413
8414 // If the target has native support for this vector match operation, lower
8415 // the intrinsic untouched; otherwise, expand it below.
8416 if (!TLI.shouldExpandVectorMatch(VT: Op1VT, SearchSize)) {
8417 visitTargetIntrinsic(I, Intrinsic);
8418 return;
8419 }
8420
8421 SDValue Ret = DAG.getConstant(Val: 0, DL: sdl, VT: ResVT);
8422
8423 for (unsigned i = 0; i < SearchSize; ++i) {
8424 SDValue Op2Elem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: sdl,
8425 VT: Op2VT.getVectorElementType(), N1: Op2,
8426 N2: DAG.getVectorIdxConstant(Val: i, DL: sdl));
8427 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: sdl, VT: Op1VT, Operand: Op2Elem);
8428 SDValue Cmp = DAG.getSetCC(DL: sdl, VT: ResVT, LHS: Op1, RHS: Splat, Cond: ISD::SETEQ);
8429 Ret = DAG.getNode(Opcode: ISD::OR, DL: sdl, VT: ResVT, N1: Ret, N2: Cmp);
8430 }
8431
8432 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::AND, DL: sdl, VT: ResVT, N1: Ret, N2: Mask));
8433 return;
8434 }
8435 case Intrinsic::vector_reverse:
8436 visitVectorReverse(I);
8437 return;
8438 case Intrinsic::vector_splice_left:
8439 case Intrinsic::vector_splice_right:
8440 visitVectorSplice(I);
8441 return;
8442 case Intrinsic::callbr_landingpad:
8443 visitCallBrLandingPad(I);
8444 return;
8445 case Intrinsic::vector_interleave2:
8446 visitVectorInterleave(I, Factor: 2);
8447 return;
8448 case Intrinsic::vector_interleave3:
8449 visitVectorInterleave(I, Factor: 3);
8450 return;
8451 case Intrinsic::vector_interleave4:
8452 visitVectorInterleave(I, Factor: 4);
8453 return;
8454 case Intrinsic::vector_interleave5:
8455 visitVectorInterleave(I, Factor: 5);
8456 return;
8457 case Intrinsic::vector_interleave6:
8458 visitVectorInterleave(I, Factor: 6);
8459 return;
8460 case Intrinsic::vector_interleave7:
8461 visitVectorInterleave(I, Factor: 7);
8462 return;
8463 case Intrinsic::vector_interleave8:
8464 visitVectorInterleave(I, Factor: 8);
8465 return;
8466 case Intrinsic::vector_deinterleave2:
8467 visitVectorDeinterleave(I, Factor: 2);
8468 return;
8469 case Intrinsic::vector_deinterleave3:
8470 visitVectorDeinterleave(I, Factor: 3);
8471 return;
8472 case Intrinsic::vector_deinterleave4:
8473 visitVectorDeinterleave(I, Factor: 4);
8474 return;
8475 case Intrinsic::vector_deinterleave5:
8476 visitVectorDeinterleave(I, Factor: 5);
8477 return;
8478 case Intrinsic::vector_deinterleave6:
8479 visitVectorDeinterleave(I, Factor: 6);
8480 return;
8481 case Intrinsic::vector_deinterleave7:
8482 visitVectorDeinterleave(I, Factor: 7);
8483 return;
8484 case Intrinsic::vector_deinterleave8:
8485 visitVectorDeinterleave(I, Factor: 8);
8486 return;
8487 case Intrinsic::experimental_vector_compress:
8488 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: sdl,
8489 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
8490 N1: getValue(V: I.getArgOperand(i: 0)),
8491 N2: getValue(V: I.getArgOperand(i: 1)),
8492 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
8493 return;
8494 case Intrinsic::experimental_convergence_anchor:
8495 case Intrinsic::experimental_convergence_entry:
8496 case Intrinsic::experimental_convergence_loop:
8497 visitConvergenceControl(I, Intrinsic);
8498 return;
8499 case Intrinsic::experimental_vector_histogram_add: {
8500 visitVectorHistogram(I, IntrinsicID: Intrinsic);
8501 return;
8502 }
8503 case Intrinsic::experimental_vector_extract_last_active: {
8504 visitVectorExtractLastActive(I, Intrinsic);
8505 return;
8506 }
8507 case Intrinsic::loop_dependence_war_mask:
8508 setValue(V: &I,
8509 NewN: DAG.getNode(Opcode: ISD::LOOP_DEPENDENCE_WAR_MASK, DL: sdl,
8510 VT: EVT::getEVT(Ty: I.getType()), N1: getValue(V: I.getOperand(i_nocapture: 0)),
8511 N2: getValue(V: I.getOperand(i_nocapture: 1)), N3: getValue(V: I.getOperand(i_nocapture: 2)),
8512 N4: DAG.getConstant(Val: 0, DL: sdl, VT: MVT::i64)));
8513 return;
8514 case Intrinsic::loop_dependence_raw_mask:
8515 setValue(V: &I,
8516 NewN: DAG.getNode(Opcode: ISD::LOOP_DEPENDENCE_RAW_MASK, DL: sdl,
8517 VT: EVT::getEVT(Ty: I.getType()), N1: getValue(V: I.getOperand(i_nocapture: 0)),
8518 N2: getValue(V: I.getOperand(i_nocapture: 1)), N3: getValue(V: I.getOperand(i_nocapture: 2)),
8519 N4: DAG.getConstant(Val: 0, DL: sdl, VT: MVT::i64)));
8520 return;
8521 }
8522}
8523
8524void SelectionDAGBuilder::pushFPOpOutChain(SDValue Result,
8525 fp::ExceptionBehavior EB) {
8526 assert(Result.getNode()->getNumValues() == 2);
8527 SDValue OutChain = Result.getValue(R: 1);
8528 assert(OutChain.getValueType() == MVT::Other);
8529
8530 // Instead of updating the root immediately, push the produced chain to the
8531 // appropriate list, deferring the update until the root is requested. In this
8532 // case, the nodes from the lists are chained using TokenFactor, indicating
8533 // that the operations are independent.
8534 //
8535 // In particular, the root is updated before any call that might access the
8536 // floating-point environment, except for constrained intrinsics.
8537 switch (EB) {
8538 case fp::ExceptionBehavior::ebMayTrap:
8539 case fp::ExceptionBehavior::ebIgnore:
8540 PendingConstrainedFP.push_back(Elt: OutChain);
8541 break;
8542 case fp::ExceptionBehavior::ebStrict:
8543 PendingConstrainedFPStrict.push_back(Elt: OutChain);
8544 break;
8545 }
8546}
8547
8548void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8549 const ConstrainedFPIntrinsic &FPI) {
8550 SDLoc sdl = getCurSDLoc();
8551
8552 // We do not need to serialize constrained FP intrinsics against
8553 // each other or against (nonvolatile) loads, so they can be
8554 // chained like loads.
8555 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
8556 SDValue Chain = getFPOperationRoot(EB);
8557 SmallVector<SDValue, 4> Opers;
8558 Opers.push_back(Elt: Chain);
8559 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8560 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: I)));
8561
8562 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8563 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: FPI.getType());
8564 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
8565
8566 SDNodeFlags Flags;
8567 if (EB == fp::ExceptionBehavior::ebIgnore)
8568 Flags.setNoFPExcept(true);
8569
8570 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &FPI))
8571 Flags.copyFMF(FPMO: *FPOp);
8572
8573 unsigned Opcode;
8574 switch (FPI.getIntrinsicID()) {
8575 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8576#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8577 case Intrinsic::INTRINSIC: \
8578 Opcode = ISD::STRICT_##DAGN; \
8579 break;
8580#include "llvm/IR/ConstrainedOps.def"
8581 case Intrinsic::experimental_constrained_fmuladd: {
8582 Opcode = ISD::STRICT_FMA;
8583 // Break fmuladd into fmul and fadd.
8584 if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
8585 !TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
8586 Opers.pop_back();
8587 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8588 pushFPOpOutChain(Result: Mul, EB);
8589 Opcode = ISD::STRICT_FADD;
8590 Opers.clear();
8591 Opers.push_back(Elt: Mul.getValue(R: 1));
8592 Opers.push_back(Elt: Mul.getValue(R: 0));
8593 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: 2)));
8594 }
8595 break;
8596 }
8597 }
8598
8599 // A few strict DAG nodes carry additional operands that are not
8600 // set up by the default code above.
8601 switch (Opcode) {
8602 default: break;
8603 case ISD::STRICT_FP_ROUND:
8604 Opers.push_back(
8605 Elt: DAG.getTargetConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
8606 break;
8607 case ISD::STRICT_FSETCC:
8608 case ISD::STRICT_FSETCCS: {
8609 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
8610 ISD::CondCode Condition = getFCmpCondCode(Pred: FPCmp->getPredicate());
8611 if (DAG.isKnownNeverNaN(Op: Opers[1]) && DAG.isKnownNeverNaN(Op: Opers[2]))
8612 Condition = getFCmpCodeWithoutNaN(CC: Condition);
8613 Opers.push_back(Elt: DAG.getCondCode(Cond: Condition));
8614 break;
8615 }
8616 }
8617
8618 SDValue Result = DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8619 pushFPOpOutChain(Result, EB);
8620
8621 SDValue FPResult = Result.getValue(R: 0);
8622 setValue(V: &FPI, NewN: FPResult);
8623}
8624
8625static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8626 std::optional<unsigned> ResOPC;
8627 switch (VPIntrin.getIntrinsicID()) {
8628 case Intrinsic::vp_ctlz: {
8629 bool IsZeroUndef = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8630 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8631 break;
8632 }
8633 case Intrinsic::vp_cttz: {
8634 bool IsZeroUndef = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8635 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8636 break;
8637 }
8638 case Intrinsic::vp_cttz_elts: {
8639 bool IsZeroPoison = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8640 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_UNDEF : ISD::VP_CTTZ_ELTS;
8641 break;
8642 }
8643#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8644 case Intrinsic::VPID: \
8645 ResOPC = ISD::VPSD; \
8646 break;
8647#include "llvm/IR/VPIntrinsics.def"
8648 }
8649
8650 if (!ResOPC)
8651 llvm_unreachable(
8652 "Inconsistency: no SDNode available for this VPIntrinsic!");
8653
8654 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8655 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8656 if (VPIntrin.getFastMathFlags().allowReassoc())
8657 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8658 : ISD::VP_REDUCE_FMUL;
8659 }
8660
8661 return *ResOPC;
8662}
8663
8664void SelectionDAGBuilder::visitVPLoad(
8665 const VPIntrinsic &VPIntrin, EVT VT,
8666 const SmallVectorImpl<SDValue> &OpValues) {
8667 SDLoc DL = getCurSDLoc();
8668 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8669 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8670 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8671 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8672 SDValue LD;
8673 // Do not serialize variable-length loads of constant memory with
8674 // anything.
8675 if (!Alignment)
8676 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8677 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8678 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8679 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8680 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8681 MachineMemOperand::Flags MMOFlags =
8682 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8683 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8684 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
8685 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo, Ranges);
8686 LD = DAG.getLoadVP(VT, dl: DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
8687 MMO, IsExpanding: false /*IsExpanding */);
8688 if (AddToChain)
8689 PendingLoads.push_back(Elt: LD.getValue(R: 1));
8690 setValue(V: &VPIntrin, NewN: LD);
8691}
8692
8693void SelectionDAGBuilder::visitVPLoadFF(
8694 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,
8695 const SmallVectorImpl<SDValue> &OpValues) {
8696 assert(OpValues.size() == 3 && "Unexpected number of operands");
8697 SDLoc DL = getCurSDLoc();
8698 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8699 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8700 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8701 const MDNode *Ranges = VPIntrin.getMetadata(KindID: LLVMContext::MD_range);
8702 SDValue LD;
8703 // Do not serialize variable-length loads of constant memory with
8704 // anything.
8705 if (!Alignment)
8706 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8707 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8708 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8709 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8710 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8711 PtrInfo: MachinePointerInfo(PtrOperand), F: MachineMemOperand::MOLoad,
8712 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo, Ranges);
8713 LD = DAG.getLoadFFVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
8714 MMO);
8715 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EVLVT, Operand: LD.getValue(R: 1));
8716 if (AddToChain)
8717 PendingLoads.push_back(Elt: LD.getValue(R: 2));
8718 setValue(V: &VPIntrin, NewN: DAG.getMergeValues(Ops: {LD.getValue(R: 0), Trunc}, dl: DL));
8719}
8720
8721void SelectionDAGBuilder::visitVPGather(
8722 const VPIntrinsic &VPIntrin, EVT VT,
8723 const SmallVectorImpl<SDValue> &OpValues) {
8724 SDLoc DL = getCurSDLoc();
8725 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8726 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8727 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8728 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8729 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8730 SDValue LD;
8731 if (!Alignment)
8732 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8733 unsigned AS =
8734 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8735 MachineMemOperand::Flags MMOFlags =
8736 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8737 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8738 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
8739 BaseAlignment: *Alignment, AAInfo, Ranges);
8740 SDValue Base, Index, Scale;
8741 bool UniformBase =
8742 getUniformBase(Ptr: PtrOperand, Base, Index, Scale, SDB: this, CurBB: VPIntrin.getParent(),
8743 ElemSize: VT.getScalarStoreSize());
8744 if (!UniformBase) {
8745 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8746 Index = getValue(V: PtrOperand);
8747 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8748 }
8749 EVT IdxVT = Index.getValueType();
8750 EVT EltTy = IdxVT.getVectorElementType();
8751 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
8752 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
8753 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
8754 }
8755 LD = DAG.getGatherVP(
8756 VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), VT, dl: DL,
8757 Ops: {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8758 IndexType: ISD::SIGNED_SCALED);
8759 PendingLoads.push_back(Elt: LD.getValue(R: 1));
8760 setValue(V: &VPIntrin, NewN: LD);
8761}
8762
8763void SelectionDAGBuilder::visitVPStore(
8764 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8765 SDLoc DL = getCurSDLoc();
8766 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
8767 EVT VT = OpValues[0].getValueType();
8768 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8769 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8770 SDValue ST;
8771 if (!Alignment)
8772 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8773 SDValue Ptr = OpValues[1];
8774 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
8775 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8776 MachineMemOperand::Flags MMOFlags =
8777 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8778 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8779 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
8780 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo);
8781 ST = DAG.getStoreVP(Chain: getMemoryRoot(), dl: DL, Val: OpValues[0], Ptr, Offset,
8782 Mask: OpValues[2], EVL: OpValues[3], MemVT: VT, MMO, AM: ISD::UNINDEXED,
8783 /* IsTruncating */ false, /*IsCompressing*/ false);
8784 DAG.setRoot(ST);
8785 setValue(V: &VPIntrin, NewN: ST);
8786}
8787
8788void SelectionDAGBuilder::visitVPScatter(
8789 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8790 SDLoc DL = getCurSDLoc();
8791 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8792 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
8793 EVT VT = OpValues[0].getValueType();
8794 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8795 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8796 SDValue ST;
8797 if (!Alignment)
8798 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8799 unsigned AS =
8800 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8801 MachineMemOperand::Flags MMOFlags =
8802 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8803 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8804 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
8805 BaseAlignment: *Alignment, AAInfo);
8806 SDValue Base, Index, Scale;
8807 bool UniformBase =
8808 getUniformBase(Ptr: PtrOperand, Base, Index, Scale, SDB: this, CurBB: VPIntrin.getParent(),
8809 ElemSize: VT.getScalarStoreSize());
8810 if (!UniformBase) {
8811 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8812 Index = getValue(V: PtrOperand);
8813 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8814 }
8815 EVT IdxVT = Index.getValueType();
8816 EVT EltTy = IdxVT.getVectorElementType();
8817 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
8818 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
8819 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
8820 }
8821 ST = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT, dl: DL,
8822 Ops: {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8823 OpValues[2], OpValues[3]},
8824 MMO, IndexType: ISD::SIGNED_SCALED);
8825 DAG.setRoot(ST);
8826 setValue(V: &VPIntrin, NewN: ST);
8827}
8828
8829void SelectionDAGBuilder::visitVPStridedLoad(
8830 const VPIntrinsic &VPIntrin, EVT VT,
8831 const SmallVectorImpl<SDValue> &OpValues) {
8832 SDLoc DL = getCurSDLoc();
8833 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8834 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8835 if (!Alignment)
8836 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8837 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8838 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8839 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8840 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8841 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8842 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8843 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8844 MachineMemOperand::Flags MMOFlags =
8845 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8846 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8847 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
8848 BaseAlignment: *Alignment, AAInfo, Ranges);
8849
8850 SDValue LD = DAG.getStridedLoadVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Stride: OpValues[1],
8851 Mask: OpValues[2], EVL: OpValues[3], MMO,
8852 IsExpanding: false /*IsExpanding*/);
8853
8854 if (AddToChain)
8855 PendingLoads.push_back(Elt: LD.getValue(R: 1));
8856 setValue(V: &VPIntrin, NewN: LD);
8857}
8858
8859void SelectionDAGBuilder::visitVPStridedStore(
8860 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8861 SDLoc DL = getCurSDLoc();
8862 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
8863 EVT VT = OpValues[0].getValueType();
8864 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8865 if (!Alignment)
8866 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8867 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8868 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8869 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8870 MachineMemOperand::Flags MMOFlags =
8871 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8872 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8873 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
8874 BaseAlignment: *Alignment, AAInfo);
8875
8876 SDValue ST = DAG.getStridedStoreVP(
8877 Chain: getMemoryRoot(), DL, Val: OpValues[0], Ptr: OpValues[1],
8878 Offset: DAG.getUNDEF(VT: OpValues[1].getValueType()), Stride: OpValues[2], Mask: OpValues[3],
8879 EVL: OpValues[4], MemVT: VT, MMO, AM: ISD::UNINDEXED, /*IsTruncating*/ false,
8880 /*IsCompressing*/ false);
8881
8882 DAG.setRoot(ST);
8883 setValue(V: &VPIntrin, NewN: ST);
8884}
8885
8886void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8887 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8888 SDLoc DL = getCurSDLoc();
8889
8890 ISD::CondCode Condition;
8891 CmpInst::Predicate CondCode = VPIntrin.getPredicate();
8892 bool IsFP = VPIntrin.getOperand(i_nocapture: 0)->getType()->isFPOrFPVectorTy();
8893 Condition = IsFP ? getFCmpCondCode(Pred: CondCode) : getICmpCondCode(Pred: CondCode);
8894
8895 SDValue Op1 = getValue(V: VPIntrin.getOperand(i_nocapture: 0));
8896 SDValue Op2 = getValue(V: VPIntrin.getOperand(i_nocapture: 1));
8897 // #2 is the condition code
8898 SDValue MaskOp = getValue(V: VPIntrin.getOperand(i_nocapture: 3));
8899 SDValue EVL = getValue(V: VPIntrin.getOperand(i_nocapture: 4));
8900 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8901 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8902 "Unexpected target EVL type");
8903 EVL = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EVLParamVT, Operand: EVL);
8904
8905 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
8906 Ty: VPIntrin.getType());
8907 if (DAG.isKnownNeverNaN(Op: Op1) && DAG.isKnownNeverNaN(Op: Op2))
8908 Condition = getFCmpCodeWithoutNaN(CC: Condition);
8909 setValue(V: &VPIntrin,
8910 NewN: DAG.getSetCCVP(DL, VT: DestVT, LHS: Op1, RHS: Op2, Cond: Condition, Mask: MaskOp, EVL));
8911}
8912
8913void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8914 const VPIntrinsic &VPIntrin) {
8915 SDLoc DL = getCurSDLoc();
8916 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8917
8918 auto IID = VPIntrin.getIntrinsicID();
8919
8920 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(Val: &VPIntrin))
8921 return visitVPCmp(VPIntrin: *CmpI);
8922
8923 SmallVector<EVT, 4> ValueVTs;
8924 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8925 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: VPIntrin.getType(), ValueVTs);
8926 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
8927
8928 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IntrinsicID: IID);
8929
8930 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8931 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8932 "Unexpected target EVL type");
8933
8934 // Request operands.
8935 SmallVector<SDValue, 7> OpValues;
8936 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8937 auto Op = getValue(V: VPIntrin.getArgOperand(i: I));
8938 if (I == EVLParamPos)
8939 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EVLParamVT, Operand: Op);
8940 OpValues.push_back(Elt: Op);
8941 }
8942
8943 switch (Opcode) {
8944 default: {
8945 SDNodeFlags SDFlags;
8946 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &VPIntrin))
8947 SDFlags.copyFMF(FPMO: *FPMO);
8948 SDValue Result = DAG.getNode(Opcode, DL, VTList: VTs, Ops: OpValues, Flags: SDFlags);
8949 setValue(V: &VPIntrin, NewN: Result);
8950 break;
8951 }
8952 case ISD::VP_LOAD:
8953 visitVPLoad(VPIntrin, VT: ValueVTs[0], OpValues);
8954 break;
8955 case ISD::VP_LOAD_FF:
8956 visitVPLoadFF(VPIntrin, VT: ValueVTs[0], EVLVT: ValueVTs[1], OpValues);
8957 break;
8958 case ISD::VP_GATHER:
8959 visitVPGather(VPIntrin, VT: ValueVTs[0], OpValues);
8960 break;
8961 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8962 visitVPStridedLoad(VPIntrin, VT: ValueVTs[0], OpValues);
8963 break;
8964 case ISD::VP_STORE:
8965 visitVPStore(VPIntrin, OpValues);
8966 break;
8967 case ISD::VP_SCATTER:
8968 visitVPScatter(VPIntrin, OpValues);
8969 break;
8970 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8971 visitVPStridedStore(VPIntrin, OpValues);
8972 break;
8973 case ISD::VP_FMULADD: {
8974 assert(OpValues.size() == 5 && "Unexpected number of operands");
8975 SDNodeFlags SDFlags;
8976 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &VPIntrin))
8977 SDFlags.copyFMF(FPMO: *FPMO);
8978 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
8979 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), ValueVTs[0])) {
8980 setValue(V: &VPIntrin, NewN: DAG.getNode(Opcode: ISD::VP_FMA, DL, VTList: VTs, Ops: OpValues, Flags: SDFlags));
8981 } else {
8982 SDValue Mul = DAG.getNode(
8983 Opcode: ISD::VP_FMUL, DL, VTList: VTs,
8984 Ops: {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, Flags: SDFlags);
8985 SDValue Add =
8986 DAG.getNode(Opcode: ISD::VP_FADD, DL, VTList: VTs,
8987 Ops: {Mul, OpValues[2], OpValues[3], OpValues[4]}, Flags: SDFlags);
8988 setValue(V: &VPIntrin, NewN: Add);
8989 }
8990 break;
8991 }
8992 case ISD::VP_IS_FPCLASS: {
8993 const DataLayout DLayout = DAG.getDataLayout();
8994 EVT DestVT = TLI.getValueType(DL: DLayout, Ty: VPIntrin.getType());
8995 auto Constant = OpValues[1]->getAsZExtVal();
8996 SDValue Check = DAG.getTargetConstant(Val: Constant, DL, VT: MVT::i32);
8997 SDValue V = DAG.getNode(Opcode: ISD::VP_IS_FPCLASS, DL, VT: DestVT,
8998 Ops: {OpValues[0], Check, OpValues[2], OpValues[3]});
8999 setValue(V: &VPIntrin, NewN: V);
9000 return;
9001 }
9002 case ISD::VP_INTTOPTR: {
9003 SDValue N = OpValues[0];
9004 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: VPIntrin.getType());
9005 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: VPIntrin.getType());
9006 N = DAG.getVPPtrExtOrTrunc(DL: getCurSDLoc(), VT: DestVT, Op: N, Mask: OpValues[1],
9007 EVL: OpValues[2]);
9008 N = DAG.getVPZExtOrTrunc(DL: getCurSDLoc(), VT: PtrMemVT, Op: N, Mask: OpValues[1],
9009 EVL: OpValues[2]);
9010 setValue(V: &VPIntrin, NewN: N);
9011 break;
9012 }
9013 case ISD::VP_PTRTOINT: {
9014 SDValue N = OpValues[0];
9015 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9016 Ty: VPIntrin.getType());
9017 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(),
9018 Ty: VPIntrin.getOperand(i_nocapture: 0)->getType());
9019 N = DAG.getVPPtrExtOrTrunc(DL: getCurSDLoc(), VT: PtrMemVT, Op: N, Mask: OpValues[1],
9020 EVL: OpValues[2]);
9021 N = DAG.getVPZExtOrTrunc(DL: getCurSDLoc(), VT: DestVT, Op: N, Mask: OpValues[1],
9022 EVL: OpValues[2]);
9023 setValue(V: &VPIntrin, NewN: N);
9024 break;
9025 }
9026 case ISD::VP_ABS:
9027 case ISD::VP_CTLZ:
9028 case ISD::VP_CTLZ_ZERO_UNDEF:
9029 case ISD::VP_CTTZ:
9030 case ISD::VP_CTTZ_ZERO_UNDEF:
9031 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
9032 case ISD::VP_CTTZ_ELTS: {
9033 SDValue Result =
9034 DAG.getNode(Opcode, DL, VTList: VTs, Ops: {OpValues[0], OpValues[2], OpValues[3]});
9035 setValue(V: &VPIntrin, NewN: Result);
9036 break;
9037 }
9038 }
9039}
9040
9041SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
9042 const BasicBlock *EHPadBB,
9043 MCSymbol *&BeginLabel) {
9044 MachineFunction &MF = DAG.getMachineFunction();
9045
9046 // Insert a label before the invoke call to mark the try range. This can be
9047 // used to detect deletion of the invoke via the MachineModuleInfo.
9048 BeginLabel = MF.getContext().createTempSymbol();
9049
9050 // For SjLj, keep track of which landing pads go with which invokes
9051 // so as to maintain the ordering of pads in the LSDA.
9052 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
9053 if (CallSiteIndex) {
9054 MF.setCallSiteBeginLabel(BeginLabel, Site: CallSiteIndex);
9055 LPadToCallSiteMap[FuncInfo.getMBB(BB: EHPadBB)].push_back(Elt: CallSiteIndex);
9056
9057 // Now that the call site is handled, stop tracking it.
9058 FuncInfo.setCurrentCallSite(0);
9059 }
9060
9061 return DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: BeginLabel);
9062}
9063
9064SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
9065 const BasicBlock *EHPadBB,
9066 MCSymbol *BeginLabel) {
9067 assert(BeginLabel && "BeginLabel should've been set");
9068
9069 MachineFunction &MF = DAG.getMachineFunction();
9070
9071 // Insert a label at the end of the invoke call to mark the try range. This
9072 // can be used to detect deletion of the invoke via the MachineModuleInfo.
9073 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
9074 Chain = DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: EndLabel);
9075
9076 // Inform MachineModuleInfo of range.
9077 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
9078 // There is a platform (e.g. wasm) that uses funclet style IR but does not
9079 // actually use outlined funclets and their LSDA info style.
9080 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
9081 assert(II && "II should've been set");
9082 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
9083 EHInfo->addIPToStateRange(II, InvokeBegin: BeginLabel, InvokeEnd: EndLabel);
9084 } else if (!isScopedEHPersonality(Pers)) {
9085 assert(EHPadBB);
9086 MF.addInvoke(LandingPad: FuncInfo.getMBB(BB: EHPadBB), BeginLabel, EndLabel);
9087 }
9088
9089 return Chain;
9090}
9091
9092std::pair<SDValue, SDValue>
9093SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
9094 const BasicBlock *EHPadBB) {
9095 MCSymbol *BeginLabel = nullptr;
9096
9097 if (EHPadBB) {
9098 // Both PendingLoads and PendingExports must be flushed here;
9099 // this call might not return.
9100 (void)getRoot();
9101 DAG.setRoot(lowerStartEH(Chain: getControlRoot(), EHPadBB, BeginLabel));
9102 CLI.setChain(getRoot());
9103 }
9104
9105 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9106 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
9107
9108 assert((CLI.IsTailCall || Result.second.getNode()) &&
9109 "Non-null chain expected with non-tail call!");
9110 assert((Result.second.getNode() || !Result.first.getNode()) &&
9111 "Null value expected with tail call!");
9112
9113 if (!Result.second.getNode()) {
9114 // As a special case, a null chain means that a tail call has been emitted
9115 // and the DAG root is already updated.
9116 HasTailCall = true;
9117
9118 // Since there's no actual continuation from this block, nothing can be
9119 // relying on us setting vregs for them.
9120 PendingExports.clear();
9121 } else {
9122 DAG.setRoot(Result.second);
9123 }
9124
9125 if (EHPadBB) {
9126 DAG.setRoot(lowerEndEH(Chain: getRoot(), II: cast_or_null<InvokeInst>(Val: CLI.CB), EHPadBB,
9127 BeginLabel));
9128 Result.second = getRoot();
9129 }
9130
9131 return Result;
9132}
9133
9134bool SelectionDAGBuilder::canTailCall(const CallBase &CB) const {
9135 bool isMustTailCall = CB.isMustTailCall();
9136
9137 // Avoid emitting tail calls in functions with the disable-tail-calls
9138 // attribute.
9139 const Function *Caller = CB.getParent()->getParent();
9140 if (!isMustTailCall &&
9141 Caller->getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
9142 return false;
9143
9144 // We can't tail call inside a function with a swifterror argument. Lowering
9145 // does not support this yet. It would have to move into the swifterror
9146 // register before the call.
9147 if (DAG.getTargetLoweringInfo().supportSwiftError() &&
9148 Caller->getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError))
9149 return false;
9150
9151 // Check if target-independent constraints permit a tail call here.
9152 // Target-dependent constraints are checked within TLI->LowerCallTo.
9153 return isInTailCallPosition(Call: CB, TM: DAG.getTarget());
9154}
9155
9156void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
9157 bool isTailCall, bool isMustTailCall,
9158 const BasicBlock *EHPadBB,
9159 const TargetLowering::PtrAuthInfo *PAI) {
9160 auto &DL = DAG.getDataLayout();
9161 FunctionType *FTy = CB.getFunctionType();
9162 Type *RetTy = CB.getType();
9163
9164 TargetLowering::ArgListTy Args;
9165 Args.reserve(n: CB.arg_size());
9166
9167 const Value *SwiftErrorVal = nullptr;
9168 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9169
9170 if (isTailCall)
9171 isTailCall = canTailCall(CB);
9172
9173 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
9174 const Value *V = *I;
9175
9176 // Skip empty types
9177 if (V->getType()->isEmptyTy())
9178 continue;
9179
9180 SDValue ArgNode = getValue(V);
9181 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
9182 Entry.setAttributes(Call: &CB, ArgIdx: I - CB.arg_begin());
9183
9184 // Use swifterror virtual register as input to the call.
9185 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
9186 SwiftErrorVal = V;
9187 // We find the virtual register for the actual swifterror argument.
9188 // Instead of using the Value, we use the virtual register instead.
9189 Entry.Node =
9190 DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
9191 VT: EVT(TLI.getPointerTy(DL)));
9192 }
9193
9194 Args.push_back(x: Entry);
9195
9196 // If we have an explicit sret argument that is an Instruction, (i.e., it
9197 // might point to function-local memory), we can't meaningfully tail-call.
9198 if (Entry.IsSRet && isa<Instruction>(Val: V))
9199 isTailCall = false;
9200 }
9201
9202 // If call site has a cfguardtarget operand bundle, create and add an
9203 // additional ArgListEntry.
9204 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_cfguardtarget)) {
9205 Value *V = Bundle->Inputs[0];
9206 TargetLowering::ArgListEntry Entry(V, getValue(V));
9207 Entry.IsCFGuardTarget = true;
9208 Args.push_back(x: Entry);
9209 }
9210
9211 // Disable tail calls if there is an swifterror argument. Targets have not
9212 // been updated to support tail calls.
9213 if (TLI.supportSwiftError() && SwiftErrorVal)
9214 isTailCall = false;
9215
9216 ConstantInt *CFIType = nullptr;
9217 if (CB.isIndirectCall()) {
9218 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi)) {
9219 if (!TLI.supportKCFIBundles())
9220 report_fatal_error(
9221 reason: "Target doesn't support calls with kcfi operand bundles.");
9222 CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
9223 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
9224 }
9225 }
9226
9227 SDValue ConvControlToken;
9228 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
9229 auto *Token = Bundle->Inputs[0].get();
9230 ConvControlToken = getValue(V: Token);
9231 }
9232
9233 GlobalValue *DeactivationSymbol = nullptr;
9234 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
9235 DeactivationSymbol = cast<GlobalValue>(Val: Bundle->Inputs[0].get());
9236 }
9237
9238 TargetLowering::CallLoweringInfo CLI(DAG);
9239 CLI.setDebugLoc(getCurSDLoc())
9240 .setChain(getRoot())
9241 .setCallee(ResultType: RetTy, FTy, Target: Callee, ArgsList: std::move(Args), Call: CB)
9242 .setTailCall(isTailCall)
9243 .setConvergent(CB.isConvergent())
9244 .setIsPreallocated(
9245 CB.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
9246 .setCFIType(CFIType)
9247 .setConvergenceControlToken(ConvControlToken)
9248 .setDeactivationSymbol(DeactivationSymbol);
9249
9250 // Set the pointer authentication info if we have it.
9251 if (PAI) {
9252 if (!TLI.supportPtrAuthBundles())
9253 report_fatal_error(
9254 reason: "This target doesn't support calls with ptrauth operand bundles.");
9255 CLI.setPtrAuth(*PAI);
9256 }
9257
9258 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9259
9260 if (Result.first.getNode()) {
9261 Result.first = lowerRangeToAssertZExt(DAG, I: CB, Op: Result.first);
9262 Result.first = lowerNoFPClassToAssertNoFPClass(DAG, I: CB, Op: Result.first);
9263 setValue(V: &CB, NewN: Result.first);
9264 }
9265
9266 // The last element of CLI.InVals has the SDValue for swifterror return.
9267 // Here we copy it to a virtual register and update SwiftErrorMap for
9268 // book-keeping.
9269 if (SwiftErrorVal && TLI.supportSwiftError()) {
9270 // Get the last element of InVals.
9271 SDValue Src = CLI.InVals.back();
9272 Register VReg =
9273 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
9274 SDValue CopyNode = CLI.DAG.getCopyToReg(Chain: Result.second, dl: CLI.DL, Reg: VReg, N: Src);
9275 DAG.setRoot(CopyNode);
9276 }
9277}
9278
9279static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
9280 SelectionDAGBuilder &Builder) {
9281 // Check to see if this load can be trivially constant folded, e.g. if the
9282 // input is from a string literal.
9283 if (const Constant *LoadInput = dyn_cast<Constant>(Val: PtrVal)) {
9284 // Cast pointer to the type we really want to load.
9285 Type *LoadTy =
9286 Type::getIntNTy(C&: PtrVal->getContext(), N: LoadVT.getScalarSizeInBits());
9287 if (LoadVT.isVector())
9288 LoadTy = FixedVectorType::get(ElementType: LoadTy, NumElts: LoadVT.getVectorNumElements());
9289 if (const Constant *LoadCst =
9290 ConstantFoldLoadFromConstPtr(C: const_cast<Constant *>(LoadInput),
9291 Ty: LoadTy, DL: Builder.DAG.getDataLayout()))
9292 return Builder.getValue(V: LoadCst);
9293 }
9294
9295 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9296 // still constant memory, the input chain can be the entry node.
9297 SDValue Root;
9298 bool ConstantMemory = false;
9299
9300 // Do not serialize (non-volatile) loads of constant memory with anything.
9301 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(P: PtrVal)) {
9302 Root = Builder.DAG.getEntryNode();
9303 ConstantMemory = true;
9304 } else {
9305 // Do not serialize non-volatile loads against each other.
9306 Root = Builder.DAG.getRoot();
9307 }
9308
9309 SDValue Ptr = Builder.getValue(V: PtrVal);
9310 SDValue LoadVal =
9311 Builder.DAG.getLoad(VT: LoadVT, dl: Builder.getCurSDLoc(), Chain: Root, Ptr,
9312 PtrInfo: MachinePointerInfo(PtrVal), Alignment: Align(1));
9313
9314 if (!ConstantMemory)
9315 Builder.PendingLoads.push_back(Elt: LoadVal.getValue(R: 1));
9316 return LoadVal;
9317}
9318
9319/// Record the value for an instruction that produces an integer result,
9320/// converting the type where necessary.
9321void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9322 SDValue Value,
9323 bool IsSigned) {
9324 EVT VT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9325 Ty: I.getType(), AllowUnknown: true);
9326 Value = DAG.getExtOrTrunc(IsSigned, Op: Value, DL: getCurSDLoc(), VT);
9327 setValue(V: &I, NewN: Value);
9328}
9329
9330/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9331/// true and lower it. Otherwise return false, and it will be lowered like a
9332/// normal call.
9333/// The caller already checked that \p I calls the appropriate LibFunc with a
9334/// correct prototype.
9335bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9336 const Value *LHS = I.getArgOperand(i: 0), *RHS = I.getArgOperand(i: 1);
9337 const Value *Size = I.getArgOperand(i: 2);
9338 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(Val: getValue(V: Size));
9339 if (CSize && CSize->getZExtValue() == 0) {
9340 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9341 Ty: I.getType(), AllowUnknown: true);
9342 setValue(V: &I, NewN: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: CallVT));
9343 return true;
9344 }
9345
9346 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9347 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9348 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: LHS), Op2: getValue(V: RHS),
9349 Op3: getValue(V: Size), CI: &I);
9350 if (Res.first.getNode()) {
9351 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9352 PendingLoads.push_back(Elt: Res.second);
9353 return true;
9354 }
9355
9356 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9357 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9358 if (!CSize || !isOnlyUsedInZeroEqualityComparison(CxtI: &I))
9359 return false;
9360
9361 // If the target has a fast compare for the given size, it will return a
9362 // preferred load type for that size. Require that the load VT is legal and
9363 // that the target supports unaligned loads of that type. Otherwise, return
9364 // INVALID.
9365 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9366 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9367 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9368 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9369 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9370 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9371 // TODO: Check alignment of src and dest ptrs.
9372 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9373 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9374 if (!TLI.isTypeLegal(VT: LVT) ||
9375 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: SrcAS) ||
9376 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: DstAS))
9377 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
9378 }
9379
9380 return LVT;
9381 };
9382
9383 // This turns into unaligned loads. We only do this if the target natively
9384 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9385 // we'll only produce a small number of byte loads.
9386 MVT LoadVT;
9387 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9388 switch (NumBitsToCompare) {
9389 default:
9390 return false;
9391 case 16:
9392 LoadVT = MVT::i16;
9393 break;
9394 case 32:
9395 LoadVT = MVT::i32;
9396 break;
9397 case 64:
9398 case 128:
9399 case 256:
9400 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9401 break;
9402 }
9403
9404 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9405 return false;
9406
9407 SDValue LoadL = getMemCmpLoad(PtrVal: LHS, LoadVT, Builder&: *this);
9408 SDValue LoadR = getMemCmpLoad(PtrVal: RHS, LoadVT, Builder&: *this);
9409
9410 // Bitcast to a wide integer type if the loads are vectors.
9411 if (LoadVT.isVector()) {
9412 EVT CmpVT = EVT::getIntegerVT(Context&: LHS->getContext(), BitWidth: LoadVT.getSizeInBits());
9413 LoadL = DAG.getBitcast(VT: CmpVT, V: LoadL);
9414 LoadR = DAG.getBitcast(VT: CmpVT, V: LoadR);
9415 }
9416
9417 SDValue Cmp = DAG.getSetCC(DL: getCurSDLoc(), VT: MVT::i1, LHS: LoadL, RHS: LoadR, Cond: ISD::SETNE);
9418 processIntegerCallValue(I, Value: Cmp, IsSigned: false);
9419 return true;
9420}
9421
9422/// See if we can lower a memchr call into an optimized form. If so, return
9423/// true and lower it. Otherwise return false, and it will be lowered like a
9424/// normal call.
9425/// The caller already checked that \p I calls the appropriate LibFunc with a
9426/// correct prototype.
9427bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9428 const Value *Src = I.getArgOperand(i: 0);
9429 const Value *Char = I.getArgOperand(i: 1);
9430 const Value *Length = I.getArgOperand(i: 2);
9431
9432 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9433 std::pair<SDValue, SDValue> Res =
9434 TSI.EmitTargetCodeForMemchr(DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(),
9435 Src: getValue(V: Src), Char: getValue(V: Char), Length: getValue(V: Length),
9436 SrcPtrInfo: MachinePointerInfo(Src));
9437 if (Res.first.getNode()) {
9438 setValue(V: &I, NewN: Res.first);
9439 PendingLoads.push_back(Elt: Res.second);
9440 return true;
9441 }
9442
9443 return false;
9444}
9445
9446/// See if we can lower a memccpy call into an optimized form. If so, return
9447/// true and lower it, otherwise return false and it will be lowered like a
9448/// normal call.
9449/// The caller already checked that \p I calls the appropriate LibFunc with a
9450/// correct prototype.
9451bool SelectionDAGBuilder::visitMemCCpyCall(const CallInst &I) {
9452 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9453 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemccpy(
9454 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Dst: getValue(V: I.getArgOperand(i: 0)),
9455 Src: getValue(V: I.getArgOperand(i: 1)), C: getValue(V: I.getArgOperand(i: 2)),
9456 Size: getValue(V: I.getArgOperand(i: 3)), CI: &I);
9457
9458 if (Res.first) {
9459 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9460 PendingLoads.push_back(Elt: Res.second);
9461 return true;
9462 }
9463 return false;
9464}
9465
9466/// See if we can lower a mempcpy call into an optimized form. If so, return
9467/// true and lower it. Otherwise return false, and it will be lowered like a
9468/// normal call.
9469/// The caller already checked that \p I calls the appropriate LibFunc with a
9470/// correct prototype.
9471bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9472 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
9473 SDValue Src = getValue(V: I.getArgOperand(i: 1));
9474 SDValue Size = getValue(V: I.getArgOperand(i: 2));
9475
9476 Align DstAlign = DAG.InferPtrAlign(Ptr: Dst).valueOrOne();
9477 Align SrcAlign = DAG.InferPtrAlign(Ptr: Src).valueOrOne();
9478 // DAG::getMemcpy needs Alignment to be defined.
9479 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
9480
9481 SDLoc sdl = getCurSDLoc();
9482
9483 // In the mempcpy context we need to pass in a false value for isTailCall
9484 // because the return pointer needs to be adjusted by the size of
9485 // the copied memory.
9486 SDValue Root = getMemoryRoot();
9487 SDValue MC = DAG.getMemcpy(
9488 Chain: Root, dl: sdl, Dst, Src, Size, Alignment, isVol: false, AlwaysInline: false, /*CI=*/nullptr,
9489 OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
9490 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)), AAInfo: I.getAAMetadata());
9491 assert(MC.getNode() != nullptr &&
9492 "** memcpy should not be lowered as TailCall in mempcpy context **");
9493 DAG.setRoot(MC);
9494
9495 // Check if Size needs to be truncated or extended.
9496 Size = DAG.getSExtOrTrunc(Op: Size, DL: sdl, VT: Dst.getValueType());
9497
9498 // Adjust return pointer to point just past the last dst byte.
9499 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Base: Dst, Offset: Size, DL: sdl);
9500 setValue(V: &I, NewN: DstPlusSize);
9501 return true;
9502}
9503
9504/// See if we can lower a strcpy call into an optimized form. If so, return
9505/// true and lower it, otherwise return false and it will be lowered like a
9506/// normal call.
9507/// The caller already checked that \p I calls the appropriate LibFunc with a
9508/// correct prototype.
9509bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9510 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9511
9512 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9513 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcpy(
9514 DAG, DL: getCurSDLoc(), Chain: getRoot(), Dest: getValue(V: Arg0), Src: getValue(V: Arg1),
9515 DestPtrInfo: MachinePointerInfo(Arg0), SrcPtrInfo: MachinePointerInfo(Arg1), isStpcpy, CI: &I);
9516 if (Res.first.getNode()) {
9517 setValue(V: &I, NewN: Res.first);
9518 DAG.setRoot(Res.second);
9519 return true;
9520 }
9521
9522 return false;
9523}
9524
9525/// See if we can lower a strcmp call into an optimized form. If so, return
9526/// true and lower it, otherwise return false and it will be lowered like a
9527/// normal call.
9528/// The caller already checked that \p I calls the appropriate LibFunc with a
9529/// correct prototype.
9530bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9531 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9532
9533 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9534 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcmp(
9535 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: Arg0), Op2: getValue(V: Arg1),
9536 Op1PtrInfo: MachinePointerInfo(Arg0), Op2PtrInfo: MachinePointerInfo(Arg1), CI: &I);
9537 if (Res.first.getNode()) {
9538 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9539 PendingLoads.push_back(Elt: Res.second);
9540 return true;
9541 }
9542
9543 return false;
9544}
9545
9546/// See if we can lower a strlen call into an optimized form. If so, return
9547/// true and lower it, otherwise return false and it will be lowered like a
9548/// normal call.
9549/// The caller already checked that \p I calls the appropriate LibFunc with a
9550/// correct prototype.
9551bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9552 const Value *Arg0 = I.getArgOperand(i: 0);
9553
9554 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9555 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(
9556 DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(), Src: getValue(V: Arg0), CI: &I);
9557 if (Res.first.getNode()) {
9558 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9559 PendingLoads.push_back(Elt: Res.second);
9560 return true;
9561 }
9562
9563 return false;
9564}
9565
9566/// See if we can lower a strnlen call into an optimized form. If so, return
9567/// true and lower it, otherwise return false and it will be lowered like a
9568/// normal call.
9569/// The caller already checked that \p I calls the appropriate LibFunc with a
9570/// correct prototype.
9571bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9572 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9573
9574 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9575 std::pair<SDValue, SDValue> Res =
9576 TSI.EmitTargetCodeForStrnlen(DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(),
9577 Src: getValue(V: Arg0), MaxLength: getValue(V: Arg1),
9578 SrcPtrInfo: MachinePointerInfo(Arg0));
9579 if (Res.first.getNode()) {
9580 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9581 PendingLoads.push_back(Elt: Res.second);
9582 return true;
9583 }
9584
9585 return false;
9586}
9587
9588/// See if we can lower a Strstr call into an optimized form. If so, return
9589/// true and lower it, otherwise return false and it will be lowered like a
9590/// normal call.
9591/// The caller already checked that \p I calls the appropriate LibFunc with a
9592/// correct prototype.
9593bool SelectionDAGBuilder::visitStrstrCall(const CallInst &I) {
9594 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9595 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9596 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrstr(
9597 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: Arg0), Op2: getValue(V: Arg1), CI: &I);
9598 if (Res.first) {
9599 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9600 PendingLoads.push_back(Elt: Res.second);
9601 return true;
9602 }
9603 return false;
9604}
9605
9606/// See if we can lower a unary floating-point operation into an SDNode with
9607/// the specified Opcode. If so, return true and lower it, otherwise return
9608/// false and it will be lowered like a normal call.
9609/// The caller already checked that \p I calls the appropriate LibFunc with a
9610/// correct prototype.
9611bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9612 unsigned Opcode) {
9613 // We already checked this call's prototype; verify it doesn't modify errno.
9614 // Do not perform optimizations for call sites that require strict
9615 // floating-point semantics.
9616 if (!I.onlyReadsMemory() || I.isStrictFP())
9617 return false;
9618
9619 SDNodeFlags Flags;
9620 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9621
9622 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
9623 setValue(V: &I,
9624 NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Tmp.getValueType(), Operand: Tmp, Flags));
9625 return true;
9626}
9627
9628/// See if we can lower a binary floating-point operation into an SDNode with
9629/// the specified Opcode. If so, return true and lower it. Otherwise return
9630/// false, and it will be lowered like a normal call.
9631/// The caller already checked that \p I calls the appropriate LibFunc with a
9632/// correct prototype.
9633bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9634 unsigned Opcode) {
9635 // We already checked this call's prototype; verify it doesn't modify errno.
9636 // Do not perform optimizations for call sites that require strict
9637 // floating-point semantics.
9638 if (!I.onlyReadsMemory() || I.isStrictFP())
9639 return false;
9640
9641 SDNodeFlags Flags;
9642 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9643
9644 SDValue Tmp0 = getValue(V: I.getArgOperand(i: 0));
9645 SDValue Tmp1 = getValue(V: I.getArgOperand(i: 1));
9646 EVT VT = Tmp0.getValueType();
9647 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT, N1: Tmp0, N2: Tmp1, Flags));
9648 return true;
9649}
9650
9651void SelectionDAGBuilder::visitCall(const CallInst &I) {
9652 // Handle inline assembly differently.
9653 if (I.isInlineAsm()) {
9654 visitInlineAsm(Call: I);
9655 return;
9656 }
9657
9658 diagnoseDontCall(CI: I);
9659
9660 if (Function *F = I.getCalledFunction()) {
9661 if (F->isDeclaration()) {
9662 // Is this an LLVM intrinsic?
9663 if (unsigned IID = F->getIntrinsicID()) {
9664 visitIntrinsicCall(I, Intrinsic: IID);
9665 return;
9666 }
9667 }
9668
9669 // Check for well-known libc/libm calls. If the function is internal, it
9670 // can't be a library call. Don't do the check if marked as nobuiltin for
9671 // some reason.
9672 // This code should not handle libcalls that are already canonicalized to
9673 // intrinsics by the middle-end.
9674 LibFunc Func;
9675 if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&
9676 LibInfo->getLibFunc(FDecl: *F, F&: Func) && LibInfo->hasOptimizedCodeGen(F: Func)) {
9677 switch (Func) {
9678 default: break;
9679 case LibFunc_bcmp:
9680 if (visitMemCmpBCmpCall(I))
9681 return;
9682 break;
9683 case LibFunc_copysign:
9684 case LibFunc_copysignf:
9685 case LibFunc_copysignl:
9686 // We already checked this call's prototype; verify it doesn't modify
9687 // errno.
9688 if (I.onlyReadsMemory()) {
9689 SDValue LHS = getValue(V: I.getArgOperand(i: 0));
9690 SDValue RHS = getValue(V: I.getArgOperand(i: 1));
9691 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: getCurSDLoc(),
9692 VT: LHS.getValueType(), N1: LHS, N2: RHS));
9693 return;
9694 }
9695 break;
9696 case LibFunc_sin:
9697 case LibFunc_sinf:
9698 case LibFunc_sinl:
9699 if (visitUnaryFloatCall(I, Opcode: ISD::FSIN))
9700 return;
9701 break;
9702 case LibFunc_cos:
9703 case LibFunc_cosf:
9704 case LibFunc_cosl:
9705 if (visitUnaryFloatCall(I, Opcode: ISD::FCOS))
9706 return;
9707 break;
9708 case LibFunc_tan:
9709 case LibFunc_tanf:
9710 case LibFunc_tanl:
9711 if (visitUnaryFloatCall(I, Opcode: ISD::FTAN))
9712 return;
9713 break;
9714 case LibFunc_asin:
9715 case LibFunc_asinf:
9716 case LibFunc_asinl:
9717 if (visitUnaryFloatCall(I, Opcode: ISD::FASIN))
9718 return;
9719 break;
9720 case LibFunc_acos:
9721 case LibFunc_acosf:
9722 case LibFunc_acosl:
9723 if (visitUnaryFloatCall(I, Opcode: ISD::FACOS))
9724 return;
9725 break;
9726 case LibFunc_atan:
9727 case LibFunc_atanf:
9728 case LibFunc_atanl:
9729 if (visitUnaryFloatCall(I, Opcode: ISD::FATAN))
9730 return;
9731 break;
9732 case LibFunc_atan2:
9733 case LibFunc_atan2f:
9734 case LibFunc_atan2l:
9735 if (visitBinaryFloatCall(I, Opcode: ISD::FATAN2))
9736 return;
9737 break;
9738 case LibFunc_sinh:
9739 case LibFunc_sinhf:
9740 case LibFunc_sinhl:
9741 if (visitUnaryFloatCall(I, Opcode: ISD::FSINH))
9742 return;
9743 break;
9744 case LibFunc_cosh:
9745 case LibFunc_coshf:
9746 case LibFunc_coshl:
9747 if (visitUnaryFloatCall(I, Opcode: ISD::FCOSH))
9748 return;
9749 break;
9750 case LibFunc_tanh:
9751 case LibFunc_tanhf:
9752 case LibFunc_tanhl:
9753 if (visitUnaryFloatCall(I, Opcode: ISD::FTANH))
9754 return;
9755 break;
9756 case LibFunc_sqrt:
9757 case LibFunc_sqrtf:
9758 case LibFunc_sqrtl:
9759 case LibFunc_sqrt_finite:
9760 case LibFunc_sqrtf_finite:
9761 case LibFunc_sqrtl_finite:
9762 if (visitUnaryFloatCall(I, Opcode: ISD::FSQRT))
9763 return;
9764 break;
9765 case LibFunc_log2:
9766 case LibFunc_log2f:
9767 case LibFunc_log2l:
9768 if (visitUnaryFloatCall(I, Opcode: ISD::FLOG2))
9769 return;
9770 break;
9771 case LibFunc_exp2:
9772 case LibFunc_exp2f:
9773 case LibFunc_exp2l:
9774 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP2))
9775 return;
9776 break;
9777 case LibFunc_exp10:
9778 case LibFunc_exp10f:
9779 case LibFunc_exp10l:
9780 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP10))
9781 return;
9782 break;
9783 case LibFunc_ldexp:
9784 case LibFunc_ldexpf:
9785 case LibFunc_ldexpl:
9786 if (visitBinaryFloatCall(I, Opcode: ISD::FLDEXP))
9787 return;
9788 break;
9789 case LibFunc_strstr:
9790 if (visitStrstrCall(I))
9791 return;
9792 break;
9793 case LibFunc_memcmp:
9794 if (visitMemCmpBCmpCall(I))
9795 return;
9796 break;
9797 case LibFunc_memccpy:
9798 if (visitMemCCpyCall(I))
9799 return;
9800 break;
9801 case LibFunc_mempcpy:
9802 if (visitMemPCpyCall(I))
9803 return;
9804 break;
9805 case LibFunc_memchr:
9806 if (visitMemChrCall(I))
9807 return;
9808 break;
9809 case LibFunc_strcpy:
9810 if (visitStrCpyCall(I, isStpcpy: false))
9811 return;
9812 break;
9813 case LibFunc_stpcpy:
9814 if (visitStrCpyCall(I, isStpcpy: true))
9815 return;
9816 break;
9817 case LibFunc_strcmp:
9818 if (visitStrCmpCall(I))
9819 return;
9820 break;
9821 case LibFunc_strlen:
9822 if (visitStrLenCall(I))
9823 return;
9824 break;
9825 case LibFunc_strnlen:
9826 if (visitStrNLenCall(I))
9827 return;
9828 break;
9829 }
9830 }
9831 }
9832
9833 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
9834 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), /*EHPadBB=*/nullptr);
9835 return;
9836 }
9837
9838 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9839 // have to do anything here to lower funclet bundles.
9840 // CFGuardTarget bundles are lowered in LowerCallTo.
9841 failForInvalidBundles(
9842 I, Name: "calls",
9843 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9844 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9845 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9846 LLVMContext::OB_convergencectrl, LLVMContext::OB_deactivation_symbol});
9847
9848 SDValue Callee = getValue(V: I.getCalledOperand());
9849
9850 if (I.hasDeoptState())
9851 LowerCallSiteWithDeoptBundle(Call: &I, Callee, EHPadBB: nullptr);
9852 else
9853 // Check if we can potentially perform a tail call. More detailed checking
9854 // is be done within LowerCallTo, after more information about the call is
9855 // known.
9856 LowerCallTo(CB: I, Callee, isTailCall: I.isTailCall(), isMustTailCall: I.isMustTailCall());
9857}
9858
9859void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(
9860 const CallBase &CB, const BasicBlock *EHPadBB) {
9861 auto PAB = CB.getOperandBundle(Name: "ptrauth");
9862 const Value *CalleeV = CB.getCalledOperand();
9863
9864 // Gather the call ptrauth data from the operand bundle:
9865 // [ i32 <key>, i64 <discriminator> ]
9866 const auto *Key = cast<ConstantInt>(Val: PAB->Inputs[0]);
9867 const Value *Discriminator = PAB->Inputs[1];
9868
9869 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
9870 assert(Discriminator->getType()->isIntegerTy(64) &&
9871 "Invalid ptrauth discriminator");
9872
9873 // Look through ptrauth constants to find the raw callee.
9874 // Do a direct unauthenticated call if we found it and everything matches.
9875 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CalleeV))
9876 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
9877 DL: DAG.getDataLayout()))
9878 return LowerCallTo(CB, Callee: getValue(V: CalleeCPA->getPointer()), isTailCall: CB.isTailCall(),
9879 isMustTailCall: CB.isMustTailCall(), EHPadBB);
9880
9881 // Functions should never be ptrauth-called directly.
9882 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
9883
9884 // Otherwise, do an authenticated indirect call.
9885 TargetLowering::PtrAuthInfo PAI = {.Key: Key->getZExtValue(),
9886 .Discriminator: getValue(V: Discriminator)};
9887
9888 LowerCallTo(CB, Callee: getValue(V: CalleeV), isTailCall: CB.isTailCall(), isMustTailCall: CB.isMustTailCall(),
9889 EHPadBB, PAI: &PAI);
9890}
9891
9892namespace {
9893
9894/// AsmOperandInfo - This contains information for each constraint that we are
9895/// lowering.
9896class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9897public:
9898 /// CallOperand - If this is the result output operand or a clobber
9899 /// this is null, otherwise it is the incoming operand to the CallInst.
9900 /// This gets modified as the asm is processed.
9901 SDValue CallOperand;
9902
9903 /// AssignedRegs - If this is a register or register class operand, this
9904 /// contains the set of register corresponding to the operand.
9905 RegsForValue AssignedRegs;
9906
9907 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9908 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9909 }
9910
9911 /// Whether or not this operand accesses memory
9912 bool hasMemory(const TargetLowering &TLI) const {
9913 // Indirect operand accesses access memory.
9914 if (isIndirect)
9915 return true;
9916
9917 for (const auto &Code : Codes)
9918 if (TLI.getConstraintType(Constraint: Code) == TargetLowering::C_Memory)
9919 return true;
9920
9921 return false;
9922 }
9923};
9924
9925
9926} // end anonymous namespace
9927
9928/// Make sure that the output operand \p OpInfo and its corresponding input
9929/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9930/// out).
9931static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9932 SDISelAsmOperandInfo &MatchingOpInfo,
9933 SelectionDAG &DAG) {
9934 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9935 return;
9936
9937 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
9938 const auto &TLI = DAG.getTargetLoweringInfo();
9939
9940 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9941 TLI.getRegForInlineAsmConstraint(TRI, Constraint: OpInfo.ConstraintCode,
9942 VT: OpInfo.ConstraintVT);
9943 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9944 TLI.getRegForInlineAsmConstraint(TRI, Constraint: MatchingOpInfo.ConstraintCode,
9945 VT: MatchingOpInfo.ConstraintVT);
9946 const bool OutOpIsIntOrFP =
9947 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
9948 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
9949 MatchingOpInfo.ConstraintVT.isFloatingPoint();
9950 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
9951 // FIXME: error out in a more elegant fashion
9952 report_fatal_error(reason: "Unsupported asm: input constraint"
9953 " with a matching output constraint of"
9954 " incompatible type!");
9955 }
9956 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9957}
9958
9959/// Get a direct memory input to behave well as an indirect operand.
9960/// This may introduce stores, hence the need for a \p Chain.
9961/// \return The (possibly updated) chain.
9962static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9963 SDISelAsmOperandInfo &OpInfo,
9964 SelectionDAG &DAG) {
9965 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9966
9967 // If we don't have an indirect input, put it in the constpool if we can,
9968 // otherwise spill it to a stack slot.
9969 // TODO: This isn't quite right. We need to handle these according to
9970 // the addressing mode that the constraint wants. Also, this may take
9971 // an additional register for the computation and we don't want that
9972 // either.
9973
9974 // If the operand is a float, integer, or vector constant, spill to a
9975 // constant pool entry to get its address.
9976 const Value *OpVal = OpInfo.CallOperandVal;
9977 if (isa<ConstantFP>(Val: OpVal) || isa<ConstantInt>(Val: OpVal) ||
9978 isa<ConstantVector>(Val: OpVal) || isa<ConstantDataVector>(Val: OpVal)) {
9979 OpInfo.CallOperand = DAG.getConstantPool(
9980 C: cast<Constant>(Val: OpVal), VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9981 return Chain;
9982 }
9983
9984 // Otherwise, create a stack slot and emit a store to it before the asm.
9985 Type *Ty = OpVal->getType();
9986 auto &DL = DAG.getDataLayout();
9987 TypeSize TySize = DL.getTypeAllocSize(Ty);
9988 MachineFunction &MF = DAG.getMachineFunction();
9989 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
9990 int StackID = 0;
9991 if (TySize.isScalable())
9992 StackID = TFI->getStackIDForScalableVectors();
9993 int SSFI = MF.getFrameInfo().CreateStackObject(Size: TySize.getKnownMinValue(),
9994 Alignment: DL.getPrefTypeAlign(Ty), isSpillSlot: false,
9995 Alloca: nullptr, ID: StackID);
9996 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: TLI.getFrameIndexTy(DL));
9997 Chain = DAG.getTruncStore(Chain, dl: Location, Val: OpInfo.CallOperand, Ptr: StackSlot,
9998 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: SSFI),
9999 SVT: TLI.getMemValueType(DL, Ty));
10000 OpInfo.CallOperand = StackSlot;
10001
10002 return Chain;
10003}
10004
10005/// GetRegistersForValue - Assign registers (virtual or physical) for the
10006/// specified operand. We prefer to assign virtual registers, to allow the
10007/// register allocator to handle the assignment process. However, if the asm
10008/// uses features that we can't model on machineinstrs, we have SDISel do the
10009/// allocation. This produces generally horrible, but correct, code.
10010///
10011/// OpInfo describes the operand
10012/// RefOpInfo describes the matching operand if any, the operand otherwise
10013static std::optional<unsigned>
10014getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
10015 SDISelAsmOperandInfo &OpInfo,
10016 SDISelAsmOperandInfo &RefOpInfo) {
10017 LLVMContext &Context = *DAG.getContext();
10018 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10019
10020 MachineFunction &MF = DAG.getMachineFunction();
10021 SmallVector<Register, 4> Regs;
10022 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10023
10024 // No work to do for memory/address operands.
10025 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10026 OpInfo.ConstraintType == TargetLowering::C_Address)
10027 return std::nullopt;
10028
10029 // If this is a constraint for a single physreg, or a constraint for a
10030 // register class, find it.
10031 unsigned AssignedReg;
10032 const TargetRegisterClass *RC;
10033 std::tie(args&: AssignedReg, args&: RC) = TLI.getRegForInlineAsmConstraint(
10034 TRI: &TRI, Constraint: RefOpInfo.ConstraintCode, VT: RefOpInfo.ConstraintVT);
10035 // RC is unset only on failure. Return immediately.
10036 if (!RC)
10037 return std::nullopt;
10038
10039 // Get the actual register value type. This is important, because the user
10040 // may have asked for (e.g.) the AX register in i32 type. We need to
10041 // remember that AX is actually i16 to get the right extension.
10042 const MVT RegVT = *TRI.legalclasstypes_begin(RC: *RC);
10043
10044 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10045 // If this is an FP operand in an integer register (or visa versa), or more
10046 // generally if the operand value disagrees with the register class we plan
10047 // to stick it in, fix the operand type.
10048 //
10049 // If this is an input value, the bitcast to the new type is done now.
10050 // Bitcast for output value is done at the end of visitInlineAsm().
10051 if ((OpInfo.Type == InlineAsm::isOutput ||
10052 OpInfo.Type == InlineAsm::isInput) &&
10053 !TRI.isTypeLegalForClass(RC: *RC, T: OpInfo.ConstraintVT)) {
10054 // Try to convert to the first EVT that the reg class contains. If the
10055 // types are identical size, use a bitcast to convert (e.g. two differing
10056 // vector types). Note: output bitcast is done at the end of
10057 // visitInlineAsm().
10058 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10059 // Exclude indirect inputs while they are unsupported because the code
10060 // to perform the load is missing and thus OpInfo.CallOperand still
10061 // refers to the input address rather than the pointed-to value.
10062 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
10063 OpInfo.CallOperand =
10064 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: RegVT, Operand: OpInfo.CallOperand);
10065 OpInfo.ConstraintVT = RegVT;
10066 // If the operand is an FP value and we want it in integer registers,
10067 // use the corresponding integer type. This turns an f64 value into
10068 // i64, which can be passed with two i32 values on a 32-bit machine.
10069 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10070 MVT VT = MVT::getIntegerVT(BitWidth: OpInfo.ConstraintVT.getSizeInBits());
10071 if (OpInfo.Type == InlineAsm::isInput)
10072 OpInfo.CallOperand =
10073 DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: OpInfo.CallOperand);
10074 OpInfo.ConstraintVT = VT;
10075 }
10076 }
10077 }
10078
10079 // No need to allocate a matching input constraint since the constraint it's
10080 // matching to has already been allocated.
10081 if (OpInfo.isMatchingInputConstraint())
10082 return std::nullopt;
10083
10084 EVT ValueVT = OpInfo.ConstraintVT;
10085 if (OpInfo.ConstraintVT == MVT::Other)
10086 ValueVT = RegVT;
10087
10088 // Initialize NumRegs.
10089 unsigned NumRegs = 1;
10090 if (OpInfo.ConstraintVT != MVT::Other)
10091 NumRegs = TLI.getNumRegisters(Context, VT: OpInfo.ConstraintVT, RegisterVT: RegVT);
10092
10093 // If this is a constraint for a specific physical register, like {r17},
10094 // assign it now.
10095
10096 // If this associated to a specific register, initialize iterator to correct
10097 // place. If virtual, make sure we have enough registers
10098
10099 // Initialize iterator if necessary
10100 TargetRegisterClass::iterator I = RC->begin();
10101 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10102
10103 // Do not check for single registers.
10104 if (AssignedReg) {
10105 I = std::find(first: I, last: RC->end(), val: AssignedReg);
10106 if (I == RC->end()) {
10107 // RC does not contain the selected register, which indicates a
10108 // mismatch between the register and the required type/bitwidth.
10109 return {AssignedReg};
10110 }
10111 }
10112
10113 for (; NumRegs; --NumRegs, ++I) {
10114 assert(I != RC->end() && "Ran out of registers to allocate!");
10115 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RegClass: RC);
10116 Regs.push_back(Elt: R);
10117 }
10118
10119 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
10120 return std::nullopt;
10121}
10122
10123static unsigned
10124findMatchingInlineAsmOperand(unsigned OperandNo,
10125 const std::vector<SDValue> &AsmNodeOperands) {
10126 // Scan until we find the definition we already emitted of this operand.
10127 unsigned CurOp = InlineAsm::Op_FirstOperand;
10128 for (; OperandNo; --OperandNo) {
10129 // Advance to the next operand.
10130 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10131 const InlineAsm::Flag F(OpFlag);
10132 assert(
10133 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
10134 "Skipped past definitions?");
10135 CurOp += F.getNumOperandRegisters() + 1;
10136 }
10137 return CurOp;
10138}
10139
10140namespace {
10141
10142class ExtraFlags {
10143 unsigned Flags = 0;
10144
10145public:
10146 explicit ExtraFlags(const CallBase &Call) {
10147 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10148 if (IA->hasSideEffects())
10149 Flags |= InlineAsm::Extra_HasSideEffects;
10150 if (IA->isAlignStack())
10151 Flags |= InlineAsm::Extra_IsAlignStack;
10152 if (IA->canThrow())
10153 Flags |= InlineAsm::Extra_MayUnwind;
10154 if (Call.isConvergent())
10155 Flags |= InlineAsm::Extra_IsConvergent;
10156 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
10157 }
10158
10159 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
10160 // Ideally, we would only check against memory constraints. However, the
10161 // meaning of an Other constraint can be target-specific and we can't easily
10162 // reason about it. Therefore, be conservative and set MayLoad/MayStore
10163 // for Other constraints as well.
10164 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10165 OpInfo.ConstraintType == TargetLowering::C_Other) {
10166 if (OpInfo.Type == InlineAsm::isInput)
10167 Flags |= InlineAsm::Extra_MayLoad;
10168 else if (OpInfo.Type == InlineAsm::isOutput)
10169 Flags |= InlineAsm::Extra_MayStore;
10170 else if (OpInfo.Type == InlineAsm::isClobber)
10171 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
10172 }
10173 }
10174
10175 unsigned get() const { return Flags; }
10176};
10177
10178} // end anonymous namespace
10179
10180static bool isFunction(SDValue Op) {
10181 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
10182 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
10183 auto Fn = dyn_cast_or_null<Function>(Val: GA->getGlobal());
10184
10185 // In normal "call dllimport func" instruction (non-inlineasm) it force
10186 // indirect access by specifing call opcode. And usually specially print
10187 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
10188 // not do in this way now. (In fact, this is similar with "Data Access"
10189 // action). So here we ignore dllimport function.
10190 if (Fn && !Fn->hasDLLImportStorageClass())
10191 return true;
10192 }
10193 }
10194 return false;
10195}
10196
10197/// visitInlineAsm - Handle a call to an InlineAsm object.
10198void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
10199 const BasicBlock *EHPadBB) {
10200 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10201
10202 /// ConstraintOperands - Information about all of the constraints.
10203 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
10204
10205 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10206 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
10207 DL: DAG.getDataLayout(), TRI: DAG.getSubtarget().getRegisterInfo(), Call);
10208
10209 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
10210 // AsmDialect, MayLoad, MayStore).
10211 bool HasSideEffect = IA->hasSideEffects();
10212 ExtraFlags ExtraInfo(Call);
10213
10214 for (auto &T : TargetConstraints) {
10215 ConstraintOperands.push_back(Elt: SDISelAsmOperandInfo(T));
10216 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
10217
10218 if (OpInfo.CallOperandVal)
10219 OpInfo.CallOperand = getValue(V: OpInfo.CallOperandVal);
10220
10221 if (!HasSideEffect)
10222 HasSideEffect = OpInfo.hasMemory(TLI);
10223
10224 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
10225 // FIXME: Could we compute this on OpInfo rather than T?
10226
10227 // Compute the constraint code and ConstraintType to use.
10228 TLI.ComputeConstraintToUse(OpInfo&: T, Op: SDValue());
10229
10230 if (T.ConstraintType == TargetLowering::C_Immediate &&
10231 OpInfo.CallOperand && !isa<ConstantSDNode>(Val: OpInfo.CallOperand))
10232 // We've delayed emitting a diagnostic like the "n" constraint because
10233 // inlining could cause an integer showing up.
10234 return emitInlineAsmError(Call, Message: "constraint '" + Twine(T.ConstraintCode) +
10235 "' expects an integer constant "
10236 "expression");
10237
10238 ExtraInfo.update(OpInfo: T);
10239 }
10240
10241 // We won't need to flush pending loads if this asm doesn't touch
10242 // memory and is nonvolatile.
10243 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
10244
10245 bool EmitEHLabels = isa<InvokeInst>(Val: Call);
10246 if (EmitEHLabels) {
10247 assert(EHPadBB && "InvokeInst must have an EHPadBB");
10248 }
10249 bool IsCallBr = isa<CallBrInst>(Val: Call);
10250
10251 if (IsCallBr || EmitEHLabels) {
10252 // If this is a callbr or invoke we need to flush pending exports since
10253 // inlineasm_br and invoke are terminators.
10254 // We need to do this before nodes are glued to the inlineasm_br node.
10255 Chain = getControlRoot();
10256 }
10257
10258 MCSymbol *BeginLabel = nullptr;
10259 if (EmitEHLabels) {
10260 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
10261 }
10262
10263 int OpNo = -1;
10264 SmallVector<StringRef> AsmStrs;
10265 IA->collectAsmStrs(AsmStrs);
10266
10267 // Second pass over the constraints: compute which constraint option to use.
10268 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10269 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10270 OpNo++;
10271
10272 // If this is an output operand with a matching input operand, look up the
10273 // matching input. If their types mismatch, e.g. one is an integer, the
10274 // other is floating point, or their sizes are different, flag it as an
10275 // error.
10276 if (OpInfo.hasMatchingInput()) {
10277 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
10278 patchMatchingInput(OpInfo, MatchingOpInfo&: Input, DAG);
10279 }
10280
10281 // Compute the constraint code and ConstraintType to use.
10282 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
10283
10284 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10285 OpInfo.Type == InlineAsm::isClobber) ||
10286 OpInfo.ConstraintType == TargetLowering::C_Address)
10287 continue;
10288
10289 // In Linux PIC model, there are 4 cases about value/label addressing:
10290 //
10291 // 1: Function call or Label jmp inside the module.
10292 // 2: Data access (such as global variable, static variable) inside module.
10293 // 3: Function call or Label jmp outside the module.
10294 // 4: Data access (such as global variable) outside the module.
10295 //
10296 // Due to current llvm inline asm architecture designed to not "recognize"
10297 // the asm code, there are quite troubles for us to treat mem addressing
10298 // differently for same value/adress used in different instuctions.
10299 // For example, in pic model, call a func may in plt way or direclty
10300 // pc-related, but lea/mov a function adress may use got.
10301 //
10302 // Here we try to "recognize" function call for the case 1 and case 3 in
10303 // inline asm. And try to adjust the constraint for them.
10304 //
10305 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10306 // label, so here we don't handle jmp function label now, but we need to
10307 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10308 if (OpInfo.isIndirect && isFunction(Op: OpInfo.CallOperand) &&
10309 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10310 TM.getCodeModel() != CodeModel::Large) {
10311 OpInfo.isIndirect = false;
10312 OpInfo.ConstraintType = TargetLowering::C_Address;
10313 }
10314
10315 // If this is a memory input, and if the operand is not indirect, do what we
10316 // need to provide an address for the memory input.
10317 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10318 !OpInfo.isIndirect) {
10319 assert((OpInfo.isMultipleAlternative ||
10320 (OpInfo.Type == InlineAsm::isInput)) &&
10321 "Can only indirectify direct input operands!");
10322
10323 // Memory operands really want the address of the value.
10324 Chain = getAddressForMemoryInput(Chain, Location: getCurSDLoc(), OpInfo, DAG);
10325
10326 // There is no longer a Value* corresponding to this operand.
10327 OpInfo.CallOperandVal = nullptr;
10328
10329 // It is now an indirect operand.
10330 OpInfo.isIndirect = true;
10331 }
10332
10333 }
10334
10335 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10336 std::vector<SDValue> AsmNodeOperands;
10337 AsmNodeOperands.push_back(x: SDValue()); // reserve space for input chain
10338 AsmNodeOperands.push_back(x: DAG.getTargetExternalSymbol(
10339 Sym: IA->getAsmString().data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout())));
10340
10341 // If we have a !srcloc metadata node associated with it, we want to attach
10342 // this to the ultimately generated inline asm machineinstr. To do this, we
10343 // pass in the third operand as this (potentially null) inline asm MDNode.
10344 const MDNode *SrcLoc = Call.getMetadata(Kind: "srcloc");
10345 AsmNodeOperands.push_back(x: DAG.getMDNode(MD: SrcLoc));
10346
10347 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10348 // bits as operand 3.
10349 AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10350 Val: ExtraInfo.get(), DL: getCurSDLoc(), VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10351
10352 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
10353 // this, assign virtual and physical registers for inputs and otput.
10354 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10355 // Assign Registers.
10356 SDISelAsmOperandInfo &RefOpInfo =
10357 OpInfo.isMatchingInputConstraint()
10358 ? ConstraintOperands[OpInfo.getMatchedOperand()]
10359 : OpInfo;
10360 const auto RegError =
10361 getRegistersForValue(DAG, DL: getCurSDLoc(), OpInfo, RefOpInfo);
10362 if (RegError) {
10363 const MachineFunction &MF = DAG.getMachineFunction();
10364 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10365 const char *RegName = TRI.getName(RegNo: *RegError);
10366 emitInlineAsmError(Call, Message: "register '" + Twine(RegName) +
10367 "' allocated for constraint '" +
10368 Twine(OpInfo.ConstraintCode) +
10369 "' does not match required type");
10370 return;
10371 }
10372
10373 auto DetectWriteToReservedRegister = [&]() {
10374 const MachineFunction &MF = DAG.getMachineFunction();
10375 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10376 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10377 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, PhysReg: Reg)) {
10378 const char *RegName = TRI.getName(RegNo: Reg);
10379 emitInlineAsmError(Call, Message: "write to reserved register '" +
10380 Twine(RegName) + "'");
10381 return true;
10382 }
10383 }
10384 return false;
10385 };
10386 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10387 (OpInfo.Type == InlineAsm::isInput &&
10388 !OpInfo.isMatchingInputConstraint())) &&
10389 "Only address as input operand is allowed.");
10390
10391 switch (OpInfo.Type) {
10392 case InlineAsm::isOutput:
10393 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10394 const InlineAsm::ConstraintCode ConstraintID =
10395 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10396 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10397 "Failed to convert memory constraint code to constraint id.");
10398
10399 // Add information to the INLINEASM node to know about this output.
10400 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
10401 OpFlags.setMemConstraint(ConstraintID);
10402 AsmNodeOperands.push_back(x: DAG.getTargetConstant(Val: OpFlags, DL: getCurSDLoc(),
10403 VT: MVT::i32));
10404 AsmNodeOperands.push_back(x: OpInfo.CallOperand);
10405 } else {
10406 // Otherwise, this outputs to a register (directly for C_Register /
10407 // C_RegisterClass, and a target-defined fashion for
10408 // C_Immediate/C_Other). Find a register that we can use.
10409 if (OpInfo.AssignedRegs.Regs.empty()) {
10410 emitInlineAsmError(
10411 Call, Message: "couldn't allocate output register for constraint '" +
10412 Twine(OpInfo.ConstraintCode) + "'");
10413 return;
10414 }
10415
10416 if (DetectWriteToReservedRegister())
10417 return;
10418
10419 // Add information to the INLINEASM node to know that this register is
10420 // set.
10421 OpInfo.AssignedRegs.AddInlineAsmOperands(
10422 Code: OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10423 : InlineAsm::Kind::RegDef,
10424 HasMatching: false, MatchingIdx: 0, dl: getCurSDLoc(), DAG, Ops&: AsmNodeOperands);
10425 }
10426 break;
10427
10428 case InlineAsm::isInput:
10429 case InlineAsm::isLabel: {
10430 SDValue InOperandVal = OpInfo.CallOperand;
10431
10432 if (OpInfo.isMatchingInputConstraint()) {
10433 // If this is required to match an output register we have already set,
10434 // just use its register.
10435 auto CurOp = findMatchingInlineAsmOperand(OperandNo: OpInfo.getMatchedOperand(),
10436 AsmNodeOperands);
10437 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
10438 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10439 if (OpInfo.isIndirect) {
10440 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10441 emitInlineAsmError(Call, Message: "inline asm not supported yet: "
10442 "don't know how to handle tied "
10443 "indirect register inputs");
10444 return;
10445 }
10446
10447 SmallVector<Register, 4> Regs;
10448 MachineFunction &MF = DAG.getMachineFunction();
10449 MachineRegisterInfo &MRI = MF.getRegInfo();
10450 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10451 auto *R = cast<RegisterSDNode>(Val&: AsmNodeOperands[CurOp+1]);
10452 Register TiedReg = R->getReg();
10453 MVT RegVT = R->getSimpleValueType(ResNo: 0);
10454 const TargetRegisterClass *RC =
10455 TiedReg.isVirtual() ? MRI.getRegClass(Reg: TiedReg)
10456 : RegVT != MVT::Untyped ? TLI.getRegClassFor(VT: RegVT)
10457 : TRI.getMinimalPhysRegClass(Reg: TiedReg);
10458 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
10459 Regs.push_back(Elt: MRI.createVirtualRegister(RegClass: RC));
10460
10461 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10462
10463 SDLoc dl = getCurSDLoc();
10464 // Use the produced MatchedRegs object to
10465 MatchedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl, Chain, Glue: &Glue, V: &Call);
10466 MatchedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::RegUse, HasMatching: true,
10467 MatchingIdx: OpInfo.getMatchedOperand(), dl, DAG,
10468 Ops&: AsmNodeOperands);
10469 break;
10470 }
10471
10472 assert(Flag.isMemKind() && "Unknown matching constraint!");
10473 assert(Flag.getNumOperandRegisters() == 1 &&
10474 "Unexpected number of operands");
10475 // Add information to the INLINEASM node to know about this input.
10476 // See InlineAsm.h isUseOperandTiedToDef.
10477 Flag.clearMemConstraint();
10478 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10479 AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10480 Val: Flag, DL: getCurSDLoc(), VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10481 AsmNodeOperands.push_back(x: AsmNodeOperands[CurOp+1]);
10482 break;
10483 }
10484
10485 // Treat indirect 'X' constraint as memory.
10486 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10487 OpInfo.isIndirect)
10488 OpInfo.ConstraintType = TargetLowering::C_Memory;
10489
10490 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10491 OpInfo.ConstraintType == TargetLowering::C_Other) {
10492 std::vector<SDValue> Ops;
10493 TLI.LowerAsmOperandForConstraint(Op: InOperandVal, Constraint: OpInfo.ConstraintCode,
10494 Ops, DAG);
10495 if (Ops.empty()) {
10496 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10497 if (isa<ConstantSDNode>(Val: InOperandVal)) {
10498 emitInlineAsmError(Call, Message: "value out of range for constraint '" +
10499 Twine(OpInfo.ConstraintCode) + "'");
10500 return;
10501 }
10502
10503 emitInlineAsmError(Call,
10504 Message: "invalid operand for inline asm constraint '" +
10505 Twine(OpInfo.ConstraintCode) + "'");
10506 return;
10507 }
10508
10509 // Add information to the INLINEASM node to know about this input.
10510 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10511 AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10512 Val: ResOpType, DL: getCurSDLoc(), VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10513 llvm::append_range(C&: AsmNodeOperands, R&: Ops);
10514 break;
10515 }
10516
10517 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10518 assert((OpInfo.isIndirect ||
10519 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10520 "Operand must be indirect to be a mem!");
10521 assert(InOperandVal.getValueType() ==
10522 TLI.getPointerTy(DAG.getDataLayout()) &&
10523 "Memory operands expect pointer values");
10524
10525 const InlineAsm::ConstraintCode ConstraintID =
10526 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10527 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10528 "Failed to convert memory constraint code to constraint id.");
10529
10530 // Add information to the INLINEASM node to know about this input.
10531 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10532 ResOpType.setMemConstraint(ConstraintID);
10533 AsmNodeOperands.push_back(x: DAG.getTargetConstant(Val: ResOpType,
10534 DL: getCurSDLoc(),
10535 VT: MVT::i32));
10536 AsmNodeOperands.push_back(x: InOperandVal);
10537 break;
10538 }
10539
10540 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10541 const InlineAsm::ConstraintCode ConstraintID =
10542 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10543 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10544 "Failed to convert memory constraint code to constraint id.");
10545
10546 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10547
10548 SDValue AsmOp = InOperandVal;
10549 if (isFunction(Op: InOperandVal)) {
10550 auto *GA = cast<GlobalAddressSDNode>(Val&: InOperandVal);
10551 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10552 AsmOp = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: getCurSDLoc(),
10553 VT: InOperandVal.getValueType(),
10554 offset: GA->getOffset());
10555 }
10556
10557 // Add information to the INLINEASM node to know about this input.
10558 ResOpType.setMemConstraint(ConstraintID);
10559
10560 AsmNodeOperands.push_back(
10561 x: DAG.getTargetConstant(Val: ResOpType, DL: getCurSDLoc(), VT: MVT::i32));
10562
10563 AsmNodeOperands.push_back(x: AsmOp);
10564 break;
10565 }
10566
10567 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10568 OpInfo.ConstraintType != TargetLowering::C_Register) {
10569 emitInlineAsmError(Call, Message: "unknown asm constraint '" +
10570 Twine(OpInfo.ConstraintCode) + "'");
10571 return;
10572 }
10573
10574 // TODO: Support this.
10575 if (OpInfo.isIndirect) {
10576 emitInlineAsmError(
10577 Call, Message: "Don't know how to handle indirect register inputs yet "
10578 "for constraint '" +
10579 Twine(OpInfo.ConstraintCode) + "'");
10580 return;
10581 }
10582
10583 // Copy the input into the appropriate registers.
10584 if (OpInfo.AssignedRegs.Regs.empty()) {
10585 emitInlineAsmError(Call,
10586 Message: "couldn't allocate input reg for constraint '" +
10587 Twine(OpInfo.ConstraintCode) + "'");
10588 return;
10589 }
10590
10591 if (DetectWriteToReservedRegister())
10592 return;
10593
10594 SDLoc dl = getCurSDLoc();
10595
10596 OpInfo.AssignedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl, Chain, Glue: &Glue,
10597 V: &Call);
10598
10599 OpInfo.AssignedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::RegUse, HasMatching: false,
10600 MatchingIdx: 0, dl, DAG, Ops&: AsmNodeOperands);
10601 break;
10602 }
10603 case InlineAsm::isClobber:
10604 // Add the clobbered value to the operand list, so that the register
10605 // allocator is aware that the physreg got clobbered.
10606 if (!OpInfo.AssignedRegs.Regs.empty())
10607 OpInfo.AssignedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::Clobber,
10608 HasMatching: false, MatchingIdx: 0, dl: getCurSDLoc(), DAG,
10609 Ops&: AsmNodeOperands);
10610 break;
10611 }
10612 }
10613
10614 // Finish up input operands. Set the input chain and add the flag last.
10615 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10616 if (Glue.getNode()) AsmNodeOperands.push_back(x: Glue);
10617
10618 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10619 Chain = DAG.getNode(Opcode: ISDOpc, DL: getCurSDLoc(),
10620 VTList: DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue), Ops: AsmNodeOperands);
10621 Glue = Chain.getValue(R: 1);
10622
10623 // Do additional work to generate outputs.
10624
10625 SmallVector<EVT, 1> ResultVTs;
10626 SmallVector<SDValue, 1> ResultValues;
10627 SmallVector<SDValue, 8> OutChains;
10628
10629 llvm::Type *CallResultType = Call.getType();
10630 ArrayRef<Type *> ResultTypes;
10631 if (StructType *StructResult = dyn_cast<StructType>(Val: CallResultType))
10632 ResultTypes = StructResult->elements();
10633 else if (!CallResultType->isVoidTy())
10634 ResultTypes = ArrayRef(CallResultType);
10635
10636 auto CurResultType = ResultTypes.begin();
10637 auto handleRegAssign = [&](SDValue V) {
10638 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10639 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10640 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: *CurResultType);
10641 ++CurResultType;
10642 // If the type of the inline asm call site return value is different but has
10643 // same size as the type of the asm output bitcast it. One example of this
10644 // is for vectors with different width / number of elements. This can
10645 // happen for register classes that can contain multiple different value
10646 // types. The preg or vreg allocated may not have the same VT as was
10647 // expected.
10648 //
10649 // This can also happen for a return value that disagrees with the register
10650 // class it is put in, eg. a double in a general-purpose register on a
10651 // 32-bit machine.
10652 if (ResultVT != V.getValueType() &&
10653 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10654 V = DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10655 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10656 V.getValueType().isInteger()) {
10657 // If a result value was tied to an input value, the computed result
10658 // may have a wider width than the expected result. Extract the
10659 // relevant portion.
10660 V = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10661 }
10662 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10663 ResultVTs.push_back(Elt: ResultVT);
10664 ResultValues.push_back(Elt: V);
10665 };
10666
10667 // Deal with output operands.
10668 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10669 if (OpInfo.Type == InlineAsm::isOutput) {
10670 SDValue Val;
10671 // Skip trivial output operands.
10672 if (OpInfo.AssignedRegs.Regs.empty())
10673 continue;
10674
10675 switch (OpInfo.ConstraintType) {
10676 case TargetLowering::C_Register:
10677 case TargetLowering::C_RegisterClass:
10678 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(),
10679 Chain, Glue: &Glue, V: &Call);
10680 break;
10681 case TargetLowering::C_Immediate:
10682 case TargetLowering::C_Other:
10683 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, DL: getCurSDLoc(),
10684 OpInfo, DAG);
10685 break;
10686 case TargetLowering::C_Memory:
10687 break; // Already handled.
10688 case TargetLowering::C_Address:
10689 break; // Silence warning.
10690 case TargetLowering::C_Unknown:
10691 assert(false && "Unexpected unknown constraint");
10692 }
10693
10694 // Indirect output manifest as stores. Record output chains.
10695 if (OpInfo.isIndirect) {
10696 const Value *Ptr = OpInfo.CallOperandVal;
10697 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10698 SDValue Store = DAG.getStore(Chain, dl: getCurSDLoc(), Val, Ptr: getValue(V: Ptr),
10699 PtrInfo: MachinePointerInfo(Ptr));
10700 OutChains.push_back(Elt: Store);
10701 } else {
10702 // generate CopyFromRegs to associated registers.
10703 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10704 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10705 for (const SDValue &V : Val->op_values())
10706 handleRegAssign(V);
10707 } else
10708 handleRegAssign(Val);
10709 }
10710 }
10711 }
10712
10713 // Set results.
10714 if (!ResultValues.empty()) {
10715 assert(CurResultType == ResultTypes.end() &&
10716 "Mismatch in number of ResultTypes");
10717 assert(ResultValues.size() == ResultTypes.size() &&
10718 "Mismatch in number of output operands in asm result");
10719
10720 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
10721 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
10722 setValue(V: &Call, NewN: V);
10723 }
10724
10725 // Collect store chains.
10726 if (!OutChains.empty())
10727 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: getCurSDLoc(), VT: MVT::Other, Ops: OutChains);
10728
10729 if (EmitEHLabels) {
10730 Chain = lowerEndEH(Chain, II: cast<InvokeInst>(Val: &Call), EHPadBB, BeginLabel);
10731 }
10732
10733 // Only Update Root if inline assembly has a memory effect.
10734 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10735 EmitEHLabels)
10736 DAG.setRoot(Chain);
10737}
10738
10739void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10740 const Twine &Message) {
10741 LLVMContext &Ctx = *DAG.getContext();
10742 Ctx.diagnose(DI: DiagnosticInfoInlineAsm(Call, Message));
10743
10744 // Make sure we leave the DAG in a valid state
10745 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10746 SmallVector<EVT, 1> ValueVTs;
10747 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: Call.getType(), ValueVTs);
10748
10749 if (ValueVTs.empty())
10750 return;
10751
10752 SmallVector<SDValue, 1> Ops;
10753 for (const EVT &VT : ValueVTs)
10754 Ops.push_back(Elt: DAG.getUNDEF(VT));
10755
10756 setValue(V: &Call, NewN: DAG.getMergeValues(Ops, dl: getCurSDLoc()));
10757}
10758
10759void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10760 DAG.setRoot(DAG.getNode(Opcode: ISD::VASTART, DL: getCurSDLoc(),
10761 VT: MVT::Other, N1: getRoot(),
10762 N2: getValue(V: I.getArgOperand(i: 0)),
10763 N3: DAG.getSrcValue(v: I.getArgOperand(i: 0))));
10764}
10765
10766void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10767 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10768 const DataLayout &DL = DAG.getDataLayout();
10769 SDValue V = DAG.getVAArg(
10770 VT: TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType()), dl: getCurSDLoc(),
10771 Chain: getRoot(), Ptr: getValue(V: I.getOperand(i_nocapture: 0)), SV: DAG.getSrcValue(v: I.getOperand(i_nocapture: 0)),
10772 Align: DL.getABITypeAlign(Ty: I.getType()).value());
10773 DAG.setRoot(V.getValue(R: 1));
10774
10775 if (I.getType()->isPointerTy())
10776 V = DAG.getPtrExtOrTrunc(
10777 Op: V, DL: getCurSDLoc(), VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()));
10778 setValue(V: &I, NewN: V);
10779}
10780
10781void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10782 DAG.setRoot(DAG.getNode(Opcode: ISD::VAEND, DL: getCurSDLoc(),
10783 VT: MVT::Other, N1: getRoot(),
10784 N2: getValue(V: I.getArgOperand(i: 0)),
10785 N3: DAG.getSrcValue(v: I.getArgOperand(i: 0))));
10786}
10787
10788void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10789 DAG.setRoot(DAG.getNode(Opcode: ISD::VACOPY, DL: getCurSDLoc(),
10790 VT: MVT::Other, N1: getRoot(),
10791 N2: getValue(V: I.getArgOperand(i: 0)),
10792 N3: getValue(V: I.getArgOperand(i: 1)),
10793 N4: DAG.getSrcValue(v: I.getArgOperand(i: 0)),
10794 N5: DAG.getSrcValue(v: I.getArgOperand(i: 1))));
10795}
10796
10797SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
10798 const Instruction &I,
10799 SDValue Op) {
10800 std::optional<ConstantRange> CR = getRange(I);
10801
10802 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
10803 return Op;
10804
10805 APInt Lo = CR->getUnsignedMin();
10806 if (!Lo.isMinValue())
10807 return Op;
10808
10809 APInt Hi = CR->getUnsignedMax();
10810 unsigned Bits = std::max(a: Hi.getActiveBits(),
10811 b: static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10812
10813 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
10814
10815 SDLoc SL = getCurSDLoc();
10816
10817 SDValue ZExt = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: Op.getValueType(), N1: Op,
10818 N2: DAG.getValueType(SmallVT));
10819 unsigned NumVals = Op.getNode()->getNumValues();
10820 if (NumVals == 1)
10821 return ZExt;
10822
10823 SmallVector<SDValue, 4> Ops;
10824
10825 Ops.push_back(Elt: ZExt);
10826 for (unsigned I = 1; I != NumVals; ++I)
10827 Ops.push_back(Elt: Op.getValue(R: I));
10828
10829 return DAG.getMergeValues(Ops, dl: SL);
10830}
10831
10832SDValue SelectionDAGBuilder::lowerNoFPClassToAssertNoFPClass(
10833 SelectionDAG &DAG, const Instruction &I, SDValue Op) {
10834 FPClassTest Classes = getNoFPClass(I);
10835 if (Classes == fcNone)
10836 return Op;
10837
10838 SDLoc SL = getCurSDLoc();
10839 SDValue TestConst = DAG.getTargetConstant(Val: Classes, DL: SDLoc(), VT: MVT::i32);
10840
10841 if (Op.getOpcode() != ISD::MERGE_VALUES) {
10842 return DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: SL, VT: Op.getValueType(), N1: Op,
10843 N2: TestConst);
10844 }
10845
10846 SmallVector<SDValue, 8> Ops(Op.getNumOperands());
10847 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
10848 SDValue MergeOp = Op.getOperand(i: I);
10849 Ops[I] = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: SL, VT: MergeOp.getValueType(),
10850 N1: MergeOp, N2: TestConst);
10851 }
10852
10853 return DAG.getMergeValues(Ops, dl: SL);
10854}
10855
10856/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10857/// the call being lowered.
10858///
10859/// This is a helper for lowering intrinsics that follow a target calling
10860/// convention or require stack pointer adjustment. Only a subset of the
10861/// intrinsic's operands need to participate in the calling convention.
10862void SelectionDAGBuilder::populateCallLoweringInfo(
10863 TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
10864 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10865 AttributeSet RetAttrs, bool IsPatchPoint) {
10866 TargetLowering::ArgListTy Args;
10867 Args.reserve(n: NumArgs);
10868
10869 // Populate the argument list.
10870 // Attributes for args start at offset 1, after the return attribute.
10871 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10872 ArgI != ArgE; ++ArgI) {
10873 const Value *V = Call->getOperand(i_nocapture: ArgI);
10874
10875 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10876
10877 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
10878 Entry.setAttributes(Call, ArgIdx: ArgI);
10879 Args.push_back(x: Entry);
10880 }
10881
10882 CLI.setDebugLoc(getCurSDLoc())
10883 .setChain(getRoot())
10884 .setCallee(CC: Call->getCallingConv(), ResultType: ReturnTy, Target: Callee, ArgsList: std::move(Args),
10885 ResultAttrs: RetAttrs)
10886 .setDiscardResult(Call->use_empty())
10887 .setIsPatchPoint(IsPatchPoint)
10888 .setIsPreallocated(
10889 Call->countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0);
10890}
10891
10892/// Add a stack map intrinsic call's live variable operands to a stackmap
10893/// or patchpoint target node's operand list.
10894///
10895/// Constants are converted to TargetConstants purely as an optimization to
10896/// avoid constant materialization and register allocation.
10897///
10898/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10899/// generate addess computation nodes, and so FinalizeISel can convert the
10900/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10901/// address materialization and register allocation, but may also be required
10902/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10903/// alloca in the entry block, then the runtime may assume that the alloca's
10904/// StackMap location can be read immediately after compilation and that the
10905/// location is valid at any point during execution (this is similar to the
10906/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10907/// only available in a register, then the runtime would need to trap when
10908/// execution reaches the StackMap in order to read the alloca's location.
10909static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10910 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
10911 SelectionDAGBuilder &Builder) {
10912 SelectionDAG &DAG = Builder.DAG;
10913 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10914 SDValue Op = Builder.getValue(V: Call.getArgOperand(i: I));
10915
10916 // Things on the stack are pointer-typed, meaning that they are already
10917 // legal and can be emitted directly to target nodes.
10918 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op)) {
10919 Ops.push_back(Elt: DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: Op.getValueType()));
10920 } else {
10921 // Otherwise emit a target independent node to be legalised.
10922 Ops.push_back(Elt: Builder.getValue(V: Call.getArgOperand(i: I)));
10923 }
10924 }
10925}
10926
10927/// Lower llvm.experimental.stackmap.
10928void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10929 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10930 // [live variables...])
10931
10932 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10933
10934 SDValue Chain, InGlue, Callee;
10935 SmallVector<SDValue, 32> Ops;
10936
10937 SDLoc DL = getCurSDLoc();
10938 Callee = getValue(V: CI.getCalledOperand());
10939
10940 // The stackmap intrinsic only records the live variables (the arguments
10941 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10942 // intrinsic, this won't be lowered to a function call. This means we don't
10943 // have to worry about calling conventions and target specific lowering code.
10944 // Instead we perform the call lowering right here.
10945 //
10946 // chain, flag = CALLSEQ_START(chain, 0, 0)
10947 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10948 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10949 //
10950 Chain = DAG.getCALLSEQ_START(Chain: getRoot(), InSize: 0, OutSize: 0, DL);
10951 InGlue = Chain.getValue(R: 1);
10952
10953 // Add the STACKMAP operands, starting with DAG house-keeping.
10954 Ops.push_back(Elt: Chain);
10955 Ops.push_back(Elt: InGlue);
10956
10957 // Add the <id>, <numShadowBytes> operands.
10958 //
10959 // These do not require legalisation, and can be emitted directly to target
10960 // constant nodes.
10961 SDValue ID = getValue(V: CI.getArgOperand(i: 0));
10962 assert(ID.getValueType() == MVT::i64);
10963 SDValue IDConst =
10964 DAG.getTargetConstant(Val: ID->getAsZExtVal(), DL, VT: ID.getValueType());
10965 Ops.push_back(Elt: IDConst);
10966
10967 SDValue Shad = getValue(V: CI.getArgOperand(i: 1));
10968 assert(Shad.getValueType() == MVT::i32);
10969 SDValue ShadConst =
10970 DAG.getTargetConstant(Val: Shad->getAsZExtVal(), DL, VT: Shad.getValueType());
10971 Ops.push_back(Elt: ShadConst);
10972
10973 // Add the live variables.
10974 addStackMapLiveVars(Call: CI, StartIdx: 2, DL, Ops, Builder&: *this);
10975
10976 // Create the STACKMAP node.
10977 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
10978 Chain = DAG.getNode(Opcode: ISD::STACKMAP, DL, VTList: NodeTys, Ops);
10979 InGlue = Chain.getValue(R: 1);
10980
10981 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: InGlue, DL);
10982
10983 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10984
10985 // Set the root to the target-lowered call chain.
10986 DAG.setRoot(Chain);
10987
10988 // Inform the Frame Information that we have a stackmap in this function.
10989 FuncInfo.MF->getFrameInfo().setHasStackMap();
10990}
10991
10992/// Lower llvm.experimental.patchpoint directly to its target opcode.
10993void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10994 const BasicBlock *EHPadBB) {
10995 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
10996 // i32 <numBytes>,
10997 // i8* <target>,
10998 // i32 <numArgs>,
10999 // [Args...],
11000 // [live variables...])
11001
11002 CallingConv::ID CC = CB.getCallingConv();
11003 bool IsAnyRegCC = CC == CallingConv::AnyReg;
11004 bool HasDef = !CB.getType()->isVoidTy();
11005 SDLoc dl = getCurSDLoc();
11006 SDValue Callee = getValue(V: CB.getArgOperand(i: PatchPointOpers::TargetPos));
11007
11008 // Handle immediate and symbolic callees.
11009 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Val&: Callee))
11010 Callee = DAG.getIntPtrConstant(Val: ConstCallee->getZExtValue(), DL: dl,
11011 /*isTarget=*/true);
11012 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11013 Callee = DAG.getTargetGlobalAddress(GV: SymbolicCallee->getGlobal(),
11014 DL: SDLoc(SymbolicCallee),
11015 VT: SymbolicCallee->getValueType(ResNo: 0));
11016
11017 // Get the real number of arguments participating in the call <numArgs>
11018 SDValue NArgVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NArgPos));
11019 unsigned NumArgs = NArgVal->getAsZExtVal();
11020
11021 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
11022 // Intrinsics include all meta-operands up to but not including CC.
11023 unsigned NumMetaOpers = PatchPointOpers::CCPos;
11024 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
11025 "Not enough arguments provided to the patchpoint intrinsic");
11026
11027 // For AnyRegCC the arguments are lowered later on manually.
11028 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11029 Type *ReturnTy =
11030 IsAnyRegCC ? Type::getVoidTy(C&: *DAG.getContext()) : CB.getType();
11031
11032 TargetLowering::CallLoweringInfo CLI(DAG);
11033 populateCallLoweringInfo(CLI, Call: &CB, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee,
11034 ReturnTy, RetAttrs: CB.getAttributes().getRetAttrs(), IsPatchPoint: true);
11035 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
11036
11037 SDNode *CallEnd = Result.second.getNode();
11038 if (CallEnd->getOpcode() == ISD::EH_LABEL)
11039 CallEnd = CallEnd->getOperand(Num: 0).getNode();
11040 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
11041 CallEnd = CallEnd->getOperand(Num: 0).getNode();
11042
11043 /// Get a call instruction from the call sequence chain.
11044 /// Tail calls are not allowed.
11045 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
11046 "Expected a callseq node.");
11047 SDNode *Call = CallEnd->getOperand(Num: 0).getNode();
11048 bool HasGlue = Call->getGluedNode();
11049
11050 // Replace the target specific call node with the patchable intrinsic.
11051 SmallVector<SDValue, 8> Ops;
11052
11053 // Push the chain.
11054 Ops.push_back(Elt: *(Call->op_begin()));
11055
11056 // Optionally, push the glue (if any).
11057 if (HasGlue)
11058 Ops.push_back(Elt: *(Call->op_end() - 1));
11059
11060 // Push the register mask info.
11061 if (HasGlue)
11062 Ops.push_back(Elt: *(Call->op_end() - 2));
11063 else
11064 Ops.push_back(Elt: *(Call->op_end() - 1));
11065
11066 // Add the <id> and <numBytes> constants.
11067 SDValue IDVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::IDPos));
11068 Ops.push_back(Elt: DAG.getTargetConstant(Val: IDVal->getAsZExtVal(), DL: dl, VT: MVT::i64));
11069 SDValue NBytesVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NBytesPos));
11070 Ops.push_back(Elt: DAG.getTargetConstant(Val: NBytesVal->getAsZExtVal(), DL: dl, VT: MVT::i32));
11071
11072 // Add the callee.
11073 Ops.push_back(Elt: Callee);
11074
11075 // Adjust <numArgs> to account for any arguments that have been passed on the
11076 // stack instead.
11077 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
11078 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
11079 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11080 Ops.push_back(Elt: DAG.getTargetConstant(Val: NumCallRegArgs, DL: dl, VT: MVT::i32));
11081
11082 // Add the calling convention
11083 Ops.push_back(Elt: DAG.getTargetConstant(Val: (unsigned)CC, DL: dl, VT: MVT::i32));
11084
11085 // Add the arguments we omitted previously. The register allocator should
11086 // place these in any free register.
11087 if (IsAnyRegCC)
11088 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
11089 Ops.push_back(Elt: getValue(V: CB.getArgOperand(i)));
11090
11091 // Push the arguments from the call instruction.
11092 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
11093 Ops.append(in_start: Call->op_begin() + 2, in_end: e);
11094
11095 // Push live variables for the stack map.
11096 addStackMapLiveVars(Call: CB, StartIdx: NumMetaOpers + NumArgs, DL: dl, Ops, Builder&: *this);
11097
11098 SDVTList NodeTys;
11099 if (IsAnyRegCC && HasDef) {
11100 // Create the return types based on the intrinsic definition
11101 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11102 SmallVector<EVT, 3> ValueVTs;
11103 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: CB.getType(), ValueVTs);
11104 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
11105
11106 // There is always a chain and a glue type at the end
11107 ValueVTs.push_back(Elt: MVT::Other);
11108 ValueVTs.push_back(Elt: MVT::Glue);
11109 NodeTys = DAG.getVTList(VTs: ValueVTs);
11110 } else
11111 NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
11112
11113 // Replace the target specific call node with a PATCHPOINT node.
11114 SDValue PPV = DAG.getNode(Opcode: ISD::PATCHPOINT, DL: dl, VTList: NodeTys, Ops);
11115
11116 // Update the NodeMap.
11117 if (HasDef) {
11118 if (IsAnyRegCC)
11119 setValue(V: &CB, NewN: SDValue(PPV.getNode(), 0));
11120 else
11121 setValue(V: &CB, NewN: Result.first);
11122 }
11123
11124 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
11125 // call sequence. Furthermore the location of the chain and glue can change
11126 // when the AnyReg calling convention is used and the intrinsic returns a
11127 // value.
11128 if (IsAnyRegCC && HasDef) {
11129 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
11130 SDValue To[] = {PPV.getValue(R: 1), PPV.getValue(R: 2)};
11131 DAG.ReplaceAllUsesOfValuesWith(From, To, Num: 2);
11132 } else
11133 DAG.ReplaceAllUsesWith(From: Call, To: PPV.getNode());
11134 DAG.DeleteNode(N: Call);
11135
11136 // Inform the Frame Information that we have a patchpoint in this function.
11137 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11138}
11139
11140void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
11141 unsigned Intrinsic) {
11142 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11143 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
11144 SDValue Op2;
11145 if (I.arg_size() > 1)
11146 Op2 = getValue(V: I.getArgOperand(i: 1));
11147 SDLoc dl = getCurSDLoc();
11148 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
11149 SDValue Res;
11150 SDNodeFlags SDFlags;
11151 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
11152 SDFlags.copyFMF(FPMO: *FPMO);
11153
11154 switch (Intrinsic) {
11155 case Intrinsic::vector_reduce_fadd:
11156 if (SDFlags.hasAllowReassociation())
11157 Res = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Op1,
11158 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FADD, DL: dl, VT, Operand: Op2, Flags: SDFlags),
11159 Flags: SDFlags);
11160 else
11161 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FADD, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
11162 break;
11163 case Intrinsic::vector_reduce_fmul:
11164 if (SDFlags.hasAllowReassociation())
11165 Res = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Op1,
11166 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FMUL, DL: dl, VT, Operand: Op2, Flags: SDFlags),
11167 Flags: SDFlags);
11168 else
11169 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FMUL, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
11170 break;
11171 case Intrinsic::vector_reduce_add:
11172 Res = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT, Operand: Op1);
11173 break;
11174 case Intrinsic::vector_reduce_mul:
11175 Res = DAG.getNode(Opcode: ISD::VECREDUCE_MUL, DL: dl, VT, Operand: Op1);
11176 break;
11177 case Intrinsic::vector_reduce_and:
11178 Res = DAG.getNode(Opcode: ISD::VECREDUCE_AND, DL: dl, VT, Operand: Op1);
11179 break;
11180 case Intrinsic::vector_reduce_or:
11181 Res = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: dl, VT, Operand: Op1);
11182 break;
11183 case Intrinsic::vector_reduce_xor:
11184 Res = DAG.getNode(Opcode: ISD::VECREDUCE_XOR, DL: dl, VT, Operand: Op1);
11185 break;
11186 case Intrinsic::vector_reduce_smax:
11187 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMAX, DL: dl, VT, Operand: Op1);
11188 break;
11189 case Intrinsic::vector_reduce_smin:
11190 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMIN, DL: dl, VT, Operand: Op1);
11191 break;
11192 case Intrinsic::vector_reduce_umax:
11193 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL: dl, VT, Operand: Op1);
11194 break;
11195 case Intrinsic::vector_reduce_umin:
11196 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMIN, DL: dl, VT, Operand: Op1);
11197 break;
11198 case Intrinsic::vector_reduce_fmax:
11199 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAX, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11200 break;
11201 case Intrinsic::vector_reduce_fmin:
11202 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMIN, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11203 break;
11204 case Intrinsic::vector_reduce_fmaximum:
11205 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAXIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11206 break;
11207 case Intrinsic::vector_reduce_fminimum:
11208 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMINIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11209 break;
11210 default:
11211 llvm_unreachable("Unhandled vector reduce intrinsic");
11212 }
11213 setValue(V: &I, NewN: Res);
11214}
11215
11216/// Returns an AttributeList representing the attributes applied to the return
11217/// value of the given call.
11218static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
11219 SmallVector<Attribute::AttrKind, 2> Attrs;
11220 if (CLI.RetSExt)
11221 Attrs.push_back(Elt: Attribute::SExt);
11222 if (CLI.RetZExt)
11223 Attrs.push_back(Elt: Attribute::ZExt);
11224 if (CLI.IsInReg)
11225 Attrs.push_back(Elt: Attribute::InReg);
11226
11227 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
11228 Kinds: Attrs);
11229}
11230
11231/// TargetLowering::LowerCallTo - This is the default LowerCallTo
11232/// implementation, which just calls LowerCall.
11233/// FIXME: When all targets are
11234/// migrated to using LowerCall, this hook should be integrated into SDISel.
11235std::pair<SDValue, SDValue>
11236TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
11237 LLVMContext &Context = CLI.RetTy->getContext();
11238
11239 // Handle the incoming return values from the call.
11240 CLI.Ins.clear();
11241 SmallVector<Type *, 4> RetOrigTys;
11242 SmallVector<TypeSize, 4> Offsets;
11243 auto &DL = CLI.DAG.getDataLayout();
11244 ComputeValueTypes(DL, Ty: CLI.OrigRetTy, Types&: RetOrigTys, Offsets: &Offsets);
11245
11246 SmallVector<EVT, 4> RetVTs;
11247 if (CLI.RetTy != CLI.OrigRetTy) {
11248 assert(RetOrigTys.size() == 1 &&
11249 "Only supported for non-aggregate returns");
11250 RetVTs.push_back(Elt: getValueType(DL, Ty: CLI.RetTy));
11251 } else {
11252 for (Type *Ty : RetOrigTys)
11253 RetVTs.push_back(Elt: getValueType(DL, Ty));
11254 }
11255
11256 if (CLI.IsPostTypeLegalization) {
11257 // If we are lowering a libcall after legalization, split the return type.
11258 SmallVector<Type *, 4> OldRetOrigTys;
11259 SmallVector<EVT, 4> OldRetVTs;
11260 SmallVector<TypeSize, 4> OldOffsets;
11261 RetOrigTys.swap(RHS&: OldRetOrigTys);
11262 RetVTs.swap(RHS&: OldRetVTs);
11263 Offsets.swap(RHS&: OldOffsets);
11264
11265 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
11266 EVT RetVT = OldRetVTs[i];
11267 uint64_t Offset = OldOffsets[i];
11268 MVT RegisterVT = getRegisterType(Context, VT: RetVT);
11269 unsigned NumRegs = getNumRegisters(Context, VT: RetVT);
11270 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
11271 RetOrigTys.append(NumInputs: NumRegs, Elt: OldRetOrigTys[i]);
11272 RetVTs.append(NumInputs: NumRegs, Elt: RegisterVT);
11273 for (unsigned j = 0; j != NumRegs; ++j)
11274 Offsets.push_back(Elt: TypeSize::getFixed(ExactSize: Offset + j * RegisterVTByteSZ));
11275 }
11276 }
11277
11278 SmallVector<ISD::OutputArg, 4> Outs;
11279 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI: *this, DL);
11280
11281 bool CanLowerReturn =
11282 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
11283 CLI.IsVarArg, Outs, Context, RetTy: CLI.RetTy);
11284
11285 SDValue DemoteStackSlot;
11286 int DemoteStackIdx = -100;
11287 if (!CanLowerReturn) {
11288 // FIXME: equivalent assert?
11289 // assert(!CS.hasInAllocaArgument() &&
11290 // "sret demotion is incompatible with inalloca");
11291 uint64_t TySize = DL.getTypeAllocSize(Ty: CLI.RetTy);
11292 Align Alignment = DL.getPrefTypeAlign(Ty: CLI.RetTy);
11293 MachineFunction &MF = CLI.DAG.getMachineFunction();
11294 DemoteStackIdx =
11295 MF.getFrameInfo().CreateStackObject(Size: TySize, Alignment, isSpillSlot: false);
11296 Type *StackSlotPtrType = PointerType::get(C&: Context, AddressSpace: DL.getAllocaAddrSpace());
11297
11298 DemoteStackSlot = CLI.DAG.getFrameIndex(FI: DemoteStackIdx, VT: getFrameIndexTy(DL));
11299 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11300 Entry.IsSRet = true;
11301 Entry.Alignment = Alignment;
11302 CLI.getArgs().insert(position: CLI.getArgs().begin(), x: Entry);
11303 CLI.NumFixedArgs += 1;
11304 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11305 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(C&: Context);
11306
11307 // sret demotion isn't compatible with tail-calls, since the sret argument
11308 // points into the callers stack frame.
11309 CLI.IsTailCall = false;
11310 } else {
11311 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11312 Ty: CLI.RetTy, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11313 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
11314 ISD::ArgFlagsTy Flags;
11315 if (NeedsRegBlock) {
11316 Flags.setInConsecutiveRegs();
11317 if (I == RetVTs.size() - 1)
11318 Flags.setInConsecutiveRegsLast();
11319 }
11320 EVT VT = RetVTs[I];
11321 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11322 unsigned NumRegs =
11323 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11324 for (unsigned i = 0; i != NumRegs; ++i) {
11325 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
11326 CLI.IsReturnValueUsed, ISD::InputArg::NoArgIndex, 0);
11327 if (CLI.RetTy->isPointerTy()) {
11328 Ret.Flags.setPointer();
11329 Ret.Flags.setPointerAddrSpace(
11330 cast<PointerType>(Val: CLI.RetTy)->getAddressSpace());
11331 }
11332 if (CLI.RetSExt)
11333 Ret.Flags.setSExt();
11334 if (CLI.RetZExt)
11335 Ret.Flags.setZExt();
11336 if (CLI.IsInReg)
11337 Ret.Flags.setInReg();
11338 CLI.Ins.push_back(Elt: Ret);
11339 }
11340 }
11341 }
11342
11343 // We push in swifterror return as the last element of CLI.Ins.
11344 ArgListTy &Args = CLI.getArgs();
11345 if (supportSwiftError()) {
11346 for (const ArgListEntry &Arg : Args) {
11347 if (Arg.IsSwiftError) {
11348 ISD::ArgFlagsTy Flags;
11349 Flags.setSwiftError();
11350 ISD::InputArg Ret(Flags, getPointerTy(DL), EVT(getPointerTy(DL)),
11351 PointerType::getUnqual(C&: Context),
11352 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
11353 CLI.Ins.push_back(Elt: Ret);
11354 }
11355 }
11356 }
11357
11358 // Handle all of the outgoing arguments.
11359 CLI.Outs.clear();
11360 CLI.OutVals.clear();
11361 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11362 SmallVector<Type *, 4> OrigArgTys;
11363 ComputeValueTypes(DL, Ty: Args[i].OrigTy, Types&: OrigArgTys);
11364 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11365 Type *FinalType = Args[i].Ty;
11366 if (Args[i].IsByVal)
11367 FinalType = Args[i].IndirectType;
11368 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11369 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11370 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
11371 ++Value) {
11372 Type *OrigArgTy = OrigArgTys[Value];
11373 Type *ArgTy = OrigArgTy;
11374 if (Args[i].Ty != Args[i].OrigTy) {
11375 assert(Value == 0 && "Only supported for non-aggregate arguments");
11376 ArgTy = Args[i].Ty;
11377 }
11378
11379 EVT VT = getValueType(DL, Ty: ArgTy);
11380 SDValue Op = SDValue(Args[i].Node.getNode(),
11381 Args[i].Node.getResNo() + Value);
11382 ISD::ArgFlagsTy Flags;
11383
11384 // Certain targets (such as MIPS), may have a different ABI alignment
11385 // for a type depending on the context. Give the target a chance to
11386 // specify the alignment it wants.
11387 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11388 Flags.setOrigAlign(OriginalAlignment);
11389
11390 if (i >= CLI.NumFixedArgs)
11391 Flags.setVarArg();
11392 if (ArgTy->isPointerTy()) {
11393 Flags.setPointer();
11394 Flags.setPointerAddrSpace(cast<PointerType>(Val: ArgTy)->getAddressSpace());
11395 }
11396 if (Args[i].IsZExt)
11397 Flags.setZExt();
11398 if (Args[i].IsSExt)
11399 Flags.setSExt();
11400 if (Args[i].IsNoExt)
11401 Flags.setNoExt();
11402 if (Args[i].IsInReg) {
11403 // If we are using vectorcall calling convention, a structure that is
11404 // passed InReg - is surely an HVA
11405 if (CLI.CallConv == CallingConv::X86_VectorCall &&
11406 isa<StructType>(Val: FinalType)) {
11407 // The first value of a structure is marked
11408 if (0 == Value)
11409 Flags.setHvaStart();
11410 Flags.setHva();
11411 }
11412 // Set InReg Flag
11413 Flags.setInReg();
11414 }
11415 if (Args[i].IsSRet)
11416 Flags.setSRet();
11417 if (Args[i].IsSwiftSelf)
11418 Flags.setSwiftSelf();
11419 if (Args[i].IsSwiftAsync)
11420 Flags.setSwiftAsync();
11421 if (Args[i].IsSwiftError)
11422 Flags.setSwiftError();
11423 if (Args[i].IsCFGuardTarget)
11424 Flags.setCFGuardTarget();
11425 if (Args[i].IsByVal)
11426 Flags.setByVal();
11427 if (Args[i].IsByRef)
11428 Flags.setByRef();
11429 if (Args[i].IsPreallocated) {
11430 Flags.setPreallocated();
11431 // Set the byval flag for CCAssignFn callbacks that don't know about
11432 // preallocated. This way we can know how many bytes we should've
11433 // allocated and how many bytes a callee cleanup function will pop. If
11434 // we port preallocated to more targets, we'll have to add custom
11435 // preallocated handling in the various CC lowering callbacks.
11436 Flags.setByVal();
11437 }
11438 if (Args[i].IsInAlloca) {
11439 Flags.setInAlloca();
11440 // Set the byval flag for CCAssignFn callbacks that don't know about
11441 // inalloca. This way we can know how many bytes we should've allocated
11442 // and how many bytes a callee cleanup function will pop. If we port
11443 // inalloca to more targets, we'll have to add custom inalloca handling
11444 // in the various CC lowering callbacks.
11445 Flags.setByVal();
11446 }
11447 Align MemAlign;
11448 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11449 unsigned FrameSize = DL.getTypeAllocSize(Ty: Args[i].IndirectType);
11450 Flags.setByValSize(FrameSize);
11451
11452 // info is not there but there are cases it cannot get right.
11453 if (auto MA = Args[i].Alignment)
11454 MemAlign = *MA;
11455 else
11456 MemAlign = getByValTypeAlignment(Ty: Args[i].IndirectType, DL);
11457 } else if (auto MA = Args[i].Alignment) {
11458 MemAlign = *MA;
11459 } else {
11460 MemAlign = OriginalAlignment;
11461 }
11462 Flags.setMemAlign(MemAlign);
11463 if (Args[i].IsNest)
11464 Flags.setNest();
11465 if (NeedsRegBlock)
11466 Flags.setInConsecutiveRegs();
11467
11468 MVT PartVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11469 unsigned NumParts =
11470 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11471 SmallVector<SDValue, 4> Parts(NumParts);
11472 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
11473
11474 if (Args[i].IsSExt)
11475 ExtendKind = ISD::SIGN_EXTEND;
11476 else if (Args[i].IsZExt)
11477 ExtendKind = ISD::ZERO_EXTEND;
11478
11479 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11480 // for now.
11481 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11482 CanLowerReturn) {
11483 assert((CLI.RetTy == Args[i].Ty ||
11484 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11485 CLI.RetTy->getPointerAddressSpace() ==
11486 Args[i].Ty->getPointerAddressSpace())) &&
11487 RetVTs.size() == NumValues && "unexpected use of 'returned'");
11488 // Before passing 'returned' to the target lowering code, ensure that
11489 // either the register MVT and the actual EVT are the same size or that
11490 // the return value and argument are extended in the same way; in these
11491 // cases it's safe to pass the argument register value unchanged as the
11492 // return register value (although it's at the target's option whether
11493 // to do so)
11494 // TODO: allow code generation to take advantage of partially preserved
11495 // registers rather than clobbering the entire register when the
11496 // parameter extension method is not compatible with the return
11497 // extension method
11498 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11499 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11500 CLI.RetZExt == Args[i].IsZExt))
11501 Flags.setReturned();
11502 }
11503
11504 getCopyToParts(DAG&: CLI.DAG, DL: CLI.DL, Val: Op, Parts: &Parts[0], NumParts, PartVT, V: CLI.CB,
11505 CallConv: CLI.CallConv, ExtendKind);
11506
11507 for (unsigned j = 0; j != NumParts; ++j) {
11508 // if it isn't first piece, alignment must be 1
11509 // For scalable vectors the scalable part is currently handled
11510 // by individual targets, so we just use the known minimum size here.
11511 ISD::OutputArg MyFlags(
11512 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
11513 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11514 if (NumParts > 1 && j == 0)
11515 MyFlags.Flags.setSplit();
11516 else if (j != 0) {
11517 MyFlags.Flags.setOrigAlign(Align(1));
11518 if (j == NumParts - 1)
11519 MyFlags.Flags.setSplitEnd();
11520 }
11521
11522 CLI.Outs.push_back(Elt: MyFlags);
11523 CLI.OutVals.push_back(Elt: Parts[j]);
11524 }
11525
11526 if (NeedsRegBlock && Value == NumValues - 1)
11527 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11528 }
11529 }
11530
11531 SmallVector<SDValue, 4> InVals;
11532 CLI.Chain = LowerCall(CLI, InVals);
11533
11534 // Update CLI.InVals to use outside of this function.
11535 CLI.InVals = InVals;
11536
11537 // Verify that the target's LowerCall behaved as expected.
11538 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11539 "LowerCall didn't return a valid chain!");
11540 assert((!CLI.IsTailCall || InVals.empty()) &&
11541 "LowerCall emitted a return value for a tail call!");
11542 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11543 "LowerCall didn't emit the correct number of values!");
11544
11545 // For a tail call, the return value is merely live-out and there aren't
11546 // any nodes in the DAG representing it. Return a special value to
11547 // indicate that a tail call has been emitted and no more Instructions
11548 // should be processed in the current block.
11549 if (CLI.IsTailCall) {
11550 CLI.DAG.setRoot(CLI.Chain);
11551 return std::make_pair(x: SDValue(), y: SDValue());
11552 }
11553
11554#ifndef NDEBUG
11555 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11556 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11557 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11558 "LowerCall emitted a value with the wrong type!");
11559 }
11560#endif
11561
11562 SmallVector<SDValue, 4> ReturnValues;
11563 if (!CanLowerReturn) {
11564 // The instruction result is the result of loading from the
11565 // hidden sret parameter.
11566 MVT PtrVT = getPointerTy(DL, AS: DL.getAllocaAddrSpace());
11567
11568 unsigned NumValues = RetVTs.size();
11569 ReturnValues.resize(N: NumValues);
11570 SmallVector<SDValue, 4> Chains(NumValues);
11571
11572 // An aggregate return value cannot wrap around the address space, so
11573 // offsets to its parts don't wrap either.
11574 MachineFunction &MF = CLI.DAG.getMachineFunction();
11575 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(ObjectIdx: DemoteStackIdx);
11576 for (unsigned i = 0; i < NumValues; ++i) {
11577 SDValue Add = CLI.DAG.getMemBasePlusOffset(
11578 Base: DemoteStackSlot, Offset: CLI.DAG.getConstant(Val: Offsets[i], DL: CLI.DL, VT: PtrVT),
11579 DL: CLI.DL, Flags: SDNodeFlags::NoUnsignedWrap);
11580 SDValue L = CLI.DAG.getLoad(
11581 VT: RetVTs[i], dl: CLI.DL, Chain: CLI.Chain, Ptr: Add,
11582 PtrInfo: MachinePointerInfo::getFixedStack(MF&: CLI.DAG.getMachineFunction(),
11583 FI: DemoteStackIdx, Offset: Offsets[i]),
11584 Alignment: HiddenSRetAlign);
11585 ReturnValues[i] = L;
11586 Chains[i] = L.getValue(R: 1);
11587 }
11588
11589 CLI.Chain = CLI.DAG.getNode(Opcode: ISD::TokenFactor, DL: CLI.DL, VT: MVT::Other, Ops: Chains);
11590 } else {
11591 // Collect the legal value parts into potentially illegal values
11592 // that correspond to the original function's return values.
11593 std::optional<ISD::NodeType> AssertOp;
11594 if (CLI.RetSExt)
11595 AssertOp = ISD::AssertSext;
11596 else if (CLI.RetZExt)
11597 AssertOp = ISD::AssertZext;
11598 unsigned CurReg = 0;
11599 for (EVT VT : RetVTs) {
11600 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11601 unsigned NumRegs =
11602 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11603
11604 ReturnValues.push_back(Elt: getCopyFromParts(
11605 DAG&: CLI.DAG, DL: CLI.DL, Parts: &InVals[CurReg], NumParts: NumRegs, PartVT: RegisterVT, ValueVT: VT, V: nullptr,
11606 InChain: CLI.Chain, CC: CLI.CallConv, AssertOp));
11607 CurReg += NumRegs;
11608 }
11609
11610 // For a function returning void, there is no return value. We can't create
11611 // such a node, so we just return a null return value in that case. In
11612 // that case, nothing will actually look at the value.
11613 if (ReturnValues.empty())
11614 return std::make_pair(x: SDValue(), y&: CLI.Chain);
11615 }
11616
11617 SDValue Res = CLI.DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: CLI.DL,
11618 VTList: CLI.DAG.getVTList(VTs: RetVTs), Ops: ReturnValues);
11619 return std::make_pair(x&: Res, y&: CLI.Chain);
11620}
11621
11622/// Places new result values for the node in Results (their number
11623/// and types must exactly match those of the original return values of
11624/// the node), or leaves Results empty, which indicates that the node is not
11625/// to be custom lowered after all.
11626void TargetLowering::LowerOperationWrapper(SDNode *N,
11627 SmallVectorImpl<SDValue> &Results,
11628 SelectionDAG &DAG) const {
11629 SDValue Res = LowerOperation(Op: SDValue(N, 0), DAG);
11630
11631 if (!Res.getNode())
11632 return;
11633
11634 // If the original node has one result, take the return value from
11635 // LowerOperation as is. It might not be result number 0.
11636 if (N->getNumValues() == 1) {
11637 Results.push_back(Elt: Res);
11638 return;
11639 }
11640
11641 // If the original node has multiple results, then the return node should
11642 // have the same number of results.
11643 assert((N->getNumValues() == Res->getNumValues()) &&
11644 "Lowering returned the wrong number of results!");
11645
11646 // Places new result values base on N result number.
11647 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11648 Results.push_back(Elt: Res.getValue(R: I));
11649}
11650
11651SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
11652 llvm_unreachable("LowerOperation not implemented for this target!");
11653}
11654
11655void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
11656 Register Reg,
11657 ISD::NodeType ExtendType) {
11658 SDValue Op = getNonRegisterValue(V);
11659 assert((Op.getOpcode() != ISD::CopyFromReg ||
11660 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11661 "Copy from a reg to the same reg!");
11662 assert(!Reg.isPhysical() && "Is a physreg");
11663
11664 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11665 // If this is an InlineAsm we have to match the registers required, not the
11666 // notional registers required by the type.
11667
11668 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11669 std::nullopt); // This is not an ABI copy.
11670 SDValue Chain = DAG.getEntryNode();
11671
11672 if (ExtendType == ISD::ANY_EXTEND) {
11673 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(Val: V);
11674 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11675 ExtendType = PreferredExtendIt->second;
11676 }
11677 RFV.getCopyToRegs(Val: Op, DAG, dl: getCurSDLoc(), Chain, Glue: nullptr, V, PreferredExtendType: ExtendType);
11678 PendingExports.push_back(Elt: Chain);
11679}
11680
11681#include "llvm/CodeGen/SelectionDAGISel.h"
11682
11683/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11684/// entry block, return true. This includes arguments used by switches, since
11685/// the switch may expand into multiple basic blocks.
11686static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11687 // With FastISel active, we may be splitting blocks, so force creation
11688 // of virtual registers for all non-dead arguments.
11689 if (FastISel)
11690 return A->use_empty();
11691
11692 const BasicBlock &Entry = A->getParent()->front();
11693 for (const User *U : A->users())
11694 if (cast<Instruction>(Val: U)->getParent() != &Entry || isa<SwitchInst>(Val: U))
11695 return false; // Use not in entry block.
11696
11697 return true;
11698}
11699
11700using ArgCopyElisionMapTy =
11701 DenseMap<const Argument *,
11702 std::pair<const AllocaInst *, const StoreInst *>>;
11703
11704/// Scan the entry block of the function in FuncInfo for arguments that look
11705/// like copies into a local alloca. Record any copied arguments in
11706/// ArgCopyElisionCandidates.
11707static void
11708findArgumentCopyElisionCandidates(const DataLayout &DL,
11709 FunctionLoweringInfo *FuncInfo,
11710 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
11711 // Record the state of every static alloca used in the entry block. Argument
11712 // allocas are all used in the entry block, so we need approximately as many
11713 // entries as we have arguments.
11714 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
11715 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
11716 unsigned NumArgs = FuncInfo->Fn->arg_size();
11717 StaticAllocas.reserve(NumEntries: NumArgs * 2);
11718
11719 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
11720 if (!V)
11721 return nullptr;
11722 V = V->stripPointerCasts();
11723 const auto *AI = dyn_cast<AllocaInst>(Val: V);
11724 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(Val: AI))
11725 return nullptr;
11726 auto Iter = StaticAllocas.insert(KV: {AI, Unknown});
11727 return &Iter.first->second;
11728 };
11729
11730 // Look for stores of arguments to static allocas. Look through bitcasts and
11731 // GEPs to handle type coercions, as long as the alloca is fully initialized
11732 // by the store. Any non-store use of an alloca escapes it and any subsequent
11733 // unanalyzed store might write it.
11734 // FIXME: Handle structs initialized with multiple stores.
11735 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11736 // Look for stores, and handle non-store uses conservatively.
11737 const auto *SI = dyn_cast<StoreInst>(Val: &I);
11738 if (!SI) {
11739 // We will look through cast uses, so ignore them completely.
11740 if (I.isCast())
11741 continue;
11742 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11743 // to allocas.
11744 if (I.isDebugOrPseudoInst())
11745 continue;
11746 // This is an unknown instruction. Assume it escapes or writes to all
11747 // static alloca operands.
11748 for (const Use &U : I.operands()) {
11749 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11750 *Info = StaticAllocaInfo::Clobbered;
11751 }
11752 continue;
11753 }
11754
11755 // If the stored value is a static alloca, mark it as escaped.
11756 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11757 *Info = StaticAllocaInfo::Clobbered;
11758
11759 // Check if the destination is a static alloca.
11760 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11761 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11762 if (!Info)
11763 continue;
11764 const AllocaInst *AI = cast<AllocaInst>(Val: Dst);
11765
11766 // Skip allocas that have been initialized or clobbered.
11767 if (*Info != StaticAllocaInfo::Unknown)
11768 continue;
11769
11770 // Check if the stored value is an argument, and that this store fully
11771 // initializes the alloca.
11772 // If the argument type has padding bits we can't directly forward a pointer
11773 // as the upper bits may contain garbage.
11774 // Don't elide copies from the same argument twice.
11775 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11776 const auto *Arg = dyn_cast<Argument>(Val);
11777 std::optional<TypeSize> AllocaSize = AI->getAllocationSize(DL);
11778 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11779 Arg->getType()->isEmptyTy() || !AllocaSize ||
11780 DL.getTypeStoreSize(Ty: Arg->getType()) != *AllocaSize ||
11781 !DL.typeSizeEqualsStoreSize(Ty: Arg->getType()) ||
11782 ArgCopyElisionCandidates.count(Val: Arg)) {
11783 *Info = StaticAllocaInfo::Clobbered;
11784 continue;
11785 }
11786
11787 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11788 << '\n');
11789
11790 // Mark this alloca and store for argument copy elision.
11791 *Info = StaticAllocaInfo::Elidable;
11792 ArgCopyElisionCandidates.insert(KV: {Arg, {AI, SI}});
11793
11794 // Stop scanning if we've seen all arguments. This will happen early in -O0
11795 // builds, which is useful, because -O0 builds have large entry blocks and
11796 // many allocas.
11797 if (ArgCopyElisionCandidates.size() == NumArgs)
11798 break;
11799 }
11800}
11801
11802/// Try to elide argument copies from memory into a local alloca. Succeeds if
11803/// ArgVal is a load from a suitable fixed stack object.
11804static void tryToElideArgumentCopy(
11805 FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
11806 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11807 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11808 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11809 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11810 // Check if this is a load from a fixed stack object.
11811 auto *LNode = dyn_cast<LoadSDNode>(Val: ArgVals[0]);
11812 if (!LNode)
11813 return;
11814 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode());
11815 if (!FINode)
11816 return;
11817
11818 // Check that the fixed stack object is the right size and alignment.
11819 // Look at the alignment that the user wrote on the alloca instead of looking
11820 // at the stack object.
11821 auto ArgCopyIter = ArgCopyElisionCandidates.find(Val: &Arg);
11822 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11823 const AllocaInst *AI = ArgCopyIter->second.first;
11824 int FixedIndex = FINode->getIndex();
11825 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11826 int OldIndex = AllocaIndex;
11827 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11828 if (MFI.getObjectSize(ObjectIdx: FixedIndex) != MFI.getObjectSize(ObjectIdx: OldIndex)) {
11829 LLVM_DEBUG(
11830 dbgs() << " argument copy elision failed due to bad fixed stack "
11831 "object size\n");
11832 return;
11833 }
11834 Align RequiredAlignment = AI->getAlign();
11835 if (MFI.getObjectAlign(ObjectIdx: FixedIndex) < RequiredAlignment) {
11836 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11837 "greater than stack argument alignment ("
11838 << DebugStr(RequiredAlignment) << " vs "
11839 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11840 return;
11841 }
11842
11843 // Perform the elision. Delete the old stack object and replace its only use
11844 // in the variable info map. Mark the stack object as mutable and aliased.
11845 LLVM_DEBUG({
11846 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11847 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11848 << '\n';
11849 });
11850 MFI.RemoveStackObject(ObjectIdx: OldIndex);
11851 MFI.setIsImmutableObjectIndex(ObjectIdx: FixedIndex, IsImmutable: false);
11852 MFI.setIsAliasedObjectIndex(ObjectIdx: FixedIndex, IsAliased: true);
11853 AllocaIndex = FixedIndex;
11854 ArgCopyElisionFrameIndexMap.insert(KV: {OldIndex, FixedIndex});
11855 for (SDValue ArgVal : ArgVals)
11856 Chains.push_back(Elt: ArgVal.getValue(R: 1));
11857
11858 // Avoid emitting code for the store implementing the copy.
11859 const StoreInst *SI = ArgCopyIter->second.second;
11860 ElidedArgCopyInstrs.insert(Ptr: SI);
11861
11862 // Check for uses of the argument again so that we can avoid exporting ArgVal
11863 // if it is't used by anything other than the store.
11864 for (const Value *U : Arg.users()) {
11865 if (U != SI) {
11866 ArgHasUses = true;
11867 break;
11868 }
11869 }
11870}
11871
11872void SelectionDAGISel::LowerArguments(const Function &F) {
11873 SelectionDAG &DAG = SDB->DAG;
11874 SDLoc dl = SDB->getCurSDLoc();
11875 const DataLayout &DL = DAG.getDataLayout();
11876 SmallVector<ISD::InputArg, 16> Ins;
11877
11878 // In Naked functions we aren't going to save any registers.
11879 if (F.hasFnAttribute(Kind: Attribute::Naked))
11880 return;
11881
11882 if (!FuncInfo->CanLowerReturn) {
11883 // Put in an sret pointer parameter before all the other parameters.
11884 MVT ValueVT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
11885
11886 ISD::ArgFlagsTy Flags;
11887 Flags.setSRet();
11888 MVT RegisterVT = TLI->getRegisterType(Context&: *DAG.getContext(), VT: ValueVT);
11889 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
11890 ISD::InputArg::NoArgIndex, 0);
11891 Ins.push_back(Elt: RetArg);
11892 }
11893
11894 // Look for stores of arguments to static allocas. Mark such arguments with a
11895 // flag to ask the target to give us the memory location of that argument if
11896 // available.
11897 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11898 findArgumentCopyElisionCandidates(DL, FuncInfo: FuncInfo.get(),
11899 ArgCopyElisionCandidates);
11900
11901 // Set up the incoming argument description vector.
11902 for (const Argument &Arg : F.args()) {
11903 unsigned ArgNo = Arg.getArgNo();
11904 SmallVector<Type *, 4> Types;
11905 ComputeValueTypes(DL: DAG.getDataLayout(), Ty: Arg.getType(), Types);
11906 bool isArgValueUsed = !Arg.use_empty();
11907 Type *FinalType = Arg.getType();
11908 if (Arg.hasAttribute(Kind: Attribute::ByVal))
11909 FinalType = Arg.getParamByValType();
11910 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11911 Ty: FinalType, CallConv: F.getCallingConv(), isVarArg: F.isVarArg(), DL);
11912 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
11913 ++Value) {
11914 Type *ArgTy = Types[Value];
11915 EVT VT = TLI->getValueType(DL, Ty: ArgTy);
11916 ISD::ArgFlagsTy Flags;
11917
11918 if (ArgTy->isPointerTy()) {
11919 Flags.setPointer();
11920 Flags.setPointerAddrSpace(cast<PointerType>(Val: ArgTy)->getAddressSpace());
11921 }
11922 if (Arg.hasAttribute(Kind: Attribute::ZExt))
11923 Flags.setZExt();
11924 if (Arg.hasAttribute(Kind: Attribute::SExt))
11925 Flags.setSExt();
11926 if (Arg.hasAttribute(Kind: Attribute::InReg)) {
11927 // If we are using vectorcall calling convention, a structure that is
11928 // passed InReg - is surely an HVA
11929 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11930 isa<StructType>(Val: Arg.getType())) {
11931 // The first value of a structure is marked
11932 if (0 == Value)
11933 Flags.setHvaStart();
11934 Flags.setHva();
11935 }
11936 // Set InReg Flag
11937 Flags.setInReg();
11938 }
11939 if (Arg.hasAttribute(Kind: Attribute::StructRet))
11940 Flags.setSRet();
11941 if (Arg.hasAttribute(Kind: Attribute::SwiftSelf))
11942 Flags.setSwiftSelf();
11943 if (Arg.hasAttribute(Kind: Attribute::SwiftAsync))
11944 Flags.setSwiftAsync();
11945 if (Arg.hasAttribute(Kind: Attribute::SwiftError))
11946 Flags.setSwiftError();
11947 if (Arg.hasAttribute(Kind: Attribute::ByVal))
11948 Flags.setByVal();
11949 if (Arg.hasAttribute(Kind: Attribute::ByRef))
11950 Flags.setByRef();
11951 if (Arg.hasAttribute(Kind: Attribute::InAlloca)) {
11952 Flags.setInAlloca();
11953 // Set the byval flag for CCAssignFn callbacks that don't know about
11954 // inalloca. This way we can know how many bytes we should've allocated
11955 // and how many bytes a callee cleanup function will pop. If we port
11956 // inalloca to more targets, we'll have to add custom inalloca handling
11957 // in the various CC lowering callbacks.
11958 Flags.setByVal();
11959 }
11960 if (Arg.hasAttribute(Kind: Attribute::Preallocated)) {
11961 Flags.setPreallocated();
11962 // Set the byval flag for CCAssignFn callbacks that don't know about
11963 // preallocated. This way we can know how many bytes we should've
11964 // allocated and how many bytes a callee cleanup function will pop. If
11965 // we port preallocated to more targets, we'll have to add custom
11966 // preallocated handling in the various CC lowering callbacks.
11967 Flags.setByVal();
11968 }
11969
11970 // Certain targets (such as MIPS), may have a different ABI alignment
11971 // for a type depending on the context. Give the target a chance to
11972 // specify the alignment it wants.
11973 const Align OriginalAlignment(
11974 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
11975 Flags.setOrigAlign(OriginalAlignment);
11976
11977 Align MemAlign;
11978 Type *ArgMemTy = nullptr;
11979 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11980 Flags.isByRef()) {
11981 if (!ArgMemTy)
11982 ArgMemTy = Arg.getPointeeInMemoryValueType();
11983
11984 uint64_t MemSize = DL.getTypeAllocSize(Ty: ArgMemTy);
11985
11986 // For in-memory arguments, size and alignment should be passed from FE.
11987 // BE will guess if this info is not there but there are cases it cannot
11988 // get right.
11989 if (auto ParamAlign = Arg.getParamStackAlign())
11990 MemAlign = *ParamAlign;
11991 else if ((ParamAlign = Arg.getParamAlign()))
11992 MemAlign = *ParamAlign;
11993 else
11994 MemAlign = TLI->getByValTypeAlignment(Ty: ArgMemTy, DL);
11995 if (Flags.isByRef())
11996 Flags.setByRefSize(MemSize);
11997 else
11998 Flags.setByValSize(MemSize);
11999 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
12000 MemAlign = *ParamAlign;
12001 } else {
12002 MemAlign = OriginalAlignment;
12003 }
12004 Flags.setMemAlign(MemAlign);
12005
12006 if (Arg.hasAttribute(Kind: Attribute::Nest))
12007 Flags.setNest();
12008 if (NeedsRegBlock)
12009 Flags.setInConsecutiveRegs();
12010 if (ArgCopyElisionCandidates.count(Val: &Arg))
12011 Flags.setCopyElisionCandidate();
12012 if (Arg.hasAttribute(Kind: Attribute::Returned))
12013 Flags.setReturned();
12014
12015 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
12016 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12017 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
12018 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12019 for (unsigned i = 0; i != NumRegs; ++i) {
12020 // For scalable vectors, use the minimum size; individual targets
12021 // are responsible for handling scalable vector arguments and
12022 // return values.
12023 ISD::InputArg MyFlags(
12024 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12025 i * RegisterVT.getStoreSize().getKnownMinValue());
12026 if (NumRegs > 1 && i == 0)
12027 MyFlags.Flags.setSplit();
12028 // if it isn't first piece, alignment must be 1
12029 else if (i > 0) {
12030 MyFlags.Flags.setOrigAlign(Align(1));
12031 if (i == NumRegs - 1)
12032 MyFlags.Flags.setSplitEnd();
12033 }
12034 Ins.push_back(Elt: MyFlags);
12035 }
12036 if (NeedsRegBlock && Value == NumValues - 1)
12037 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
12038 }
12039 }
12040
12041 // Call the target to set up the argument values.
12042 SmallVector<SDValue, 8> InVals;
12043 SDValue NewRoot = TLI->LowerFormalArguments(
12044 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
12045
12046 // Verify that the target's LowerFormalArguments behaved as expected.
12047 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
12048 "LowerFormalArguments didn't return a valid chain!");
12049 assert(InVals.size() == Ins.size() &&
12050 "LowerFormalArguments didn't emit the correct number of values!");
12051 LLVM_DEBUG({
12052 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
12053 assert(InVals[i].getNode() &&
12054 "LowerFormalArguments emitted a null value!");
12055 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
12056 "LowerFormalArguments emitted a value with the wrong type!");
12057 }
12058 });
12059
12060 // Update the DAG with the new chain value resulting from argument lowering.
12061 DAG.setRoot(NewRoot);
12062
12063 // Set up the argument values.
12064 unsigned i = 0;
12065 if (!FuncInfo->CanLowerReturn) {
12066 // Create a virtual register for the sret pointer, and put in a copy
12067 // from the sret argument into it.
12068 MVT VT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
12069 MVT RegVT = TLI->getRegisterType(Context&: *CurDAG->getContext(), VT);
12070 std::optional<ISD::NodeType> AssertOp;
12071 SDValue ArgValue =
12072 getCopyFromParts(DAG, DL: dl, Parts: &InVals[0], NumParts: 1, PartVT: RegVT, ValueVT: VT, V: nullptr, InChain: NewRoot,
12073 CC: F.getCallingConv(), AssertOp);
12074
12075 MachineFunction& MF = SDB->DAG.getMachineFunction();
12076 MachineRegisterInfo& RegInfo = MF.getRegInfo();
12077 Register SRetReg =
12078 RegInfo.createVirtualRegister(RegClass: TLI->getRegClassFor(VT: RegVT));
12079 FuncInfo->DemoteRegister = SRetReg;
12080 NewRoot =
12081 SDB->DAG.getCopyToReg(Chain: NewRoot, dl: SDB->getCurSDLoc(), Reg: SRetReg, N: ArgValue);
12082 DAG.setRoot(NewRoot);
12083
12084 // i indexes lowered arguments. Bump it past the hidden sret argument.
12085 ++i;
12086 }
12087
12088 SmallVector<SDValue, 4> Chains;
12089 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12090 for (const Argument &Arg : F.args()) {
12091 SmallVector<SDValue, 4> ArgValues;
12092 SmallVector<EVT, 4> ValueVTs;
12093 ComputeValueVTs(TLI: *TLI, DL: DAG.getDataLayout(), Ty: Arg.getType(), ValueVTs);
12094 unsigned NumValues = ValueVTs.size();
12095 if (NumValues == 0)
12096 continue;
12097
12098 bool ArgHasUses = !Arg.use_empty();
12099
12100 // Elide the copying store if the target loaded this argument from a
12101 // suitable fixed stack object.
12102 if (Ins[i].Flags.isCopyElisionCandidate()) {
12103 unsigned NumParts = 0;
12104 for (EVT VT : ValueVTs)
12105 NumParts += TLI->getNumRegistersForCallingConv(Context&: *CurDAG->getContext(),
12106 CC: F.getCallingConv(), VT);
12107
12108 tryToElideArgumentCopy(FuncInfo&: *FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
12109 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
12110 ArgVals: ArrayRef(&InVals[i], NumParts), ArgHasUses);
12111 }
12112
12113 // If this argument is unused then remember its value. It is used to generate
12114 // debugging information.
12115 bool isSwiftErrorArg =
12116 TLI->supportSwiftError() &&
12117 Arg.hasAttribute(Kind: Attribute::SwiftError);
12118 if (!ArgHasUses && !isSwiftErrorArg) {
12119 SDB->setUnusedArgValue(V: &Arg, NewN: InVals[i]);
12120
12121 // Also remember any frame index for use in FastISel.
12122 if (FrameIndexSDNode *FI =
12123 dyn_cast<FrameIndexSDNode>(Val: InVals[i].getNode()))
12124 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12125 }
12126
12127 for (unsigned Val = 0; Val != NumValues; ++Val) {
12128 EVT VT = ValueVTs[Val];
12129 MVT PartVT = TLI->getRegisterTypeForCallingConv(Context&: *CurDAG->getContext(),
12130 CC: F.getCallingConv(), VT);
12131 unsigned NumParts = TLI->getNumRegistersForCallingConv(
12132 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12133
12134 // Even an apparent 'unused' swifterror argument needs to be returned. So
12135 // we do generate a copy for it that can be used on return from the
12136 // function.
12137 if (ArgHasUses || isSwiftErrorArg) {
12138 std::optional<ISD::NodeType> AssertOp;
12139 if (Arg.hasAttribute(Kind: Attribute::SExt))
12140 AssertOp = ISD::AssertSext;
12141 else if (Arg.hasAttribute(Kind: Attribute::ZExt))
12142 AssertOp = ISD::AssertZext;
12143
12144 SDValue OutVal =
12145 getCopyFromParts(DAG, DL: dl, Parts: &InVals[i], NumParts, PartVT, ValueVT: VT, V: nullptr,
12146 InChain: NewRoot, CC: F.getCallingConv(), AssertOp);
12147
12148 FPClassTest NoFPClass = Arg.getNoFPClass();
12149 if (NoFPClass != fcNone) {
12150 SDValue SDNoFPClass = DAG.getTargetConstant(
12151 Val: static_cast<uint64_t>(NoFPClass), DL: dl, VT: MVT::i32);
12152 OutVal = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: OutVal.getValueType(),
12153 N1: OutVal, N2: SDNoFPClass);
12154 }
12155 ArgValues.push_back(Elt: OutVal);
12156 }
12157
12158 i += NumParts;
12159 }
12160
12161 // We don't need to do anything else for unused arguments.
12162 if (ArgValues.empty())
12163 continue;
12164
12165 // Note down frame index.
12166 if (FrameIndexSDNode *FI =
12167 dyn_cast<FrameIndexSDNode>(Val: ArgValues[0].getNode()))
12168 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12169
12170 SDValue Res = DAG.getMergeValues(Ops: ArrayRef(ArgValues.data(), NumValues),
12171 dl: SDB->getCurSDLoc());
12172
12173 SDB->setValue(V: &Arg, NewN: Res);
12174 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
12175 // We want to associate the argument with the frame index, among
12176 // involved operands, that correspond to the lowest address. The
12177 // getCopyFromParts function, called earlier, is swapping the order of
12178 // the operands to BUILD_PAIR depending on endianness. The result of
12179 // that swapping is that the least significant bits of the argument will
12180 // be in the first operand of the BUILD_PAIR node, and the most
12181 // significant bits will be in the second operand.
12182 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
12183 if (LoadSDNode *LNode =
12184 dyn_cast<LoadSDNode>(Val: Res.getOperand(i: LowAddressOp).getNode()))
12185 if (FrameIndexSDNode *FI =
12186 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
12187 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12188 }
12189
12190 // Analyses past this point are naive and don't expect an assertion.
12191 if (Res.getOpcode() == ISD::AssertZext)
12192 Res = Res.getOperand(i: 0);
12193
12194 // Update the SwiftErrorVRegDefMap.
12195 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
12196 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
12197 if (Reg.isVirtual())
12198 SwiftError->setCurrentVReg(MBB: FuncInfo->MBB, SwiftError->getFunctionArg(),
12199 Reg);
12200 }
12201
12202 // If this argument is live outside of the entry block, insert a copy from
12203 // wherever we got it to the vreg that other BB's will reference it as.
12204 if (Res.getOpcode() == ISD::CopyFromReg) {
12205 // If we can, though, try to skip creating an unnecessary vreg.
12206 // FIXME: This isn't very clean... it would be nice to make this more
12207 // general.
12208 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
12209 if (Reg.isVirtual()) {
12210 FuncInfo->ValueMap[&Arg] = Reg;
12211 continue;
12212 }
12213 }
12214 if (!isOnlyUsedInEntryBlock(A: &Arg, FastISel: TM.Options.EnableFastISel)) {
12215 FuncInfo->InitializeRegForValue(V: &Arg);
12216 SDB->CopyToExportRegsIfNeeded(V: &Arg);
12217 }
12218 }
12219
12220 if (!Chains.empty()) {
12221 Chains.push_back(Elt: NewRoot);
12222 NewRoot = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
12223 }
12224
12225 DAG.setRoot(NewRoot);
12226
12227 assert(i == InVals.size() && "Argument register count mismatch!");
12228
12229 // If any argument copy elisions occurred and we have debug info, update the
12230 // stale frame indices used in the dbg.declare variable info table.
12231 if (!ArgCopyElisionFrameIndexMap.empty()) {
12232 for (MachineFunction::VariableDbgInfo &VI :
12233 MF->getInStackSlotVariableDbgInfo()) {
12234 auto I = ArgCopyElisionFrameIndexMap.find(Val: VI.getStackSlot());
12235 if (I != ArgCopyElisionFrameIndexMap.end())
12236 VI.updateStackSlot(NewSlot: I->second);
12237 }
12238 }
12239
12240 // Finally, if the target has anything special to do, allow it to do so.
12241 emitFunctionEntryCode();
12242}
12243
12244/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
12245/// ensure constants are generated when needed. Remember the virtual registers
12246/// that need to be added to the Machine PHI nodes as input. We cannot just
12247/// directly add them, because expansion might result in multiple MBB's for one
12248/// BB. As such, the start of the BB might correspond to a different MBB than
12249/// the end.
12250void
12251SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
12252 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12253
12254 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12255
12256 // Check PHI nodes in successors that expect a value to be available from this
12257 // block.
12258 for (const BasicBlock *SuccBB : successors(I: LLVMBB->getTerminator())) {
12259 if (!isa<PHINode>(Val: SuccBB->begin())) continue;
12260 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(BB: SuccBB);
12261
12262 // If this terminator has multiple identical successors (common for
12263 // switches), only handle each succ once.
12264 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
12265 continue;
12266
12267 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
12268
12269 // At this point we know that there is a 1-1 correspondence between LLVM PHI
12270 // nodes and Machine PHI nodes, but the incoming operands have not been
12271 // emitted yet.
12272 for (const PHINode &PN : SuccBB->phis()) {
12273 // Ignore dead phi's.
12274 if (PN.use_empty())
12275 continue;
12276
12277 // Skip empty types
12278 if (PN.getType()->isEmptyTy())
12279 continue;
12280
12281 Register Reg;
12282 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
12283
12284 if (const auto *C = dyn_cast<Constant>(Val: PHIOp)) {
12285 Register &RegOut = ConstantsOut[C];
12286 if (!RegOut) {
12287 RegOut = FuncInfo.CreateRegs(V: &PN);
12288 // We need to zero/sign extend ConstantInt phi operands to match
12289 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12290 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12291 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
12292 ExtendType = TLI.signExtendConstant(C: CI) ? ISD::SIGN_EXTEND
12293 : ISD::ZERO_EXTEND;
12294 CopyValueToVirtualRegister(V: C, Reg: RegOut, ExtendType);
12295 }
12296 Reg = RegOut;
12297 } else {
12298 DenseMap<const Value *, Register>::iterator I =
12299 FuncInfo.ValueMap.find(Val: PHIOp);
12300 if (I != FuncInfo.ValueMap.end())
12301 Reg = I->second;
12302 else {
12303 assert(isa<AllocaInst>(PHIOp) &&
12304 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12305 "Didn't codegen value into a register!??");
12306 Reg = FuncInfo.CreateRegs(V: &PN);
12307 CopyValueToVirtualRegister(V: PHIOp, Reg);
12308 }
12309 }
12310
12311 // Remember that this register needs to added to the machine PHI node as
12312 // the input for this MBB.
12313 SmallVector<EVT, 4> ValueVTs;
12314 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: PN.getType(), ValueVTs);
12315 for (EVT VT : ValueVTs) {
12316 const unsigned NumRegisters = TLI.getNumRegisters(Context&: *DAG.getContext(), VT);
12317 for (unsigned i = 0; i != NumRegisters; ++i)
12318 FuncInfo.PHINodesToUpdate.emplace_back(args: &*MBBI++, args: Reg + i);
12319 Reg += NumRegisters;
12320 }
12321 }
12322 }
12323
12324 ConstantsOut.clear();
12325}
12326
12327MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12328 MachineFunction::iterator I(MBB);
12329 if (++I == FuncInfo.MF->end())
12330 return nullptr;
12331 return &*I;
12332}
12333
12334/// During lowering new call nodes can be created (such as memset, etc.).
12335/// Those will become new roots of the current DAG, but complications arise
12336/// when they are tail calls. In such cases, the call lowering will update
12337/// the root, but the builder still needs to know that a tail call has been
12338/// lowered in order to avoid generating an additional return.
12339void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12340 // If the node is null, we do have a tail call.
12341 if (MaybeTC.getNode() != nullptr)
12342 DAG.setRoot(MaybeTC);
12343 else
12344 HasTailCall = true;
12345}
12346
12347void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12348 MachineBasicBlock *SwitchMBB,
12349 MachineBasicBlock *DefaultMBB) {
12350 MachineFunction *CurMF = FuncInfo.MF;
12351 MachineBasicBlock *NextMBB = nullptr;
12352 MachineFunction::iterator BBI(W.MBB);
12353 if (++BBI != FuncInfo.MF->end())
12354 NextMBB = &*BBI;
12355
12356 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12357
12358 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12359
12360 if (Size == 2 && W.MBB == SwitchMBB) {
12361 // If any two of the cases has the same destination, and if one value
12362 // is the same as the other, but has one bit unset that the other has set,
12363 // use bit manipulation to do two compares at once. For example:
12364 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12365 // TODO: This could be extended to merge any 2 cases in switches with 3
12366 // cases.
12367 // TODO: Handle cases where W.CaseBB != SwitchBB.
12368 CaseCluster &Small = *W.FirstCluster;
12369 CaseCluster &Big = *W.LastCluster;
12370
12371 if (Small.Low == Small.High && Big.Low == Big.High &&
12372 Small.MBB == Big.MBB) {
12373 const APInt &SmallValue = Small.Low->getValue();
12374 const APInt &BigValue = Big.Low->getValue();
12375
12376 // Check that there is only one bit different.
12377 APInt CommonBit = BigValue ^ SmallValue;
12378 if (CommonBit.isPowerOf2()) {
12379 SDValue CondLHS = getValue(V: Cond);
12380 EVT VT = CondLHS.getValueType();
12381 SDLoc DL = getCurSDLoc();
12382
12383 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: CondLHS,
12384 N2: DAG.getConstant(Val: CommonBit, DL, VT));
12385 SDValue Cond = DAG.getSetCC(
12386 DL, VT: MVT::i1, LHS: Or, RHS: DAG.getConstant(Val: BigValue | SmallValue, DL, VT),
12387 Cond: ISD::SETEQ);
12388
12389 // Update successor info.
12390 // Both Small and Big will jump to Small.BB, so we sum up the
12391 // probabilities.
12392 addSuccessorWithProb(Src: SwitchMBB, Dst: Small.MBB, Prob: Small.Prob + Big.Prob);
12393 if (BPI)
12394 addSuccessorWithProb(
12395 Src: SwitchMBB, Dst: DefaultMBB,
12396 // The default destination is the first successor in IR.
12397 Prob: BPI->getEdgeProbability(Src: SwitchMBB->getBasicBlock(), IndexInSuccessors: (unsigned)0));
12398 else
12399 addSuccessorWithProb(Src: SwitchMBB, Dst: DefaultMBB);
12400
12401 // Insert the true branch.
12402 SDValue BrCond =
12403 DAG.getNode(Opcode: ISD::BRCOND, DL, VT: MVT::Other, N1: getControlRoot(), N2: Cond,
12404 N3: DAG.getBasicBlock(MBB: Small.MBB));
12405 // Insert the false branch.
12406 BrCond = DAG.getNode(Opcode: ISD::BR, DL, VT: MVT::Other, N1: BrCond,
12407 N2: DAG.getBasicBlock(MBB: DefaultMBB));
12408
12409 DAG.setRoot(BrCond);
12410 return;
12411 }
12412 }
12413 }
12414
12415 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12416 // Here, we order cases by probability so the most likely case will be
12417 // checked first. However, two clusters can have the same probability in
12418 // which case their relative ordering is non-deterministic. So we use Low
12419 // as a tie-breaker as clusters are guaranteed to never overlap.
12420 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
12421 Comp: [](const CaseCluster &a, const CaseCluster &b) {
12422 return a.Prob != b.Prob ?
12423 a.Prob > b.Prob :
12424 a.Low->getValue().slt(RHS: b.Low->getValue());
12425 });
12426
12427 // Rearrange the case blocks so that the last one falls through if possible
12428 // without changing the order of probabilities.
12429 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12430 --I;
12431 if (I->Prob > W.LastCluster->Prob)
12432 break;
12433 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12434 std::swap(a&: *I, b&: *W.LastCluster);
12435 break;
12436 }
12437 }
12438 }
12439
12440 // Compute total probability.
12441 BranchProbability DefaultProb = W.DefaultProb;
12442 BranchProbability UnhandledProbs = DefaultProb;
12443 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12444 UnhandledProbs += I->Prob;
12445
12446 MachineBasicBlock *CurMBB = W.MBB;
12447 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12448 bool FallthroughUnreachable = false;
12449 MachineBasicBlock *Fallthrough;
12450 if (I == W.LastCluster) {
12451 // For the last cluster, fall through to the default destination.
12452 Fallthrough = DefaultMBB;
12453 FallthroughUnreachable = isa<UnreachableInst>(
12454 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
12455 } else {
12456 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
12457 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
12458 // Put Cond in a virtual register to make it available from the new blocks.
12459 ExportFromCurrentBlock(V: Cond);
12460 }
12461 UnhandledProbs -= I->Prob;
12462
12463 switch (I->Kind) {
12464 case CC_JumpTable: {
12465 // FIXME: Optimize away range check based on pivot comparisons.
12466 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12467 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12468
12469 // The jump block hasn't been inserted yet; insert it here.
12470 MachineBasicBlock *JumpMBB = JT->MBB;
12471 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
12472
12473 auto JumpProb = I->Prob;
12474 auto FallthroughProb = UnhandledProbs;
12475
12476 // If the default statement is a target of the jump table, we evenly
12477 // distribute the default probability to successors of CurMBB. Also
12478 // update the probability on the edge from JumpMBB to Fallthrough.
12479 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12480 SE = JumpMBB->succ_end();
12481 SI != SE; ++SI) {
12482 if (*SI == DefaultMBB) {
12483 JumpProb += DefaultProb / 2;
12484 FallthroughProb -= DefaultProb / 2;
12485 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
12486 JumpMBB->normalizeSuccProbs();
12487 break;
12488 }
12489 }
12490
12491 // If the default clause is unreachable, propagate that knowledge into
12492 // JTH->FallthroughUnreachable which will use it to suppress the range
12493 // check.
12494 //
12495 // However, don't do this if we're doing branch target enforcement,
12496 // because a table branch _without_ a range check can be a tempting JOP
12497 // gadget - out-of-bounds inputs that are impossible in correct
12498 // execution become possible again if an attacker can influence the
12499 // control flow. So if an attacker doesn't already have a BTI bypass
12500 // available, we don't want them to be able to get one out of this
12501 // table branch.
12502 if (FallthroughUnreachable) {
12503 Function &CurFunc = CurMF->getFunction();
12504 if (!CurFunc.hasFnAttribute(Kind: "branch-target-enforcement"))
12505 JTH->FallthroughUnreachable = true;
12506 }
12507
12508 if (!JTH->FallthroughUnreachable)
12509 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
12510 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
12511 CurMBB->normalizeSuccProbs();
12512
12513 // The jump table header will be inserted in our current block, do the
12514 // range check, and fall through to our fallthrough block.
12515 JTH->HeaderBB = CurMBB;
12516 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12517
12518 // If we're in the right place, emit the jump table header right now.
12519 if (CurMBB == SwitchMBB) {
12520 visitJumpTableHeader(JT&: *JT, JTH&: *JTH, SwitchBB: SwitchMBB);
12521 JTH->Emitted = true;
12522 }
12523 break;
12524 }
12525 case CC_BitTests: {
12526 // FIXME: Optimize away range check based on pivot comparisons.
12527 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12528
12529 // The bit test blocks haven't been inserted yet; insert them here.
12530 for (BitTestCase &BTC : BTB->Cases)
12531 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
12532
12533 // Fill in fields of the BitTestBlock.
12534 BTB->Parent = CurMBB;
12535 BTB->Default = Fallthrough;
12536
12537 BTB->DefaultProb = UnhandledProbs;
12538 // If the cases in bit test don't form a contiguous range, we evenly
12539 // distribute the probability on the edge to Fallthrough to two
12540 // successors of CurMBB.
12541 if (!BTB->ContiguousRange) {
12542 BTB->Prob += DefaultProb / 2;
12543 BTB->DefaultProb -= DefaultProb / 2;
12544 }
12545
12546 if (FallthroughUnreachable)
12547 BTB->FallthroughUnreachable = true;
12548
12549 // If we're in the right place, emit the bit test header right now.
12550 if (CurMBB == SwitchMBB) {
12551 visitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
12552 BTB->Emitted = true;
12553 }
12554 break;
12555 }
12556 case CC_Range: {
12557 const Value *RHS, *LHS, *MHS;
12558 ISD::CondCode CC;
12559 if (I->Low == I->High) {
12560 // Check Cond == I->Low.
12561 CC = ISD::SETEQ;
12562 LHS = Cond;
12563 RHS=I->Low;
12564 MHS = nullptr;
12565 } else {
12566 // Check I->Low <= Cond <= I->High.
12567 CC = ISD::SETLE;
12568 LHS = I->Low;
12569 MHS = Cond;
12570 RHS = I->High;
12571 }
12572
12573 // If Fallthrough is unreachable, fold away the comparison.
12574 if (FallthroughUnreachable)
12575 CC = ISD::SETTRUE;
12576
12577 // The false probability is the sum of all unhandled cases.
12578 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12579 getCurSDLoc(), I->Prob, UnhandledProbs);
12580
12581 if (CurMBB == SwitchMBB)
12582 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12583 else
12584 SL->SwitchCases.push_back(x: CB);
12585
12586 break;
12587 }
12588 }
12589 CurMBB = Fallthrough;
12590 }
12591}
12592
12593void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12594 const SwitchWorkListItem &W,
12595 Value *Cond,
12596 MachineBasicBlock *SwitchMBB) {
12597 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12598 "Clusters not sorted?");
12599 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12600
12601 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12602 SL->computeSplitWorkItemInfo(W);
12603
12604 // Use the first element on the right as pivot since we will make less-than
12605 // comparisons against it.
12606 CaseClusterIt PivotCluster = FirstRight;
12607 assert(PivotCluster > W.FirstCluster);
12608 assert(PivotCluster <= W.LastCluster);
12609
12610 CaseClusterIt FirstLeft = W.FirstCluster;
12611 CaseClusterIt LastRight = W.LastCluster;
12612
12613 const ConstantInt *Pivot = PivotCluster->Low;
12614
12615 // New blocks will be inserted immediately after the current one.
12616 MachineFunction::iterator BBI(W.MBB);
12617 ++BBI;
12618
12619 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12620 // we can branch to its destination directly if it's squeezed exactly in
12621 // between the known lower bound and Pivot - 1.
12622 MachineBasicBlock *LeftMBB;
12623 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12624 FirstLeft->Low == W.GE &&
12625 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12626 LeftMBB = FirstLeft->MBB;
12627 } else {
12628 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12629 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
12630 WorkList.push_back(
12631 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
12632 // Put Cond in a virtual register to make it available from the new blocks.
12633 ExportFromCurrentBlock(V: Cond);
12634 }
12635
12636 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12637 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12638 // directly if RHS.High equals the current upper bound.
12639 MachineBasicBlock *RightMBB;
12640 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12641 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12642 RightMBB = FirstRight->MBB;
12643 } else {
12644 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12645 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
12646 WorkList.push_back(
12647 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
12648 // Put Cond in a virtual register to make it available from the new blocks.
12649 ExportFromCurrentBlock(V: Cond);
12650 }
12651
12652 // Create the CaseBlock record that will be used to lower the branch.
12653 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12654 getCurSDLoc(), LeftProb, RightProb);
12655
12656 if (W.MBB == SwitchMBB)
12657 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12658 else
12659 SL->SwitchCases.push_back(x: CB);
12660}
12661
12662// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12663// from the swith statement.
12664static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
12665 BranchProbability PeeledCaseProb) {
12666 if (PeeledCaseProb == BranchProbability::getOne())
12667 return BranchProbability::getZero();
12668 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12669
12670 uint32_t Numerator = CaseProb.getNumerator();
12671 uint32_t Denominator = SwitchProb.scale(Num: CaseProb.getDenominator());
12672 return BranchProbability(Numerator, std::max(a: Numerator, b: Denominator));
12673}
12674
12675// Try to peel the top probability case if it exceeds the threshold.
12676// Return current MachineBasicBlock for the switch statement if the peeling
12677// does not occur.
12678// If the peeling is performed, return the newly created MachineBasicBlock
12679// for the peeled switch statement. Also update Clusters to remove the peeled
12680// case. PeeledCaseProb is the BranchProbability for the peeled case.
12681MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12682 const SwitchInst &SI, CaseClusterVector &Clusters,
12683 BranchProbability &PeeledCaseProb) {
12684 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12685 // Don't perform if there is only one cluster or optimizing for size.
12686 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12687 TM.getOptLevel() == CodeGenOptLevel::None ||
12688 SwitchMBB->getParent()->getFunction().hasMinSize())
12689 return SwitchMBB;
12690
12691 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
12692 unsigned PeeledCaseIndex = 0;
12693 bool SwitchPeeled = false;
12694 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12695 CaseCluster &CC = Clusters[Index];
12696 if (CC.Prob < TopCaseProb)
12697 continue;
12698 TopCaseProb = CC.Prob;
12699 PeeledCaseIndex = Index;
12700 SwitchPeeled = true;
12701 }
12702 if (!SwitchPeeled)
12703 return SwitchMBB;
12704
12705 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12706 << TopCaseProb << "\n");
12707
12708 // Record the MBB for the peeled switch statement.
12709 MachineFunction::iterator BBI(SwitchMBB);
12710 ++BBI;
12711 MachineBasicBlock *PeeledSwitchMBB =
12712 FuncInfo.MF->CreateMachineBasicBlock(BB: SwitchMBB->getBasicBlock());
12713 FuncInfo.MF->insert(MBBI: BBI, MBB: PeeledSwitchMBB);
12714
12715 ExportFromCurrentBlock(V: SI.getCondition());
12716 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12717 SwitchWorkListItem W = {.MBB: SwitchMBB, .FirstCluster: PeeledCaseIt, .LastCluster: PeeledCaseIt,
12718 .GE: nullptr, .LT: nullptr, .DefaultProb: TopCaseProb.getCompl()};
12719 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB: PeeledSwitchMBB);
12720
12721 Clusters.erase(position: PeeledCaseIt);
12722 for (CaseCluster &CC : Clusters) {
12723 LLVM_DEBUG(
12724 dbgs() << "Scale the probablity for one cluster, before scaling: "
12725 << CC.Prob << "\n");
12726 CC.Prob = scaleCaseProbality(CaseProb: CC.Prob, PeeledCaseProb: TopCaseProb);
12727 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12728 }
12729 PeeledCaseProb = TopCaseProb;
12730 return PeeledSwitchMBB;
12731}
12732
12733void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12734 // Extract cases from the switch.
12735 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12736 CaseClusterVector Clusters;
12737 Clusters.reserve(n: SI.getNumCases());
12738 for (auto I : SI.cases()) {
12739 MachineBasicBlock *Succ = FuncInfo.getMBB(BB: I.getCaseSuccessor());
12740 const ConstantInt *CaseVal = I.getCaseValue();
12741 BranchProbability Prob =
12742 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
12743 : BranchProbability(1, SI.getNumCases() + 1);
12744 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
12745 }
12746
12747 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(BB: SI.getDefaultDest());
12748
12749 // Cluster adjacent cases with the same destination. We do this at all
12750 // optimization levels because it's cheap to do and will make codegen faster
12751 // if there are many clusters.
12752 sortAndRangeify(Clusters);
12753
12754 // The branch probablity of the peeled case.
12755 BranchProbability PeeledCaseProb = BranchProbability::getZero();
12756 MachineBasicBlock *PeeledSwitchMBB =
12757 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12758
12759 // If there is only the default destination, jump there directly.
12760 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12761 if (Clusters.empty()) {
12762 assert(PeeledSwitchMBB == SwitchMBB);
12763 SwitchMBB->addSuccessor(Succ: DefaultMBB);
12764 if (DefaultMBB != NextBlock(MBB: SwitchMBB)) {
12765 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other,
12766 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: DefaultMBB)));
12767 }
12768 return;
12769 }
12770
12771 SL->findJumpTables(Clusters, SI: &SI, SL: getCurSDLoc(), DefaultMBB, PSI: DAG.getPSI(),
12772 BFI: DAG.getBFI());
12773 SL->findBitTestClusters(Clusters, SI: &SI);
12774
12775 LLVM_DEBUG({
12776 dbgs() << "Case clusters: ";
12777 for (const CaseCluster &C : Clusters) {
12778 if (C.Kind == CC_JumpTable)
12779 dbgs() << "JT:";
12780 if (C.Kind == CC_BitTests)
12781 dbgs() << "BT:";
12782
12783 C.Low->getValue().print(dbgs(), true);
12784 if (C.Low != C.High) {
12785 dbgs() << '-';
12786 C.High->getValue().print(dbgs(), true);
12787 }
12788 dbgs() << ' ';
12789 }
12790 dbgs() << '\n';
12791 });
12792
12793 assert(!Clusters.empty());
12794 SwitchWorkList WorkList;
12795 CaseClusterIt First = Clusters.begin();
12796 CaseClusterIt Last = Clusters.end() - 1;
12797 auto DefaultProb = getEdgeProbability(Src: PeeledSwitchMBB, Dst: DefaultMBB);
12798 // Scale the branchprobability for DefaultMBB if the peel occurs and
12799 // DefaultMBB is not replaced.
12800 if (PeeledCaseProb != BranchProbability::getZero() &&
12801 DefaultMBB == FuncInfo.getMBB(BB: SI.getDefaultDest()))
12802 DefaultProb = scaleCaseProbality(CaseProb: DefaultProb, PeeledCaseProb);
12803 WorkList.push_back(
12804 Elt: {.MBB: PeeledSwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
12805
12806 while (!WorkList.empty()) {
12807 SwitchWorkListItem W = WorkList.pop_back_val();
12808 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12809
12810 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12811 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12812 // For optimized builds, lower large range as a balanced binary tree.
12813 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB);
12814 continue;
12815 }
12816
12817 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB);
12818 }
12819}
12820
12821void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12822 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12823 auto DL = getCurSDLoc();
12824 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
12825 setValue(V: &I, NewN: DAG.getStepVector(DL, ResVT: ResultVT));
12826}
12827
12828void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12829 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12830 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
12831
12832 SDLoc DL = getCurSDLoc();
12833 SDValue V = getValue(V: I.getOperand(i_nocapture: 0));
12834 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12835
12836 if (VT.isScalableVector()) {
12837 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V));
12838 return;
12839 }
12840
12841 // Use VECTOR_SHUFFLE for the fixed-length vector
12842 // to maintain existing behavior.
12843 SmallVector<int, 8> Mask;
12844 unsigned NumElts = VT.getVectorMinNumElements();
12845 for (unsigned i = 0; i != NumElts; ++i)
12846 Mask.push_back(Elt: NumElts - 1 - i);
12847
12848 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V, N2: DAG.getUNDEF(VT), Mask));
12849}
12850
12851void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
12852 unsigned Factor) {
12853 auto DL = getCurSDLoc();
12854 SDValue InVec = getValue(V: I.getOperand(i_nocapture: 0));
12855
12856 SmallVector<EVT, 4> ValueVTs;
12857 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
12858 ValueVTs);
12859
12860 EVT OutVT = ValueVTs[0];
12861 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12862
12863 SmallVector<SDValue, 4> SubVecs(Factor);
12864 for (unsigned i = 0; i != Factor; ++i) {
12865 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
12866 SubVecs[i] = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OutVT, N1: InVec,
12867 N2: DAG.getVectorIdxConstant(Val: OutNumElts * i, DL));
12868 }
12869
12870 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
12871 // from existing legalisation and combines.
12872 if (OutVT.isFixedLengthVector() && Factor == 2) {
12873 SDValue Even = DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: SubVecs[0], N2: SubVecs[1],
12874 Mask: createStrideMask(Start: 0, Stride: 2, VF: OutNumElts));
12875 SDValue Odd = DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: SubVecs[0], N2: SubVecs[1],
12876 Mask: createStrideMask(Start: 1, Stride: 2, VF: OutNumElts));
12877 SDValue Res = DAG.getMergeValues(Ops: {Even, Odd}, dl: getCurSDLoc());
12878 setValue(V: &I, NewN: Res);
12879 return;
12880 }
12881
12882 SDValue Res = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
12883 VTList: DAG.getVTList(VTs: ValueVTs), Ops: SubVecs);
12884 setValue(V: &I, NewN: Res);
12885}
12886
12887void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
12888 unsigned Factor) {
12889 auto DL = getCurSDLoc();
12890 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12891 EVT InVT = getValue(V: I.getOperand(i_nocapture: 0)).getValueType();
12892 EVT OutVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
12893
12894 SmallVector<SDValue, 8> InVecs(Factor);
12895 for (unsigned i = 0; i < Factor; ++i) {
12896 InVecs[i] = getValue(V: I.getOperand(i_nocapture: i));
12897 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
12898 "Expected VTs to be the same");
12899 }
12900
12901 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
12902 // from existing legalisation and combines.
12903 if (OutVT.isFixedLengthVector() && Factor == 2) {
12904 unsigned NumElts = InVT.getVectorMinNumElements();
12905 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: InVecs);
12906 setValue(V: &I, NewN: DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: V, N2: DAG.getUNDEF(VT: OutVT),
12907 Mask: createInterleaveMask(VF: NumElts, NumVecs: 2)));
12908 return;
12909 }
12910
12911 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
12912 SDValue Res =
12913 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, VTList: DAG.getVTList(VTs: ValueVTs), Ops: InVecs);
12914
12915 SmallVector<SDValue, 8> Results(Factor);
12916 for (unsigned i = 0; i < Factor; ++i)
12917 Results[i] = Res.getValue(R: i);
12918
12919 Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: Results);
12920 setValue(V: &I, NewN: Res);
12921}
12922
12923void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12924 SmallVector<EVT, 4> ValueVTs;
12925 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
12926 ValueVTs);
12927 unsigned NumValues = ValueVTs.size();
12928 if (NumValues == 0) return;
12929
12930 SmallVector<SDValue, 4> Values(NumValues);
12931 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
12932
12933 for (unsigned i = 0; i != NumValues; ++i)
12934 Values[i] = DAG.getNode(Opcode: ISD::FREEZE, DL: getCurSDLoc(), VT: ValueVTs[i],
12935 Operand: SDValue(Op.getNode(), Op.getResNo() + i));
12936
12937 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
12938 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
12939}
12940
12941void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12942 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12943 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
12944
12945 SDLoc DL = getCurSDLoc();
12946 SDValue V1 = getValue(V: I.getOperand(i_nocapture: 0));
12947 SDValue V2 = getValue(V: I.getOperand(i_nocapture: 1));
12948 const bool IsLeft = I.getIntrinsicID() == Intrinsic::vector_splice_left;
12949
12950 // VECTOR_SHUFFLE doesn't support a scalable or non-constant mask.
12951 if (VT.isScalableVector() || !isa<ConstantInt>(Val: I.getOperand(i_nocapture: 2))) {
12952 SDValue Offset = DAG.getZExtOrTrunc(
12953 Op: getValue(V: I.getOperand(i_nocapture: 2)), DL, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
12954 setValue(V: &I, NewN: DAG.getNode(Opcode: IsLeft ? ISD::VECTOR_SPLICE_LEFT
12955 : ISD::VECTOR_SPLICE_RIGHT,
12956 DL, VT, N1: V1, N2: V2, N3: Offset));
12957 return;
12958 }
12959 uint64_t Imm = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->getZExtValue();
12960
12961 unsigned NumElts = VT.getVectorNumElements();
12962
12963 uint64_t Idx = IsLeft ? Imm : NumElts - Imm;
12964
12965 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12966 SmallVector<int, 8> Mask;
12967 for (unsigned i = 0; i < NumElts; ++i)
12968 Mask.push_back(Elt: Idx + i);
12969 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask));
12970}
12971
12972// Consider the following MIR after SelectionDAG, which produces output in
12973// phyregs in the first case or virtregs in the second case.
12974//
12975// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12976// %5:gr32 = COPY $ebx
12977// %6:gr32 = COPY $edx
12978// %1:gr32 = COPY %6:gr32
12979// %0:gr32 = COPY %5:gr32
12980//
12981// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12982// %1:gr32 = COPY %6:gr32
12983// %0:gr32 = COPY %5:gr32
12984//
12985// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12986// Given %1, we'd like to return $edx in the first case and %6 in the second.
12987//
12988// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12989// to a single virtreg (such as %0). The remaining outputs monotonically
12990// increase in virtreg number from there. If a callbr has no outputs, then it
12991// should not have a corresponding callbr landingpad; in fact, the callbr
12992// landingpad would not even be able to refer to such a callbr.
12993static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
12994 MachineInstr *MI = MRI.def_begin(RegNo: Reg)->getParent();
12995 // There is definitely at least one copy.
12996 assert(MI->getOpcode() == TargetOpcode::COPY &&
12997 "start of copy chain MUST be COPY");
12998 Reg = MI->getOperand(i: 1).getReg();
12999
13000 // If the copied register in the first copy must be virtual.
13001 assert(Reg.isVirtual() && "expected COPY of virtual register");
13002 MI = MRI.def_begin(RegNo: Reg)->getParent();
13003
13004 // There may be an optional second copy.
13005 if (MI->getOpcode() == TargetOpcode::COPY) {
13006 assert(Reg.isVirtual() && "expected COPY of virtual register");
13007 Reg = MI->getOperand(i: 1).getReg();
13008 assert(Reg.isPhysical() && "expected COPY of physical register");
13009 } else {
13010 // The start of the chain must be an INLINEASM_BR.
13011 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13012 "end of copy chain MUST be INLINEASM_BR");
13013 }
13014
13015 return Reg;
13016}
13017
13018// We must do this walk rather than the simpler
13019// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
13020// otherwise we will end up with copies of virtregs only valid along direct
13021// edges.
13022void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
13023 SmallVector<EVT, 8> ResultVTs;
13024 SmallVector<SDValue, 8> ResultValues;
13025 const auto *CBR =
13026 cast<CallBrInst>(Val: I.getParent()->getUniquePredecessor()->getTerminator());
13027
13028 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13029 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
13030 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
13031
13032 Register InitialDef = FuncInfo.ValueMap[CBR];
13033 SDValue Chain = DAG.getRoot();
13034
13035 // Re-parse the asm constraints string.
13036 TargetLowering::AsmOperandInfoVector TargetConstraints =
13037 TLI.ParseConstraints(DL: DAG.getDataLayout(), TRI, Call: *CBR);
13038 for (auto &T : TargetConstraints) {
13039 SDISelAsmOperandInfo OpInfo(T);
13040 if (OpInfo.Type != InlineAsm::isOutput)
13041 continue;
13042
13043 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
13044 // individual constraint.
13045 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
13046
13047 switch (OpInfo.ConstraintType) {
13048 case TargetLowering::C_Register:
13049 case TargetLowering::C_RegisterClass: {
13050 // Fill in OpInfo.AssignedRegs.Regs.
13051 getRegistersForValue(DAG, DL: getCurSDLoc(), OpInfo, RefOpInfo&: OpInfo);
13052
13053 // getRegistersForValue may produce 1 to many registers based on whether
13054 // the OpInfo.ConstraintVT is legal on the target or not.
13055 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
13056 Register OriginalDef = FollowCopyChain(MRI, Reg: InitialDef++);
13057 if (OriginalDef.isPhysical())
13058 FuncInfo.MBB->addLiveIn(PhysReg: OriginalDef);
13059 // Update the assigned registers to use the original defs.
13060 Reg = OriginalDef;
13061 }
13062
13063 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
13064 DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V: CBR);
13065 ResultValues.push_back(Elt: V);
13066 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
13067 break;
13068 }
13069 case TargetLowering::C_Other: {
13070 SDValue Flag;
13071 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Glue&: Flag, DL: getCurSDLoc(),
13072 OpInfo, DAG);
13073 ++InitialDef;
13074 ResultValues.push_back(Elt: V);
13075 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
13076 break;
13077 }
13078 default:
13079 break;
13080 }
13081 }
13082 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
13083 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
13084 setValue(V: &I, NewN: V);
13085}
13086