1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/Analysis/AliasAnalysis.h"
24#include "llvm/Analysis/BranchProbabilityInfo.h"
25#include "llvm/Analysis/ConstantFolding.h"
26#include "llvm/Analysis/Loads.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/TargetTransformInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/Analysis/VectorUtils.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
34#include "llvm/CodeGen/CodeGenCommonISel.h"
35#include "llvm/CodeGen/FunctionLoweringInfo.h"
36#include "llvm/CodeGen/GCMetadata.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/MachineBasicBlock.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineInstrBuilder.h"
42#include "llvm/CodeGen/MachineInstrBundleIterator.h"
43#include "llvm/CodeGen/MachineMemOperand.h"
44#include "llvm/CodeGen/MachineModuleInfo.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/MachineRegisterInfo.h"
47#include "llvm/CodeGen/SelectionDAG.h"
48#include "llvm/CodeGen/SelectionDAGNodes.h"
49#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
50#include "llvm/CodeGen/StackMaps.h"
51#include "llvm/CodeGen/SwiftErrorValueTracking.h"
52#include "llvm/CodeGen/TargetFrameLowering.h"
53#include "llvm/CodeGen/TargetInstrInfo.h"
54#include "llvm/CodeGen/TargetOpcodes.h"
55#include "llvm/CodeGen/TargetRegisterInfo.h"
56#include "llvm/CodeGen/TargetSubtargetInfo.h"
57#include "llvm/CodeGen/WinEHFuncInfo.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
64#include "llvm/IR/ConstantRange.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
68#include "llvm/IR/DebugInfoMetadata.h"
69#include "llvm/IR/DerivedTypes.h"
70#include "llvm/IR/DiagnosticInfo.h"
71#include "llvm/IR/EHPersonalities.h"
72#include "llvm/IR/Function.h"
73#include "llvm/IR/GetElementPtrTypeIterator.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
76#include "llvm/IR/Instructions.h"
77#include "llvm/IR/IntrinsicInst.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
84#include "llvm/IR/Metadata.h"
85#include "llvm/IR/Module.h"
86#include "llvm/IR/Operator.h"
87#include "llvm/IR/PatternMatch.h"
88#include "llvm/IR/Statepoint.h"
89#include "llvm/IR/Type.h"
90#include "llvm/IR/User.h"
91#include "llvm/IR/Value.h"
92#include "llvm/MC/MCContext.h"
93#include "llvm/Support/AtomicOrdering.h"
94#include "llvm/Support/Casting.h"
95#include "llvm/Support/CommandLine.h"
96#include "llvm/Support/Compiler.h"
97#include "llvm/Support/Debug.h"
98#include "llvm/Support/InstructionCost.h"
99#include "llvm/Support/MathExtras.h"
100#include "llvm/Support/raw_ostream.h"
101#include "llvm/Target/TargetMachine.h"
102#include "llvm/Target/TargetOptions.h"
103#include "llvm/TargetParser/Triple.h"
104#include "llvm/Transforms/Utils/Local.h"
105#include <cstddef>
106#include <limits>
107#include <optional>
108#include <tuple>
109
110using namespace llvm;
111using namespace PatternMatch;
112using namespace SwitchCG;
113
114#define DEBUG_TYPE "isel"
115
116/// LimitFloatPrecision - Generate low-precision inline sequences for
117/// some float libcalls (6, 8 or 12 bits).
118static unsigned LimitFloatPrecision;
119
120static cl::opt<bool>
121 InsertAssertAlign("insert-assert-align", cl::init(Val: true),
122 cl::desc("Insert the experimental `assertalign` node."),
123 cl::ReallyHidden);
124
125static cl::opt<unsigned, true>
126 LimitFPPrecision("limit-float-precision",
127 cl::desc("Generate low-precision inline sequences "
128 "for some float libcalls"),
129 cl::location(L&: LimitFloatPrecision), cl::Hidden,
130 cl::init(Val: 0));
131
132static cl::opt<unsigned> SwitchPeelThreshold(
133 "switch-peel-threshold", cl::Hidden, cl::init(Val: 66),
134 cl::desc("Set the case probability threshold for peeling the case from a "
135 "switch statement. A value greater than 100 will void this "
136 "optimization"));
137
138// Limit the width of DAG chains. This is important in general to prevent
139// DAG-based analysis from blowing up. For example, alias analysis and
140// load clustering may not complete in reasonable time. It is difficult to
141// recognize and avoid this situation within each individual analysis, and
142// future analyses are likely to have the same behavior. Limiting DAG width is
143// the safe approach and will be especially important with global DAGs.
144//
145// MaxParallelChains default is arbitrarily high to avoid affecting
146// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
147// sequence over this should have been converted to llvm.memcpy by the
148// frontend. It is easy to induce this behavior with .ll code such as:
149// %buffer = alloca [4096 x i8]
150// %data = load [4096 x i8]* %argPtr
151// store [4096 x i8] %data, [4096 x i8]* %buffer
152static const unsigned MaxParallelChains = 64;
153
154static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
155 const SDValue *Parts, unsigned NumParts,
156 MVT PartVT, EVT ValueVT, const Value *V,
157 SDValue InChain,
158 std::optional<CallingConv::ID> CC);
159
160/// getCopyFromParts - Create a value that contains the specified legal parts
161/// combined into the value they represent. If the parts combine to a type
162/// larger than ValueVT then AssertOp can be used to specify whether the extra
163/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
164/// (ISD::AssertSext).
165static SDValue
166getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
167 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
168 SDValue InChain,
169 std::optional<CallingConv::ID> CC = std::nullopt,
170 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
171 // Let the target assemble the parts if it wants to
172 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
173 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
174 PartVT, ValueVT, CC))
175 return Val;
176
177 if (ValueVT.isVector())
178 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
179 InChain, CC);
180
181 assert(NumParts > 0 && "No parts to assemble!");
182 SDValue Val = Parts[0];
183
184 if (NumParts > 1) {
185 // Assemble the value from multiple parts.
186 if (ValueVT.isInteger()) {
187 unsigned PartBits = PartVT.getSizeInBits();
188 unsigned ValueBits = ValueVT.getSizeInBits();
189
190 // Assemble the power of 2 part.
191 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
192 unsigned RoundBits = PartBits * RoundParts;
193 EVT RoundVT = RoundBits == ValueBits ?
194 ValueVT : EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits);
195 SDValue Lo, Hi;
196
197 EVT HalfVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits/2);
198
199 if (RoundParts > 2) {
200 Lo = getCopyFromParts(DAG, DL, Parts, NumParts: RoundParts / 2, PartVT, ValueVT: HalfVT, V,
201 InChain);
202 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts / 2, NumParts: RoundParts / 2,
203 PartVT, ValueVT: HalfVT, V, InChain);
204 } else {
205 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[0]);
206 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[1]);
207 }
208
209 if (DAG.getDataLayout().isBigEndian())
210 std::swap(a&: Lo, b&: Hi);
211
212 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: RoundVT, N1: Lo, N2: Hi);
213
214 if (RoundParts < NumParts) {
215 // Assemble the trailing non-power-of-2 part.
216 unsigned OddParts = NumParts - RoundParts;
217 EVT OddVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OddParts * PartBits);
218 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts, NumParts: OddParts, PartVT,
219 ValueVT: OddVT, V, InChain, CC);
220
221 // Combine the round and odd parts.
222 Lo = Val;
223 if (DAG.getDataLayout().isBigEndian())
224 std::swap(a&: Lo, b&: Hi);
225 EVT TotalVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
226 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: TotalVT, Operand: Hi);
227 Hi = DAG.getNode(
228 Opcode: ISD::SHL, DL, VT: TotalVT, N1: Hi,
229 N2: DAG.getShiftAmountConstant(Val: Lo.getValueSizeInBits(), VT: TotalVT, DL));
230 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: TotalVT, Operand: Lo);
231 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: TotalVT, N1: Lo, N2: Hi);
232 }
233 } else if (PartVT.isFloatingPoint()) {
234 // FP split into multiple FP parts (for ppcf128)
235 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
236 "Unexpected split");
237 SDValue Lo, Hi;
238 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EVT(MVT::f64), Operand: Parts[0]);
239 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EVT(MVT::f64), Operand: Parts[1]);
240 if (TLI.hasBigEndianPartOrdering(VT: ValueVT, DL: DAG.getDataLayout()))
241 std::swap(a&: Lo, b&: Hi);
242 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: ValueVT, N1: Lo, N2: Hi);
243 } else {
244 // FP split into integer parts (soft fp)
245 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
246 !PartVT.isVector() && "Unexpected split");
247 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
248 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, ValueVT: IntVT, V,
249 InChain, CC);
250 }
251 }
252
253 // There is now one part, held in Val. Correct it to match ValueVT.
254 // PartEVT is the type of the register class that holds the value.
255 // ValueVT is the type of the inline asm operation.
256 EVT PartEVT = Val.getValueType();
257
258 if (PartEVT == ValueVT)
259 return Val;
260
261 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
262 ValueVT.bitsLT(VT: PartEVT)) {
263 // For an FP value in an integer part, we need to truncate to the right
264 // width first.
265 PartEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
266 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: PartEVT, Operand: Val);
267 }
268
269 // Handle types that have the same size.
270 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
271 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
272
273 // Handle types with different sizes.
274 if (PartEVT.isInteger() && ValueVT.isInteger()) {
275 if (ValueVT.bitsLT(VT: PartEVT)) {
276 // For a truncate, see if we have any information to
277 // indicate whether the truncated bits will always be
278 // zero or sign-extension.
279 if (AssertOp)
280 Val = DAG.getNode(Opcode: *AssertOp, DL, VT: PartEVT, N1: Val,
281 N2: DAG.getValueType(ValueVT));
282 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
283 }
284 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
285 }
286
287 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
288 // FP_ROUND's are always exact here.
289 if (ValueVT.bitsLT(VT: Val.getValueType())) {
290
291 SDValue NoChange =
292 DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
293
294 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
295 Kind: llvm::Attribute::StrictFP)) {
296 return DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL,
297 VTList: DAG.getVTList(VT1: ValueVT, VT2: MVT::Other), N1: InChain, N2: Val,
298 N3: NoChange);
299 }
300
301 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ValueVT, N1: Val, N2: NoChange);
302 }
303
304 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: ValueVT, Operand: Val);
305 }
306
307 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
308 // then truncating.
309 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
310 ValueVT.bitsLT(VT: PartEVT)) {
311 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Val);
312 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
313 }
314
315 report_fatal_error(reason: "Unknown mismatch in getCopyFromParts!");
316}
317
318static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
319 const Twine &ErrMsg) {
320 const Instruction *I = dyn_cast_or_null<Instruction>(Val: V);
321 if (!I)
322 return Ctx.emitError(ErrorStr: ErrMsg);
323
324 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
325 if (CI->isInlineAsm()) {
326 return Ctx.diagnose(DI: DiagnosticInfoInlineAsm(
327 *CI, ErrMsg + ", possible invalid constraint for vector type"));
328 }
329
330 return Ctx.emitError(I, ErrorStr: ErrMsg);
331}
332
333/// getCopyFromPartsVector - Create a value that contains the specified legal
334/// parts combined into the value they represent. If the parts combine to a
335/// type larger than ValueVT then AssertOp can be used to specify whether the
336/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
337/// ValueVT (ISD::AssertSext).
338static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
339 const SDValue *Parts, unsigned NumParts,
340 MVT PartVT, EVT ValueVT, const Value *V,
341 SDValue InChain,
342 std::optional<CallingConv::ID> CallConv) {
343 assert(ValueVT.isVector() && "Not a vector value");
344 assert(NumParts > 0 && "No parts to assemble!");
345 const bool IsABIRegCopy = CallConv.has_value();
346
347 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
348 SDValue Val = Parts[0];
349
350 // Handle a multi-element vector.
351 if (NumParts > 1) {
352 EVT IntermediateVT;
353 MVT RegisterVT;
354 unsigned NumIntermediates;
355 unsigned NumRegs;
356
357 if (IsABIRegCopy) {
358 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
359 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT,
360 NumIntermediates, RegisterVT);
361 } else {
362 NumRegs =
363 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
364 NumIntermediates, RegisterVT);
365 }
366
367 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
368 NumParts = NumRegs; // Silence a compiler warning.
369 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
370 assert(RegisterVT.getSizeInBits() ==
371 Parts[0].getSimpleValueType().getSizeInBits() &&
372 "Part type sizes don't match!");
373
374 // Assemble the parts into intermediate operands.
375 SmallVector<SDValue, 8> Ops(NumIntermediates);
376 if (NumIntermediates == NumParts) {
377 // If the register was not expanded, truncate or copy the value,
378 // as appropriate.
379 for (unsigned i = 0; i != NumParts; ++i)
380 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i], NumParts: 1, PartVT, ValueVT: IntermediateVT,
381 V, InChain, CC: CallConv);
382 } else if (NumParts > 0) {
383 // If the intermediate type was expanded, build the intermediate
384 // operands from the parts.
385 assert(NumParts % NumIntermediates == 0 &&
386 "Must expand into a divisible number of parts!");
387 unsigned Factor = NumParts / NumIntermediates;
388 for (unsigned i = 0; i != NumIntermediates; ++i)
389 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i * Factor], NumParts: Factor, PartVT,
390 ValueVT: IntermediateVT, V, InChain, CC: CallConv);
391 }
392
393 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
394 // intermediate operands.
395 EVT BuiltVectorTy =
396 IntermediateVT.isVector()
397 ? EVT::getVectorVT(
398 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(),
399 EC: IntermediateVT.getVectorElementCount() * NumParts)
400 : EVT::getVectorVT(Context&: *DAG.getContext(),
401 VT: IntermediateVT.getScalarType(),
402 NumElements: NumIntermediates);
403 Val = DAG.getNode(Opcode: IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
404 : ISD::BUILD_VECTOR,
405 DL, VT: BuiltVectorTy, Ops);
406 }
407
408 // There is now one part, held in Val. Correct it to match ValueVT.
409 EVT PartEVT = Val.getValueType();
410
411 if (PartEVT == ValueVT)
412 return Val;
413
414 if (PartEVT.isVector()) {
415 // Vector/Vector bitcast.
416 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
417 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
418
419 // If the parts vector has more elements than the value vector, then we
420 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
421 // Extract the elements we want.
422 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
423 assert((PartEVT.getVectorElementCount().getKnownMinValue() >
424 ValueVT.getVectorElementCount().getKnownMinValue()) &&
425 (PartEVT.getVectorElementCount().isScalable() ==
426 ValueVT.getVectorElementCount().isScalable()) &&
427 "Cannot narrow, it would be a lossy transformation");
428 PartEVT =
429 EVT::getVectorVT(Context&: *DAG.getContext(), VT: PartEVT.getVectorElementType(),
430 EC: ValueVT.getVectorElementCount());
431 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: PartEVT, N1: Val,
432 N2: DAG.getVectorIdxConstant(Val: 0, DL));
433 if (PartEVT == ValueVT)
434 return Val;
435 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
436 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
437
438 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
439 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
440 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
441 }
442
443 // Promoted vector extract
444 return DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueVT);
445 }
446
447 // Trivial bitcast if the types are the same size and the destination
448 // vector type is legal.
449 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
450 TLI.isTypeLegal(VT: ValueVT))
451 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
452
453 if (ValueVT.getVectorNumElements() != 1) {
454 // Certain ABIs require that vectors are passed as integers. For vectors
455 // are the same size, this is an obvious bitcast.
456 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
457 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
458 } else if (ValueVT.bitsLT(VT: PartEVT)) {
459 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
460 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
461 // Drop the extra bits.
462 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
463 return DAG.getBitcast(VT: ValueVT, V: Val);
464 }
465
466 diagnosePossiblyInvalidConstraint(
467 Ctx&: *DAG.getContext(), V, ErrMsg: "non-trivial scalar-to-vector conversion");
468 return DAG.getUNDEF(VT: ValueVT);
469 }
470
471 // Handle cases such as i8 -> <1 x i1>
472 EVT ValueSVT = ValueVT.getVectorElementType();
473 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
474 unsigned ValueSize = ValueSVT.getSizeInBits();
475 if (ValueSize == PartEVT.getSizeInBits()) {
476 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueSVT, Operand: Val);
477 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
478 // It's possible a scalar floating point type gets softened to integer and
479 // then promoted to a larger integer. If PartEVT is the larger integer
480 // we need to truncate it and then bitcast to the FP type.
481 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
482 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
483 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
484 Val = DAG.getBitcast(VT: ValueSVT, V: Val);
485 } else {
486 Val = ValueVT.isFloatingPoint()
487 ? DAG.getFPExtendOrRound(Op: Val, DL, VT: ValueSVT)
488 : DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueSVT);
489 }
490 }
491
492 return DAG.getBuildVector(VT: ValueVT, DL, Ops: Val);
493}
494
495static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
496 SDValue Val, SDValue *Parts, unsigned NumParts,
497 MVT PartVT, const Value *V,
498 std::optional<CallingConv::ID> CallConv);
499
500/// getCopyToParts - Create a series of nodes that contain the specified value
501/// split into legal parts. If the parts contain more bits than Val, then, for
502/// integers, ExtendKind can be used to specify how to generate the extra bits.
503static void
504getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
505 unsigned NumParts, MVT PartVT, const Value *V,
506 std::optional<CallingConv::ID> CallConv = std::nullopt,
507 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
508 // Let the target split the parts if it wants to
509 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
510 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
511 CC: CallConv))
512 return;
513 EVT ValueVT = Val.getValueType();
514
515 // Handle the vector case separately.
516 if (ValueVT.isVector())
517 return getCopyToPartsVector(DAG, dl: DL, Val, Parts, NumParts, PartVT, V,
518 CallConv);
519
520 unsigned OrigNumParts = NumParts;
521 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
522 "Copying to an illegal type!");
523
524 if (NumParts == 0)
525 return;
526
527 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
528 EVT PartEVT = PartVT;
529 if (PartEVT == ValueVT) {
530 assert(NumParts == 1 && "No-op copy with multiple parts!");
531 Parts[0] = Val;
532 return;
533 }
534
535 unsigned PartBits = PartVT.getSizeInBits();
536 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
537 // If the parts cover more bits than the value has, promote the value.
538 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
539 assert(NumParts == 1 && "Do not know what to promote to!");
540 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
541 } else {
542 if (ValueVT.isFloatingPoint()) {
543 // FP values need to be bitcast, then extended if they are being put
544 // into a larger container.
545 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
546 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
547 }
548 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
549 ValueVT.isInteger() &&
550 "Unknown mismatch!");
551 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
552 Val = DAG.getNode(Opcode: ExtendKind, DL, VT: ValueVT, Operand: Val);
553 if (PartVT == MVT::x86mmx)
554 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
555 }
556 } else if (PartBits == ValueVT.getSizeInBits()) {
557 // Different types of the same size.
558 assert(NumParts == 1 && PartEVT != ValueVT);
559 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
560 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
561 // If the parts cover less bits than value has, truncate the value.
562 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
563 ValueVT.isInteger() &&
564 "Unknown mismatch!");
565 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
566 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
567 if (PartVT == MVT::x86mmx)
568 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
569 }
570
571 // The value may have changed - recompute ValueVT.
572 ValueVT = Val.getValueType();
573 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
574 "Failed to tile the value with PartVT!");
575
576 if (NumParts == 1) {
577 if (PartEVT != ValueVT) {
578 diagnosePossiblyInvalidConstraint(Ctx&: *DAG.getContext(), V,
579 ErrMsg: "scalar-to-vector conversion failed");
580 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
581 }
582
583 Parts[0] = Val;
584 return;
585 }
586
587 // Expand the value into multiple parts.
588 if (NumParts & (NumParts - 1)) {
589 // The number of parts is not a power of 2. Split off and copy the tail.
590 assert(PartVT.isInteger() && ValueVT.isInteger() &&
591 "Do not know what to expand to!");
592 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
593 unsigned RoundBits = RoundParts * PartBits;
594 unsigned OddParts = NumParts - RoundParts;
595 SDValue OddVal = DAG.getNode(Opcode: ISD::SRL, DL, VT: ValueVT, N1: Val,
596 N2: DAG.getShiftAmountConstant(Val: RoundBits, VT: ValueVT, DL));
597
598 getCopyToParts(DAG, DL, Val: OddVal, Parts: Parts + RoundParts, NumParts: OddParts, PartVT, V,
599 CallConv);
600
601 if (DAG.getDataLayout().isBigEndian())
602 // The odd parts were reversed by getCopyToParts - unreverse them.
603 std::reverse(first: Parts + RoundParts, last: Parts + NumParts);
604
605 NumParts = RoundParts;
606 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
607 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
608 }
609
610 // The number of parts is a power of 2. Repeatedly bisect the value using
611 // EXTRACT_ELEMENT.
612 Parts[0] = DAG.getNode(Opcode: ISD::BITCAST, DL,
613 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
614 BitWidth: ValueVT.getSizeInBits()),
615 Operand: Val);
616
617 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
618 for (unsigned i = 0; i < NumParts; i += StepSize) {
619 unsigned ThisBits = StepSize * PartBits / 2;
620 EVT ThisVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ThisBits);
621 SDValue &Part0 = Parts[i];
622 SDValue &Part1 = Parts[i+StepSize/2];
623
624 Part1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
625 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 1, DL));
626 Part0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
627 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 0, DL));
628
629 if (ThisBits == PartBits && ThisVT != PartVT) {
630 Part0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part0);
631 Part1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part1);
632 }
633 }
634 }
635
636 if (DAG.getDataLayout().isBigEndian())
637 std::reverse(first: Parts, last: Parts + OrigNumParts);
638}
639
640static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
641 const SDLoc &DL, EVT PartVT) {
642 if (!PartVT.isVector())
643 return SDValue();
644
645 EVT ValueVT = Val.getValueType();
646 EVT PartEVT = PartVT.getVectorElementType();
647 EVT ValueEVT = ValueVT.getVectorElementType();
648 ElementCount PartNumElts = PartVT.getVectorElementCount();
649 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
650
651 // We only support widening vectors with equivalent element types and
652 // fixed/scalable properties. If a target needs to widen a fixed-length type
653 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
654 if (ElementCount::isKnownLE(LHS: PartNumElts, RHS: ValueNumElts) ||
655 PartNumElts.isScalable() != ValueNumElts.isScalable())
656 return SDValue();
657
658 // Have a try for bf16 because some targets share its ABI with fp16.
659 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
660 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
661 "Cannot widen to illegal type");
662 Val = DAG.getNode(
663 Opcode: ISD::BITCAST, DL,
664 VT: ValueVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::f16), Operand: Val);
665 } else if (PartEVT != ValueEVT) {
666 return SDValue();
667 }
668
669 // Widening a scalable vector to another scalable vector is done by inserting
670 // the vector into a larger undef one.
671 if (PartNumElts.isScalable())
672 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: PartVT, N1: DAG.getUNDEF(VT: PartVT),
673 N2: Val, N3: DAG.getVectorIdxConstant(Val: 0, DL));
674
675 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
676 // undef elements.
677 SmallVector<SDValue, 16> Ops;
678 DAG.ExtractVectorElements(Op: Val, Args&: Ops);
679 SDValue EltUndef = DAG.getUNDEF(VT: PartEVT);
680 Ops.append(NumInputs: (PartNumElts - ValueNumElts).getFixedValue(), Elt: EltUndef);
681
682 // FIXME: Use CONCAT for 2x -> 4x.
683 return DAG.getBuildVector(VT: PartVT, DL, Ops);
684}
685
686/// getCopyToPartsVector - Create a series of nodes that contain the specified
687/// value split into legal parts.
688static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
689 SDValue Val, SDValue *Parts, unsigned NumParts,
690 MVT PartVT, const Value *V,
691 std::optional<CallingConv::ID> CallConv) {
692 EVT ValueVT = Val.getValueType();
693 assert(ValueVT.isVector() && "Not a vector");
694 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
695 const bool IsABIRegCopy = CallConv.has_value();
696
697 if (NumParts == 1) {
698 EVT PartEVT = PartVT;
699 if (PartEVT == ValueVT) {
700 // Nothing to do.
701 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
702 // Bitconvert vector->vector case.
703 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
704 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
705 Val = Widened;
706 } else if (PartVT.isVector() &&
707 PartEVT.getVectorElementType().bitsGE(
708 VT: ValueVT.getVectorElementType()) &&
709 PartEVT.getVectorElementCount() ==
710 ValueVT.getVectorElementCount()) {
711
712 // Promoted vector extract
713 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
714 } else if (PartEVT.isVector() &&
715 PartEVT.getVectorElementType() !=
716 ValueVT.getVectorElementType() &&
717 TLI.getTypeAction(Context&: *DAG.getContext(), VT: ValueVT) ==
718 TargetLowering::TypeWidenVector) {
719 // Combination of widening and promotion.
720 EVT WidenVT =
721 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
722 EC: PartVT.getVectorElementCount());
723 SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: WidenVT);
724 Val = DAG.getAnyExtOrTrunc(Op: Widened, DL, VT: PartVT);
725 } else {
726 // Don't extract an integer from a float vector. This can happen if the
727 // FP type gets softened to integer and then promoted. The promotion
728 // prevents it from being picked up by the earlier bitcast case.
729 if (ValueVT.getVectorElementCount().isScalar() &&
730 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
731 // If we reach this condition and PartVT is FP, this means that
732 // ValueVT is also FP and both have a different size, otherwise we
733 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
734 // would be invalid since that would mean the smaller FP type has to
735 // be extended to the larger one.
736 if (PartVT.isFloatingPoint()) {
737 Val = DAG.getBitcast(VT: ValueVT.getScalarType(), V: Val);
738 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
739 } else
740 Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: PartVT, N1: Val,
741 N2: DAG.getVectorIdxConstant(Val: 0, DL));
742 } else {
743 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
744 assert(PartVT.getFixedSizeInBits() > ValueSize &&
745 "lossy conversion of vector to scalar type");
746 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
747 Val = DAG.getBitcast(VT: IntermediateType, V: Val);
748 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
749 }
750 }
751
752 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
753 Parts[0] = Val;
754 return;
755 }
756
757 // Handle a multi-element vector.
758 EVT IntermediateVT;
759 MVT RegisterVT;
760 unsigned NumIntermediates;
761 unsigned NumRegs;
762 if (IsABIRegCopy) {
763 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
764 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT, NumIntermediates,
765 RegisterVT);
766 } else {
767 NumRegs =
768 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
769 NumIntermediates, RegisterVT);
770 }
771
772 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
773 NumParts = NumRegs; // Silence a compiler warning.
774 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
775
776 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
777 "Mixing scalable and fixed vectors when copying in parts");
778
779 std::optional<ElementCount> DestEltCnt;
780
781 if (IntermediateVT.isVector())
782 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
783 else
784 DestEltCnt = ElementCount::getFixed(MinVal: NumIntermediates);
785
786 EVT BuiltVectorTy = EVT::getVectorVT(
787 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(), EC: *DestEltCnt);
788
789 if (ValueVT == BuiltVectorTy) {
790 // Nothing to do.
791 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
792 // Bitconvert vector->vector case.
793 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BuiltVectorTy, Operand: Val);
794 } else {
795 if (BuiltVectorTy.getVectorElementType().bitsGT(
796 VT: ValueVT.getVectorElementType())) {
797 // Integer promotion.
798 ValueVT = EVT::getVectorVT(Context&: *DAG.getContext(),
799 VT: BuiltVectorTy.getVectorElementType(),
800 EC: ValueVT.getVectorElementCount());
801 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
802 }
803
804 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: BuiltVectorTy)) {
805 Val = Widened;
806 }
807 }
808
809 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
810
811 // Split the vector into intermediate operands.
812 SmallVector<SDValue, 8> Ops(NumIntermediates);
813 for (unsigned i = 0; i != NumIntermediates; ++i) {
814 if (IntermediateVT.isVector()) {
815 // This does something sensible for scalable vectors - see the
816 // definition of EXTRACT_SUBVECTOR for further details.
817 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
818 Ops[i] =
819 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: IntermediateVT, N1: Val,
820 N2: DAG.getVectorIdxConstant(Val: i * IntermediateNumElts, DL));
821 } else {
822 Ops[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IntermediateVT, N1: Val,
823 N2: DAG.getVectorIdxConstant(Val: i, DL));
824 }
825 }
826
827 // Split the intermediate operands into legal parts.
828 if (NumParts == NumIntermediates) {
829 // If the register was not expanded, promote or copy the value,
830 // as appropriate.
831 for (unsigned i = 0; i != NumParts; ++i)
832 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i], NumParts: 1, PartVT, V, CallConv);
833 } else if (NumParts > 0) {
834 // If the intermediate type was expanded, split each the value into
835 // legal parts.
836 assert(NumIntermediates != 0 && "division by zero");
837 assert(NumParts % NumIntermediates == 0 &&
838 "Must expand into a divisible number of parts!");
839 unsigned Factor = NumParts / NumIntermediates;
840 for (unsigned i = 0; i != NumIntermediates; ++i)
841 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i * Factor], NumParts: Factor, PartVT, V,
842 CallConv);
843 }
844}
845
846static void failForInvalidBundles(const CallBase &I, StringRef Name,
847 ArrayRef<uint32_t> AllowedBundles) {
848 if (I.hasOperandBundlesOtherThan(IDs: AllowedBundles)) {
849 ListSeparator LS;
850 std::string Error;
851 raw_string_ostream OS(Error);
852 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
853 OperandBundleUse U = I.getOperandBundleAt(Index: i);
854 if (!is_contained(Range&: AllowedBundles, Element: U.getTagID()))
855 OS << LS << U.getTagName();
856 }
857 reportFatalUsageError(
858 reason: Twine("cannot lower ", Name)
859 .concat(Suffix: Twine(" with arbitrary operand bundles: ", Error)));
860 }
861}
862
863RegsForValue::RegsForValue(const SmallVector<Register, 4> &regs, MVT regvt,
864 EVT valuevt, std::optional<CallingConv::ID> CC)
865 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
866 RegCount(1, regs.size()), CallConv(CC) {}
867
868RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
869 const DataLayout &DL, Register Reg, Type *Ty,
870 std::optional<CallingConv::ID> CC) {
871 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
872
873 CallConv = CC;
874
875 for (EVT ValueVT : ValueVTs) {
876 unsigned NumRegs =
877 isABIMangled()
878 ? TLI.getNumRegistersForCallingConv(Context, CC: *CC, VT: ValueVT)
879 : TLI.getNumRegisters(Context, VT: ValueVT);
880 MVT RegisterVT =
881 isABIMangled()
882 ? TLI.getRegisterTypeForCallingConv(Context, CC: *CC, VT: ValueVT)
883 : TLI.getRegisterType(Context, VT: ValueVT);
884 for (unsigned i = 0; i != NumRegs; ++i)
885 Regs.push_back(Elt: Reg + i);
886 RegVTs.push_back(Elt: RegisterVT);
887 RegCount.push_back(Elt: NumRegs);
888 Reg = Reg.id() + NumRegs;
889 }
890}
891
892SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
893 FunctionLoweringInfo &FuncInfo,
894 const SDLoc &dl, SDValue &Chain,
895 SDValue *Glue, const Value *V) const {
896 // A Value with type {} or [0 x %t] needs no registers.
897 if (ValueVTs.empty())
898 return SDValue();
899
900 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
901
902 // Assemble the legal parts into the final values.
903 SmallVector<SDValue, 4> Values(ValueVTs.size());
904 SmallVector<SDValue, 8> Parts;
905 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
906 // Copy the legal parts from the registers.
907 EVT ValueVT = ValueVTs[Value];
908 unsigned NumRegs = RegCount[Value];
909 MVT RegisterVT = isABIMangled()
910 ? TLI.getRegisterTypeForCallingConv(
911 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
912 : RegVTs[Value];
913
914 Parts.resize(N: NumRegs);
915 for (unsigned i = 0; i != NumRegs; ++i) {
916 SDValue P;
917 if (!Glue) {
918 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT);
919 } else {
920 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT, Glue: *Glue);
921 *Glue = P.getValue(R: 2);
922 }
923
924 Chain = P.getValue(R: 1);
925 Parts[i] = P;
926
927 // If the source register was virtual and if we know something about it,
928 // add an assert node.
929 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
930 continue;
931
932 const FunctionLoweringInfo::LiveOutInfo *LOI =
933 FuncInfo.GetLiveOutRegInfo(Reg: Regs[Part+i]);
934 if (!LOI)
935 continue;
936
937 unsigned RegSize = RegisterVT.getScalarSizeInBits();
938 unsigned NumSignBits = LOI->NumSignBits;
939 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
940
941 if (NumZeroBits == RegSize) {
942 // The current value is a zero.
943 // Explicitly express that as it would be easier for
944 // optimizations to kick in.
945 Parts[i] = DAG.getConstant(Val: 0, DL: dl, VT: RegisterVT);
946 continue;
947 }
948
949 // FIXME: We capture more information than the dag can represent. For
950 // now, just use the tightest assertzext/assertsext possible.
951 bool isSExt;
952 EVT FromVT(MVT::Other);
953 if (NumZeroBits) {
954 FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
955 isSExt = false;
956 } else if (NumSignBits > 1) {
957 FromVT =
958 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumSignBits + 1);
959 isSExt = true;
960 } else {
961 continue;
962 }
963 // Add an assertion node.
964 assert(FromVT != MVT::Other);
965 Parts[i] = DAG.getNode(Opcode: isSExt ? ISD::AssertSext : ISD::AssertZext, DL: dl,
966 VT: RegisterVT, N1: P, N2: DAG.getValueType(FromVT));
967 }
968
969 Values[Value] = getCopyFromParts(DAG, DL: dl, Parts: Parts.begin(), NumParts: NumRegs,
970 PartVT: RegisterVT, ValueVT, V, InChain: Chain, CC: CallConv);
971 Part += NumRegs;
972 Parts.clear();
973 }
974
975 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values);
976}
977
978void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
979 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
980 const Value *V,
981 ISD::NodeType PreferredExtendType) const {
982 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
983 ISD::NodeType ExtendKind = PreferredExtendType;
984
985 // Get the list of the values's legal parts.
986 unsigned NumRegs = Regs.size();
987 SmallVector<SDValue, 8> Parts(NumRegs);
988 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
989 unsigned NumParts = RegCount[Value];
990
991 MVT RegisterVT = isABIMangled()
992 ? TLI.getRegisterTypeForCallingConv(
993 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
994 : RegVTs[Value];
995
996 if (ExtendKind == ISD::ANY_EXTEND)
997 if (TLI.isZExtFree(Val: peekThroughFreeze(V: Val), VT2: RegisterVT))
998 ExtendKind = ISD::ZERO_EXTEND;
999
1000 getCopyToParts(DAG, DL: dl, Val: Val.getValue(R: Val.getResNo() + Value), Parts: &Parts[Part],
1001 NumParts, PartVT: RegisterVT, V, CallConv, ExtendKind);
1002 Part += NumParts;
1003 }
1004
1005 // Copy the parts into the registers.
1006 SmallVector<SDValue, 8> Chains(NumRegs);
1007 for (unsigned i = 0; i != NumRegs; ++i) {
1008 SDValue Part;
1009 if (!Glue) {
1010 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i]);
1011 } else {
1012 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i], Glue: *Glue);
1013 *Glue = Part.getValue(R: 1);
1014 }
1015
1016 Chains[i] = Part.getValue(R: 0);
1017 }
1018
1019 if (NumRegs == 1 || Glue)
1020 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1021 // flagged to it. That is the CopyToReg nodes and the user are considered
1022 // a single scheduling unit. If we create a TokenFactor and return it as
1023 // chain, then the TokenFactor is both a predecessor (operand) of the
1024 // user as well as a successor (the TF operands are flagged to the user).
1025 // c1, f1 = CopyToReg
1026 // c2, f2 = CopyToReg
1027 // c3 = TokenFactor c1, c2
1028 // ...
1029 // = op c3, ..., f2
1030 Chain = Chains[NumRegs-1];
1031 else
1032 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
1033}
1034
1035void RegsForValue::AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching,
1036 unsigned MatchingIdx, const SDLoc &dl,
1037 SelectionDAG &DAG,
1038 std::vector<SDValue> &Ops) const {
1039 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1040
1041 InlineAsm::Flag Flag(Code, Regs.size());
1042 if (HasMatching)
1043 Flag.setMatchingOp(MatchingIdx);
1044 else if (!Regs.empty() && Regs.front().isVirtual()) {
1045 // Put the register class of the virtual registers in the flag word. That
1046 // way, later passes can recompute register class constraints for inline
1047 // assembly as well as normal instructions.
1048 // Don't do this for tied operands that can use the regclass information
1049 // from the def.
1050 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
1051 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Regs.front());
1052 Flag.setRegClass(RC->getID());
1053 }
1054
1055 SDValue Res = DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32);
1056 Ops.push_back(x: Res);
1057
1058 if (Code == InlineAsm::Kind::Clobber) {
1059 // Clobbers should always have a 1:1 mapping with registers, and may
1060 // reference registers that have illegal (e.g. vector) types. Hence, we
1061 // shouldn't try to apply any sort of splitting logic to them.
1062 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1063 "No 1:1 mapping from clobbers to regs?");
1064 Register SP = TLI.getStackPointerRegisterToSaveRestore();
1065 (void)SP;
1066 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1067 Ops.push_back(x: DAG.getRegister(Reg: Regs[I], VT: RegVTs[I]));
1068 assert(
1069 (Regs[I] != SP ||
1070 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
1071 "If we clobbered the stack pointer, MFI should know about it.");
1072 }
1073 return;
1074 }
1075
1076 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1077 MVT RegisterVT = RegVTs[Value];
1078 unsigned NumRegs = TLI.getNumRegisters(Context&: *DAG.getContext(), VT: ValueVTs[Value],
1079 RegisterVT);
1080 for (unsigned i = 0; i != NumRegs; ++i) {
1081 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1082 Register TheReg = Regs[Reg++];
1083 Ops.push_back(x: DAG.getRegister(Reg: TheReg, VT: RegisterVT));
1084 }
1085 }
1086}
1087
1088SmallVector<std::pair<Register, TypeSize>, 4>
1089RegsForValue::getRegsAndSizes() const {
1090 SmallVector<std::pair<Register, TypeSize>, 4> OutVec;
1091 unsigned I = 0;
1092 for (auto CountAndVT : zip_first(t: RegCount, u: RegVTs)) {
1093 unsigned RegCount = std::get<0>(t&: CountAndVT);
1094 MVT RegisterVT = std::get<1>(t&: CountAndVT);
1095 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1096 for (unsigned E = I + RegCount; I != E; ++I)
1097 OutVec.push_back(Elt: std::make_pair(x: Regs[I], y&: RegisterSize));
1098 }
1099 return OutVec;
1100}
1101
1102void SelectionDAGBuilder::init(GCFunctionInfo *gfi, BatchAAResults *aa,
1103 AssumptionCache *ac, const TargetLibraryInfo *li,
1104 const TargetTransformInfo &TTI) {
1105 BatchAA = aa;
1106 AC = ac;
1107 GFI = gfi;
1108 LibInfo = li;
1109 Context = DAG.getContext();
1110 LPadToCallSiteMap.clear();
1111 this->TTI = &TTI;
1112 SL->init(tli: DAG.getTargetLoweringInfo(), tm: TM, dl: DAG.getDataLayout());
1113 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1114 M: *DAG.getMachineFunction().getFunction().getParent());
1115}
1116
1117void SelectionDAGBuilder::clear() {
1118 NodeMap.clear();
1119 UnusedArgNodeMap.clear();
1120 PendingLoads.clear();
1121 PendingExports.clear();
1122 PendingConstrainedFP.clear();
1123 PendingConstrainedFPStrict.clear();
1124 CurInst = nullptr;
1125 HasTailCall = false;
1126 SDNodeOrder = LowestSDNodeOrder;
1127 StatepointLowering.clear();
1128}
1129
1130void SelectionDAGBuilder::clearDanglingDebugInfo() {
1131 DanglingDebugInfoMap.clear();
1132}
1133
1134// Update DAG root to include dependencies on Pending chains.
1135SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1136 SDValue Root = DAG.getRoot();
1137
1138 if (Pending.empty())
1139 return Root;
1140
1141 // Add current root to PendingChains, unless we already indirectly
1142 // depend on it.
1143 if (Root.getOpcode() != ISD::EntryToken) {
1144 unsigned i = 0, e = Pending.size();
1145 for (; i != e; ++i) {
1146 assert(Pending[i].getNode()->getNumOperands() > 1);
1147 if (Pending[i].getNode()->getOperand(Num: 0) == Root)
1148 break; // Don't add the root if we already indirectly depend on it.
1149 }
1150
1151 if (i == e)
1152 Pending.push_back(Elt: Root);
1153 }
1154
1155 if (Pending.size() == 1)
1156 Root = Pending[0];
1157 else
1158 Root = DAG.getTokenFactor(DL: getCurSDLoc(), Vals&: Pending);
1159
1160 DAG.setRoot(Root);
1161 Pending.clear();
1162 return Root;
1163}
1164
1165SDValue SelectionDAGBuilder::getMemoryRoot() {
1166 return updateRoot(Pending&: PendingLoads);
1167}
1168
1169SDValue SelectionDAGBuilder::getFPOperationRoot(fp::ExceptionBehavior EB) {
1170 // If the new exception behavior differs from that of the pending
1171 // ones, chain up them and update the root.
1172 switch (EB) {
1173 case fp::ExceptionBehavior::ebMayTrap:
1174 case fp::ExceptionBehavior::ebIgnore:
1175 // Floating-point exceptions produced by such operations are not intended
1176 // to be observed, so the sequence of these operations does not need to be
1177 // preserved.
1178 //
1179 // They however must not be mixed with the instructions that have strict
1180 // exception behavior. Placing an operation with 'ebIgnore' behavior between
1181 // 'ebStrict' operations could distort the observed exception behavior.
1182 if (!PendingConstrainedFPStrict.empty()) {
1183 assert(PendingConstrainedFP.empty());
1184 updateRoot(Pending&: PendingConstrainedFPStrict);
1185 }
1186 break;
1187 case fp::ExceptionBehavior::ebStrict:
1188 // Floating-point exception produced by these operations may be observed, so
1189 // they must be correctly chained. If trapping on FP exceptions is
1190 // disabled, the exceptions can be observed only by functions that read
1191 // exception flags, like 'llvm.get_fpenv' or 'fetestexcept'. It means that
1192 // the order of operations is not significant between barriers.
1193 //
1194 // If trapping is enabled, each operation becomes an implicit observation
1195 // point, so the operations must be sequenced according their original
1196 // source order.
1197 if (!PendingConstrainedFP.empty()) {
1198 assert(PendingConstrainedFPStrict.empty());
1199 updateRoot(Pending&: PendingConstrainedFP);
1200 }
1201 // TODO: Add support for trapping-enabled scenarios.
1202 }
1203 return DAG.getRoot();
1204}
1205
1206SDValue SelectionDAGBuilder::getRoot() {
1207 // Chain up all pending constrained intrinsics together with all
1208 // pending loads, by simply appending them to PendingLoads and
1209 // then calling getMemoryRoot().
1210 PendingLoads.reserve(N: PendingLoads.size() +
1211 PendingConstrainedFP.size() +
1212 PendingConstrainedFPStrict.size());
1213 PendingLoads.append(in_start: PendingConstrainedFP.begin(),
1214 in_end: PendingConstrainedFP.end());
1215 PendingLoads.append(in_start: PendingConstrainedFPStrict.begin(),
1216 in_end: PendingConstrainedFPStrict.end());
1217 PendingConstrainedFP.clear();
1218 PendingConstrainedFPStrict.clear();
1219 return getMemoryRoot();
1220}
1221
1222SDValue SelectionDAGBuilder::getControlRoot() {
1223 // We need to emit pending fpexcept.strict constrained intrinsics,
1224 // so append them to the PendingExports list.
1225 PendingExports.append(in_start: PendingConstrainedFPStrict.begin(),
1226 in_end: PendingConstrainedFPStrict.end());
1227 PendingConstrainedFPStrict.clear();
1228 return updateRoot(Pending&: PendingExports);
1229}
1230
1231void SelectionDAGBuilder::handleDebugDeclare(Value *Address,
1232 DILocalVariable *Variable,
1233 DIExpression *Expression,
1234 DebugLoc DL) {
1235 assert(Variable && "Missing variable");
1236
1237 // Check if address has undef value.
1238 if (!Address || isa<UndefValue>(Val: Address) ||
1239 (Address->use_empty() && !isa<Argument>(Val: Address))) {
1240 LLVM_DEBUG(
1241 dbgs()
1242 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1243 return;
1244 }
1245
1246 bool IsParameter = Variable->isParameter() || isa<Argument>(Val: Address);
1247
1248 SDValue &N = NodeMap[Address];
1249 if (!N.getNode() && isa<Argument>(Val: Address))
1250 // Check unused arguments map.
1251 N = UnusedArgNodeMap[Address];
1252 SDDbgValue *SDV;
1253 if (N.getNode()) {
1254 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Val: Address))
1255 Address = BCI->getOperand(i_nocapture: 0);
1256 // Parameters are handled specially.
1257 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: N.getNode());
1258 if (IsParameter && FINode) {
1259 // Byval parameter. We have a frame index at this point.
1260 SDV = DAG.getFrameIndexDbgValue(Var: Variable, Expr: Expression, FI: FINode->getIndex(),
1261 /*IsIndirect*/ true, DL, O: SDNodeOrder);
1262 } else if (isa<Argument>(Val: Address)) {
1263 // Address is an argument, so try to emit its dbg value using
1264 // virtual register info from the FuncInfo.ValueMap.
1265 EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1266 Kind: FuncArgumentDbgValueKind::Declare, N);
1267 return;
1268 } else {
1269 SDV = DAG.getDbgValue(Var: Variable, Expr: Expression, N: N.getNode(), R: N.getResNo(),
1270 IsIndirect: true, DL, O: SDNodeOrder);
1271 }
1272 DAG.AddDbgValue(DB: SDV, isParameter: IsParameter);
1273 } else {
1274 // If Address is an argument then try to emit its dbg value using
1275 // virtual register info from the FuncInfo.ValueMap.
1276 if (!EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1277 Kind: FuncArgumentDbgValueKind::Declare, N)) {
1278 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1279 << " (could not emit func-arg dbg_value)\n");
1280 }
1281 }
1282}
1283
1284void SelectionDAGBuilder::visitDbgInfo(const Instruction &I) {
1285 // Add SDDbgValue nodes for any var locs here. Do so before updating
1286 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1287 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1288 // Add SDDbgValue nodes for any var locs here. Do so before updating
1289 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1290 for (auto It = FnVarLocs->locs_begin(Before: &I), End = FnVarLocs->locs_end(Before: &I);
1291 It != End; ++It) {
1292 auto *Var = FnVarLocs->getDILocalVariable(ID: It->VariableID);
1293 dropDanglingDebugInfo(Variable: Var, Expr: It->Expr);
1294 if (It->Values.isKillLocation(Expression: It->Expr)) {
1295 handleKillDebugValue(Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder);
1296 continue;
1297 }
1298 SmallVector<Value *> Values(It->Values.location_ops());
1299 if (!handleDebugValue(Values, Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder,
1300 IsVariadic: It->Values.hasArgList())) {
1301 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1302 addDanglingDebugInfo(Values&: Vals,
1303 Var: FnVarLocs->getDILocalVariable(ID: It->VariableID),
1304 Expr: It->Expr, IsVariadic: Vals.size() > 1, DL: It->DL, Order: SDNodeOrder);
1305 }
1306 }
1307 }
1308
1309 // We must skip DbgVariableRecords if they've already been processed above as
1310 // we have just emitted the debug values resulting from assignment tracking
1311 // analysis, making any existing DbgVariableRecords redundant (and probably
1312 // less correct). We still need to process DbgLabelRecords. This does sink
1313 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1314 // be important as it does so deterministcally and ordering between
1315 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1316 // printing).
1317 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1318 // Is there is any debug-info attached to this instruction, in the form of
1319 // DbgRecord non-instruction debug-info records.
1320 for (DbgRecord &DR : I.getDbgRecordRange()) {
1321 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1322 assert(DLR->getLabel() && "Missing label");
1323 SDDbgLabel *SDV =
1324 DAG.getDbgLabel(Label: DLR->getLabel(), DL: DLR->getDebugLoc(), O: SDNodeOrder);
1325 DAG.AddDbgLabel(DB: SDV);
1326 continue;
1327 }
1328
1329 if (SkipDbgVariableRecords)
1330 continue;
1331 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1332 DILocalVariable *Variable = DVR.getVariable();
1333 DIExpression *Expression = DVR.getExpression();
1334 dropDanglingDebugInfo(Variable, Expr: Expression);
1335
1336 if (DVR.getType() == DbgVariableRecord::LocationType::Declare) {
1337 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1338 continue;
1339 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1340 << "\n");
1341 handleDebugDeclare(Address: DVR.getVariableLocationOp(OpIdx: 0), Variable, Expression,
1342 DL: DVR.getDebugLoc());
1343 continue;
1344 }
1345
1346 // A DbgVariableRecord with no locations is a kill location.
1347 SmallVector<Value *, 4> Values(DVR.location_ops());
1348 if (Values.empty()) {
1349 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1350 Order: SDNodeOrder);
1351 continue;
1352 }
1353
1354 // A DbgVariableRecord with an undef or absent location is also a kill
1355 // location.
1356 if (llvm::any_of(Range&: Values,
1357 P: [](Value *V) { return !V || isa<UndefValue>(Val: V); })) {
1358 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1359 Order: SDNodeOrder);
1360 continue;
1361 }
1362
1363 bool IsVariadic = DVR.hasArgList();
1364 if (!handleDebugValue(Values, Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1365 Order: SDNodeOrder, IsVariadic)) {
1366 addDanglingDebugInfo(Values, Var: Variable, Expr: Expression, IsVariadic,
1367 DL: DVR.getDebugLoc(), Order: SDNodeOrder);
1368 }
1369 }
1370}
1371
1372void SelectionDAGBuilder::visit(const Instruction &I) {
1373 visitDbgInfo(I);
1374
1375 // Set up outgoing PHI node register values before emitting the terminator.
1376 if (I.isTerminator()) {
1377 HandlePHINodesInSuccessorBlocks(LLVMBB: I.getParent());
1378 }
1379
1380 ++SDNodeOrder;
1381 CurInst = &I;
1382
1383 // Set inserted listener only if required.
1384 bool NodeInserted = false;
1385 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1386 MDNode *PCSectionsMD = I.getMetadata(KindID: LLVMContext::MD_pcsections);
1387 MDNode *MMRA = I.getMetadata(KindID: LLVMContext::MD_mmra);
1388 if (PCSectionsMD || MMRA) {
1389 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1390 args&: DAG, args: [&](SDNode *) { NodeInserted = true; });
1391 }
1392
1393 visit(Opcode: I.getOpcode(), I);
1394
1395 if (!I.isTerminator() && !HasTailCall &&
1396 !isa<GCStatepointInst>(Val: I)) // statepoints handle their exports internally
1397 CopyToExportRegsIfNeeded(V: &I);
1398
1399 // Handle metadata.
1400 if (PCSectionsMD || MMRA) {
1401 auto It = NodeMap.find(Val: &I);
1402 if (It != NodeMap.end()) {
1403 if (PCSectionsMD)
1404 DAG.addPCSections(Node: It->second.getNode(), MD: PCSectionsMD);
1405 if (MMRA)
1406 DAG.addMMRAMetadata(Node: It->second.getNode(), MMRA);
1407 } else if (NodeInserted) {
1408 // This should not happen; if it does, don't let it go unnoticed so we can
1409 // fix it. Relevant visit*() function is probably missing a setValue().
1410 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1411 << I.getModule()->getName() << "]\n";
1412 LLVM_DEBUG(I.dump());
1413 assert(false);
1414 }
1415 }
1416
1417 CurInst = nullptr;
1418}
1419
1420void SelectionDAGBuilder::visitPHI(const PHINode &) {
1421 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1422}
1423
1424void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1425 // Note: this doesn't use InstVisitor, because it has to work with
1426 // ConstantExpr's in addition to instructions.
1427 switch (Opcode) {
1428 default: llvm_unreachable("Unknown instruction type encountered!");
1429 // Build the switch statement using the Instruction.def file.
1430#define HANDLE_INST(NUM, OPCODE, CLASS) \
1431 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1432#include "llvm/IR/Instruction.def"
1433 }
1434}
1435
1436static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,
1437 DILocalVariable *Variable,
1438 DebugLoc DL, unsigned Order,
1439 SmallVectorImpl<Value *> &Values,
1440 DIExpression *Expression) {
1441 // For variadic dbg_values we will now insert poison.
1442 // FIXME: We can potentially recover these!
1443 SmallVector<SDDbgOperand, 2> Locs;
1444 for (const Value *V : Values) {
1445 auto *Poison = PoisonValue::get(T: V->getType());
1446 Locs.push_back(Elt: SDDbgOperand::fromConst(Const: Poison));
1447 }
1448 SDDbgValue *SDV = DAG.getDbgValueList(Var: Variable, Expr: Expression, Locs, Dependencies: {},
1449 /*IsIndirect=*/false, DL, O: Order,
1450 /*IsVariadic=*/true);
1451 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1452 return true;
1453}
1454
1455void SelectionDAGBuilder::addDanglingDebugInfo(SmallVectorImpl<Value *> &Values,
1456 DILocalVariable *Var,
1457 DIExpression *Expr,
1458 bool IsVariadic, DebugLoc DL,
1459 unsigned Order) {
1460 if (IsVariadic) {
1461 handleDanglingVariadicDebugInfo(DAG, Variable: Var, DL, Order, Values, Expression: Expr);
1462 return;
1463 }
1464 // TODO: Dangling debug info will eventually either be resolved or produce
1465 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1466 // between the original dbg.value location and its resolved DBG_VALUE,
1467 // which we should ideally fill with an extra poison DBG_VALUE.
1468 assert(Values.size() == 1);
1469 DanglingDebugInfoMap[Values[0]].emplace_back(args&: Var, args&: Expr, args&: DL, args&: Order);
1470}
1471
1472void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1473 const DIExpression *Expr) {
1474 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1475 DIVariable *DanglingVariable = DDI.getVariable();
1476 DIExpression *DanglingExpr = DDI.getExpression();
1477 if (DanglingVariable == Variable && Expr->fragmentsOverlap(Other: DanglingExpr)) {
1478 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1479 << printDDI(nullptr, DDI) << "\n");
1480 return true;
1481 }
1482 return false;
1483 };
1484
1485 for (auto &DDIMI : DanglingDebugInfoMap) {
1486 DanglingDebugInfoVector &DDIV = DDIMI.second;
1487
1488 // If debug info is to be dropped, run it through final checks to see
1489 // whether it can be salvaged.
1490 for (auto &DDI : DDIV)
1491 if (isMatchingDbgValue(DDI))
1492 salvageUnresolvedDbgValue(V: DDIMI.first, DDI);
1493
1494 erase_if(C&: DDIV, P: isMatchingDbgValue);
1495 }
1496}
1497
1498// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1499// generate the debug data structures now that we've seen its definition.
1500void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1501 SDValue Val) {
1502 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(Key: V);
1503 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1504 return;
1505
1506 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1507 for (auto &DDI : DDIV) {
1508 DebugLoc DL = DDI.getDebugLoc();
1509 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1510 DILocalVariable *Variable = DDI.getVariable();
1511 DIExpression *Expr = DDI.getExpression();
1512 assert(Variable->isValidLocationForIntrinsic(DL) &&
1513 "Expected inlined-at fields to agree");
1514 SDDbgValue *SDV;
1515 if (Val.getNode()) {
1516 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1517 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1518 // we couldn't resolve it directly when examining the DbgValue intrinsic
1519 // in the first place we should not be more successful here). Unless we
1520 // have some test case that prove this to be correct we should avoid
1521 // calling EmitFuncArgumentDbgValue here.
1522 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1523 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1524 Kind: FuncArgumentDbgValueKind::Value, N: Val)) {
1525 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1526 << printDDI(V, DDI) << "\n");
1527 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1528 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1529 // inserted after the definition of Val when emitting the instructions
1530 // after ISel. An alternative could be to teach
1531 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1532 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1533 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1534 << ValSDNodeOrder << "\n");
1535 SDV = getDbgValue(N: Val, Variable, Expr, dl: DL,
1536 DbgSDNodeOrder: std::max(a: DbgSDNodeOrder, b: ValSDNodeOrder));
1537 DAG.AddDbgValue(DB: SDV, isParameter: false);
1538 } else
1539 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1540 << printDDI(V, DDI)
1541 << " in EmitFuncArgumentDbgValue\n");
1542 } else {
1543 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1544 << "\n");
1545 auto Poison = PoisonValue::get(T: V->getType());
1546 auto SDV =
1547 DAG.getConstantDbgValue(Var: Variable, Expr, C: Poison, DL, O: DbgSDNodeOrder);
1548 DAG.AddDbgValue(DB: SDV, isParameter: false);
1549 }
1550 }
1551 DDIV.clear();
1552}
1553
1554void SelectionDAGBuilder::salvageUnresolvedDbgValue(const Value *V,
1555 DanglingDebugInfo &DDI) {
1556 // TODO: For the variadic implementation, instead of only checking the fail
1557 // state of `handleDebugValue`, we need know specifically which values were
1558 // invalid, so that we attempt to salvage only those values when processing
1559 // a DIArgList.
1560 const Value *OrigV = V;
1561 DILocalVariable *Var = DDI.getVariable();
1562 DIExpression *Expr = DDI.getExpression();
1563 DebugLoc DL = DDI.getDebugLoc();
1564 unsigned SDOrder = DDI.getSDNodeOrder();
1565
1566 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1567 // that DW_OP_stack_value is desired.
1568 bool StackValue = true;
1569
1570 // Can this Value can be encoded without any further work?
1571 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false))
1572 return;
1573
1574 // Attempt to salvage back through as many instructions as possible. Bail if
1575 // a non-instruction is seen, such as a constant expression or global
1576 // variable. FIXME: Further work could recover those too.
1577 while (isa<Instruction>(Val: V)) {
1578 const Instruction &VAsInst = *cast<const Instruction>(Val: V);
1579 // Temporary "0", awaiting real implementation.
1580 SmallVector<uint64_t, 16> Ops;
1581 SmallVector<Value *, 4> AdditionalValues;
1582 V = salvageDebugInfoImpl(I&: const_cast<Instruction &>(VAsInst),
1583 CurrentLocOps: Expr->getNumLocationOperands(), Ops,
1584 AdditionalValues);
1585 // If we cannot salvage any further, and haven't yet found a suitable debug
1586 // expression, bail out.
1587 if (!V)
1588 break;
1589
1590 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1591 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1592 // here for variadic dbg_values, remove that condition.
1593 if (!AdditionalValues.empty())
1594 break;
1595
1596 // New value and expr now represent this debuginfo.
1597 Expr = DIExpression::appendOpsToArg(Expr, Ops, ArgNo: 0, StackValue);
1598
1599 // Some kind of simplification occurred: check whether the operand of the
1600 // salvaged debug expression can be encoded in this DAG.
1601 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false)) {
1602 LLVM_DEBUG(
1603 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1604 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1605 return;
1606 }
1607 }
1608
1609 // This was the final opportunity to salvage this debug information, and it
1610 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1611 // any earlier variable location.
1612 assert(OrigV && "V shouldn't be null");
1613 auto *Poison = PoisonValue::get(T: OrigV->getType());
1614 auto *SDV = DAG.getConstantDbgValue(Var, Expr, C: Poison, DL, O: SDNodeOrder);
1615 DAG.AddDbgValue(DB: SDV, isParameter: false);
1616 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1617 << printDDI(OrigV, DDI) << "\n");
1618}
1619
1620void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var,
1621 DIExpression *Expr,
1622 DebugLoc DbgLoc,
1623 unsigned Order) {
1624 Value *Poison = PoisonValue::get(T: Type::getInt1Ty(C&: *Context));
1625 DIExpression *NewExpr =
1626 const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr));
1627 handleDebugValue(Values: Poison, Var, Expr: NewExpr, DbgLoc, Order,
1628 /*IsVariadic*/ false);
1629}
1630
1631bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1632 DILocalVariable *Var,
1633 DIExpression *Expr, DebugLoc DbgLoc,
1634 unsigned Order, bool IsVariadic) {
1635 if (Values.empty())
1636 return true;
1637
1638 // Filter EntryValue locations out early.
1639 if (visitEntryValueDbgValue(Values, Variable: Var, Expr, DbgLoc))
1640 return true;
1641
1642 SmallVector<SDDbgOperand> LocationOps;
1643 SmallVector<SDNode *> Dependencies;
1644 for (const Value *V : Values) {
1645 // Constant value.
1646 if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V) || isa<UndefValue>(Val: V) ||
1647 isa<ConstantPointerNull>(Val: V)) {
1648 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: V));
1649 continue;
1650 }
1651
1652 // Look through IntToPtr constants.
1653 if (auto *CE = dyn_cast<ConstantExpr>(Val: V))
1654 if (CE->getOpcode() == Instruction::IntToPtr) {
1655 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: CE->getOperand(i_nocapture: 0)));
1656 continue;
1657 }
1658
1659 // If the Value is a frame index, we can create a FrameIndex debug value
1660 // without relying on the DAG at all.
1661 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
1662 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
1663 if (SI != FuncInfo.StaticAllocaMap.end()) {
1664 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: SI->second));
1665 continue;
1666 }
1667 }
1668
1669 // Do not use getValue() in here; we don't want to generate code at
1670 // this point if it hasn't been done yet.
1671 SDValue N = NodeMap[V];
1672 if (!N.getNode() && isa<Argument>(Val: V)) // Check unused arguments map.
1673 N = UnusedArgNodeMap[V];
1674
1675 if (N.getNode()) {
1676 // Only emit func arg dbg value for non-variadic dbg.values for now.
1677 if (!IsVariadic &&
1678 EmitFuncArgumentDbgValue(V, Variable: Var, Expr, DL: DbgLoc,
1679 Kind: FuncArgumentDbgValueKind::Value, N))
1680 return true;
1681 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
1682 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1683 // describe stack slot locations.
1684 //
1685 // Consider "int x = 0; int *px = &x;". There are two kinds of
1686 // interesting debug values here after optimization:
1687 //
1688 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1689 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1690 //
1691 // Both describe the direct values of their associated variables.
1692 Dependencies.push_back(Elt: N.getNode());
1693 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex()));
1694 continue;
1695 }
1696 LocationOps.emplace_back(
1697 Args: SDDbgOperand::fromNode(Node: N.getNode(), ResNo: N.getResNo()));
1698 continue;
1699 }
1700
1701 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1702 // Special rules apply for the first dbg.values of parameter variables in a
1703 // function. Identify them by the fact they reference Argument Values, that
1704 // they're parameters, and they are parameters of the current function. We
1705 // need to let them dangle until they get an SDNode.
1706 bool IsParamOfFunc =
1707 isa<Argument>(Val: V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1708 if (IsParamOfFunc)
1709 return false;
1710
1711 // The value is not used in this block yet (or it would have an SDNode).
1712 // We still want the value to appear for the user if possible -- if it has
1713 // an associated VReg, we can refer to that instead.
1714 auto VMI = FuncInfo.ValueMap.find(Val: V);
1715 if (VMI != FuncInfo.ValueMap.end()) {
1716 Register Reg = VMI->second;
1717 // If this is a PHI node, it may be split up into several MI PHI nodes
1718 // (in FunctionLoweringInfo::set).
1719 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1720 V->getType(), std::nullopt);
1721 if (RFV.occupiesMultipleRegs()) {
1722 // FIXME: We could potentially support variadic dbg_values here.
1723 if (IsVariadic)
1724 return false;
1725 unsigned Offset = 0;
1726 unsigned BitsToDescribe = 0;
1727 if (auto VarSize = Var->getSizeInBits())
1728 BitsToDescribe = *VarSize;
1729 if (auto Fragment = Expr->getFragmentInfo())
1730 BitsToDescribe = Fragment->SizeInBits;
1731 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1732 // Bail out if all bits are described already.
1733 if (Offset >= BitsToDescribe)
1734 break;
1735 // TODO: handle scalable vectors.
1736 unsigned RegisterSize = RegAndSize.second;
1737 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1738 ? BitsToDescribe - Offset
1739 : RegisterSize;
1740 auto FragmentExpr = DIExpression::createFragmentExpression(
1741 Expr, OffsetInBits: Offset, SizeInBits: FragmentSize);
1742 if (!FragmentExpr)
1743 continue;
1744 SDDbgValue *SDV = DAG.getVRegDbgValue(
1745 Var, Expr: *FragmentExpr, VReg: RegAndSize.first, IsIndirect: false, DL: DbgLoc, O: Order);
1746 DAG.AddDbgValue(DB: SDV, isParameter: false);
1747 Offset += RegisterSize;
1748 }
1749 return true;
1750 }
1751 // We can use simple vreg locations for variadic dbg_values as well.
1752 LocationOps.emplace_back(Args: SDDbgOperand::fromVReg(VReg: Reg));
1753 continue;
1754 }
1755 // We failed to create a SDDbgOperand for V.
1756 return false;
1757 }
1758
1759 // We have created a SDDbgOperand for each Value in Values.
1760 assert(!LocationOps.empty());
1761 SDDbgValue *SDV =
1762 DAG.getDbgValueList(Var, Expr, Locs: LocationOps, Dependencies,
1763 /*IsIndirect=*/false, DL: DbgLoc, O: Order, IsVariadic);
1764 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1765 return true;
1766}
1767
1768void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1769 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1770 for (auto &Pair : DanglingDebugInfoMap)
1771 for (auto &DDI : Pair.second)
1772 salvageUnresolvedDbgValue(V: const_cast<Value *>(Pair.first), DDI);
1773 clearDanglingDebugInfo();
1774}
1775
1776/// getCopyFromRegs - If there was virtual register allocated for the value V
1777/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1778SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1779 auto It = FuncInfo.ValueMap.find(Val: V);
1780 SDValue Result;
1781
1782 if (It != FuncInfo.ValueMap.end()) {
1783 Register InReg = It->second;
1784
1785 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1786 DAG.getDataLayout(), InReg, Ty,
1787 std::nullopt); // This is not an ABI copy.
1788 SDValue Chain = DAG.getEntryNode();
1789 Result = RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr,
1790 V);
1791 resolveDanglingDebugInfo(V, Val: Result);
1792 }
1793
1794 return Result;
1795}
1796
1797/// getValue - Return an SDValue for the given Value.
1798SDValue SelectionDAGBuilder::getValue(const Value *V) {
1799 // If we already have an SDValue for this value, use it. It's important
1800 // to do this first, so that we don't create a CopyFromReg if we already
1801 // have a regular SDValue.
1802 SDValue &N = NodeMap[V];
1803 if (N.getNode()) return N;
1804
1805 // If there's a virtual register allocated and initialized for this
1806 // value, use it.
1807 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
1808 return copyFromReg;
1809
1810 // Otherwise create a new SDValue and remember it.
1811 SDValue Val = getValueImpl(V);
1812 NodeMap[V] = Val;
1813 resolveDanglingDebugInfo(V, Val);
1814 return Val;
1815}
1816
1817void SelectionDAGBuilder::setValueToPoison(const Value *V, const SDLoc &dl) {
1818 if (V->getType()->isVoidTy())
1819 return;
1820
1821 SmallVector<EVT, 4> ValueVTs;
1822 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
1823 Ty: V->getType(), ValueVTs);
1824 setValue(V, NewN: DAG.getErrorMergeValues(ResultTypes: ValueVTs, Chain: SDValue(), dl));
1825}
1826
1827/// getNonRegisterValue - Return an SDValue for the given Value, but
1828/// don't look in FuncInfo.ValueMap for a virtual register.
1829SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1830 // If we already have an SDValue for this value, use it.
1831 SDValue &N = NodeMap[V];
1832 if (N.getNode()) {
1833 if (isIntOrFPConstant(V: N)) {
1834 // Remove the debug location from the node as the node is about to be used
1835 // in a location which may differ from the original debug location. This
1836 // is relevant to Constant and ConstantFP nodes because they can appear
1837 // as constant expressions inside PHI nodes.
1838 N->setDebugLoc(DebugLoc());
1839 }
1840 return N;
1841 }
1842
1843 // Otherwise create a new SDValue and remember it.
1844 SDValue Val = getValueImpl(V);
1845 NodeMap[V] = Val;
1846 resolveDanglingDebugInfo(V, Val);
1847 return Val;
1848}
1849
1850/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1851/// Create an SDValue for the given value.
1852SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1853 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1854
1855 if (const Constant *C = dyn_cast<Constant>(Val: V)) {
1856 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: V->getType(), AllowUnknown: true);
1857
1858 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
1859 SDLoc DL = getCurSDLoc();
1860
1861 // DAG.getConstant() may attempt to legalise the vector constant which can
1862 // significantly change the combines applied to the DAG. To reduce the
1863 // divergence when enabling ConstantInt based vectors we try to construct
1864 // the DAG in the same way as shufflevector based splats. TODO: The
1865 // divergence sometimes leads to better optimisations. Ideally we should
1866 // prevent DAG.getConstant() from legalising too early but there are some
1867 // degradations preventing this.
1868 if (VT.isScalableVector())
1869 return DAG.getNode(
1870 Opcode: ISD::SPLAT_VECTOR, DL, VT,
1871 Operand: DAG.getConstant(Val: CI->getValue(), DL, VT: VT.getVectorElementType()));
1872 if (VT.isFixedLengthVector())
1873 return DAG.getSplatBuildVector(
1874 VT, DL,
1875 Op: DAG.getConstant(Val: CI->getValue(), DL, VT: VT.getVectorElementType()));
1876 return DAG.getConstant(Val: *CI, DL, VT);
1877 }
1878
1879 if (const ConstantByte *CB = dyn_cast<ConstantByte>(Val: C))
1880 return DAG.getConstant(Val: CB->getValue(), DL: getCurSDLoc(), VT);
1881
1882 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: C))
1883 return DAG.getGlobalAddress(GV, DL: getCurSDLoc(), VT);
1884
1885 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
1886 return DAG.getNode(Opcode: ISD::PtrAuthGlobalAddress, DL: getCurSDLoc(), VT,
1887 N1: getValue(V: CPA->getPointer()), N2: getValue(V: CPA->getKey()),
1888 N3: getValue(V: CPA->getAddrDiscriminator()),
1889 N4: getValue(V: CPA->getDiscriminator()));
1890 }
1891
1892 if (isa<ConstantPointerNull>(Val: C))
1893 return DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT);
1894
1895 if (match(V: C, P: m_VScale()))
1896 return DAG.getVScale(DL: getCurSDLoc(), VT, MulImm: APInt(VT.getSizeInBits(), 1));
1897
1898 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C))
1899 return DAG.getConstantFP(V: *CFP, DL: getCurSDLoc(), VT);
1900
1901 if (isa<UndefValue>(Val: C) && !V->getType()->isAggregateType())
1902 return isa<PoisonValue>(Val: C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1903
1904 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
1905 visit(Opcode: CE->getOpcode(), I: *CE);
1906 SDValue N1 = NodeMap[V];
1907 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1908 return N1;
1909 }
1910
1911 if (isa<ConstantStruct>(Val: C) || isa<ConstantArray>(Val: C)) {
1912 SmallVector<SDValue, 4> Constants;
1913 for (const Use &U : C->operands()) {
1914 SDNode *Val = getValue(V: U).getNode();
1915 // If the operand is an empty aggregate, there are no values.
1916 if (!Val) continue;
1917 // Add each leaf value from the operand to the Constants list
1918 // to form a flattened list of all the values.
1919 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1920 Constants.push_back(Elt: SDValue(Val, i));
1921 }
1922
1923 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
1924 }
1925
1926 if (const ConstantDataSequential *CDS =
1927 dyn_cast<ConstantDataSequential>(Val: C)) {
1928 SmallVector<SDValue, 4> Ops;
1929 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1930 SDNode *Val = getValue(V: CDS->getElementAsConstant(i)).getNode();
1931 // Add each leaf value from the operand to the Constants list
1932 // to form a flattened list of all the values.
1933 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1934 Ops.push_back(Elt: SDValue(Val, i));
1935 }
1936
1937 if (isa<ArrayType>(Val: CDS->getType()))
1938 return DAG.getMergeValues(Ops, dl: getCurSDLoc());
1939 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
1940 }
1941
1942 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1943 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1944 "Unknown struct or array constant!");
1945
1946 SmallVector<EVT, 4> ValueVTs;
1947 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: C->getType(), ValueVTs);
1948 unsigned NumElts = ValueVTs.size();
1949 if (NumElts == 0)
1950 return SDValue(); // empty struct
1951 SmallVector<SDValue, 4> Constants(NumElts);
1952 for (unsigned i = 0; i != NumElts; ++i) {
1953 EVT EltVT = ValueVTs[i];
1954 if (isa<UndefValue>(Val: C))
1955 Constants[i] = DAG.getUNDEF(VT: EltVT);
1956 else if (EltVT.isFloatingPoint())
1957 Constants[i] = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
1958 else
1959 Constants[i] = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
1960 }
1961
1962 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
1963 }
1964
1965 if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C))
1966 return DAG.getBlockAddress(BA, VT);
1967
1968 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C))
1969 return getValue(V: Equiv->getGlobalValue());
1970
1971 if (const auto *NC = dyn_cast<NoCFIValue>(Val: C))
1972 return getValue(V: NC->getGlobalValue());
1973
1974 if (VT == MVT::aarch64svcount) {
1975 assert(C->isNullValue() && "Can only zero this target type!");
1976 return DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT,
1977 Operand: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::nxv16i1));
1978 }
1979
1980 if (VT.isRISCVVectorTuple()) {
1981 assert(C->isNullValue() && "Can only zero this target type!");
1982 return DAG.getNode(
1983 Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT,
1984 Operand: DAG.getNode(
1985 Opcode: ISD::SPLAT_VECTOR, DL: getCurSDLoc(),
1986 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8,
1987 NumElements: VT.getSizeInBits().getKnownMinValue() / 8, IsScalable: true),
1988 Operand: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::getIntegerVT(BitWidth: 8))));
1989 }
1990
1991 if (VT == MVT::externref || VT == MVT::funcref) {
1992 assert(C->isNullValue() && "Can only zero this target type!");
1993 // The zero value of a WebAssembly reference type is the null reference,
1994 // materialized with ref.null.
1995 Intrinsic::ID IID = VT == MVT::externref ? Intrinsic::wasm_ref_null_extern
1996 : Intrinsic::wasm_ref_null_func;
1997 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: getCurSDLoc(), VT,
1998 Operand: DAG.getTargetConstant(Val: IID, DL: getCurSDLoc(), VT: MVT::i32));
1999 }
2000
2001 VectorType *VecTy = cast<VectorType>(Val: V->getType());
2002
2003 // Now that we know the number and type of the elements, get that number of
2004 // elements into the Ops array based on what kind of constant it is.
2005 if (const ConstantVector *CV = dyn_cast<ConstantVector>(Val: C)) {
2006 SmallVector<SDValue, 16> Ops;
2007 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2008 for (unsigned i = 0; i != NumElements; ++i)
2009 Ops.push_back(Elt: getValue(V: CV->getOperand(i_nocapture: i)));
2010
2011 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
2012 }
2013
2014 if (isa<ConstantAggregateZero>(Val: C)) {
2015 EVT EltVT =
2016 TLI.getValueType(DL: DAG.getDataLayout(), Ty: VecTy->getElementType());
2017
2018 SDValue Op;
2019 if (EltVT.isFloatingPoint())
2020 Op = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2021 else
2022 Op = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
2023
2024 return DAG.getSplat(VT, DL: getCurSDLoc(), Op);
2025 }
2026
2027 llvm_unreachable("Unknown vector constant");
2028 }
2029
2030 // If this is a static alloca, generate it as the frameindex instead of
2031 // computation.
2032 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
2033 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
2034 if (SI != FuncInfo.StaticAllocaMap.end())
2035 return DAG.getFrameIndex(
2036 FI: SI->second, VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: AI->getType()));
2037 }
2038
2039 // If this is an instruction which fast-isel has deferred, select it now.
2040 if (const Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
2041 Register InReg = FuncInfo.InitializeRegForValue(V: Inst);
2042 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
2043 Inst->getType(), std::nullopt);
2044 SDValue Chain = DAG.getEntryNode();
2045 return RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V);
2046 }
2047
2048 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(Val: V))
2049 return DAG.getMDNode(MD: cast<MDNode>(Val: MD->getMetadata()));
2050
2051 if (const auto *BB = dyn_cast<BasicBlock>(Val: V))
2052 return DAG.getBasicBlock(MBB: FuncInfo.getMBB(BB));
2053
2054 llvm_unreachable("Can't get register for value!");
2055}
2056
2057void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
2058 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2059 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
2060 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
2061 bool IsSEH = isAsynchronousEHPersonality(Pers);
2062 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
2063 if (IsSEH) {
2064 // For SEH, EHCont Guard needs to know that this catchpad is a target.
2065 CatchPadMBB->setIsEHContTarget(true);
2066 DAG.getMachineFunction().setHasEHContTarget(true);
2067 } else
2068 CatchPadMBB->setIsEHScopeEntry();
2069 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
2070 if (IsMSVCCXX || IsCoreCLR)
2071 CatchPadMBB->setIsEHFuncletEntry();
2072}
2073
2074void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
2075 // Update machine-CFG edge.
2076 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(BB: I.getSuccessor());
2077 FuncInfo.MBB->addSuccessor(Succ: TargetMBB);
2078
2079 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2080 bool IsSEH = isAsynchronousEHPersonality(Pers);
2081 if (IsSEH) {
2082 // If this is not a fall-through branch or optimizations are switched off,
2083 // emit the branch.
2084 if (TargetMBB != NextBlock(MBB: FuncInfo.MBB) ||
2085 TM.getOptLevel() == CodeGenOptLevel::None)
2086 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other,
2087 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: TargetMBB)));
2088 return;
2089 }
2090
2091 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2092 TargetMBB->setIsEHContTarget(true);
2093 DAG.getMachineFunction().setHasEHContTarget(true);
2094
2095 // Figure out the funclet membership for the catchret's successor.
2096 // This will be used by the FuncletLayout pass to determine how to order the
2097 // BB's.
2098 // A 'catchret' returns to the outer scope's color.
2099 Value *ParentPad = I.getCatchSwitchParentPad();
2100 const BasicBlock *SuccessorColor;
2101 if (isa<ConstantTokenNone>(Val: ParentPad))
2102 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2103 else
2104 SuccessorColor = cast<Instruction>(Val: ParentPad)->getParent();
2105 assert(SuccessorColor && "No parent funclet for catchret!");
2106 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(BB: SuccessorColor);
2107 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2108
2109 // Create the terminator node.
2110 SDValue Ret = DAG.getNode(Opcode: ISD::CATCHRET, DL: getCurSDLoc(), VT: MVT::Other,
2111 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: TargetMBB),
2112 N3: DAG.getBasicBlock(MBB: SuccessorColorMBB));
2113 DAG.setRoot(Ret);
2114}
2115
2116void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2117 // Don't emit any special code for the cleanuppad instruction. It just marks
2118 // the start of an EH scope/funclet.
2119 FuncInfo.MBB->setIsEHScopeEntry();
2120 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2121 if (Pers != EHPersonality::Wasm_CXX) {
2122 FuncInfo.MBB->setIsEHFuncletEntry();
2123 FuncInfo.MBB->setIsCleanupFuncletEntry();
2124 }
2125}
2126
2127/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2128/// many places it could ultimately go. In the IR, we have a single unwind
2129/// destination, but in the machine CFG, we enumerate all the possible blocks.
2130/// This function skips over imaginary basic blocks that hold catchswitch
2131/// instructions, and finds all the "real" machine
2132/// basic block destinations. As those destinations may not be successors of
2133/// EHPadBB, here we also calculate the edge probability to those destinations.
2134/// The passed-in Prob is the edge probability to EHPadBB.
2135static void findUnwindDestinations(
2136 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2137 BranchProbability Prob,
2138 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2139 &UnwindDests) {
2140 EHPersonality Personality =
2141 classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2142 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2143 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2144 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2145 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2146
2147 while (EHPadBB) {
2148 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
2149 BasicBlock *NewEHPadBB = nullptr;
2150 if (isa<LandingPadInst>(Val: Pad)) {
2151 // Stop on landingpads. They are not funclets.
2152 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2153 break;
2154 } else if (isa<CleanupPadInst>(Val: Pad)) {
2155 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2156 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2157 // which always catches an exception.
2158 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2159 UnwindDests.back().first->setIsEHScopeEntry();
2160 // In Wasm, EH scopes are not funclets
2161 if (!IsWasmCXX)
2162 UnwindDests.back().first->setIsEHFuncletEntry();
2163 break;
2164 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
2165 // Add the catchpad handlers to the possible destinations.
2166 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2167 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: CatchPadBB), Args&: Prob);
2168 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2169 if (IsMSVCCXX || IsCoreCLR)
2170 UnwindDests.back().first->setIsEHFuncletEntry();
2171 if (!IsSEH)
2172 UnwindDests.back().first->setIsEHScopeEntry();
2173 }
2174 NewEHPadBB = CatchSwitch->getUnwindDest();
2175 } else {
2176 continue;
2177 }
2178
2179 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2180 if (BPI && NewEHPadBB)
2181 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
2182 EHPadBB = NewEHPadBB;
2183 }
2184}
2185
2186void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2187 // Update successor info.
2188 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2189 auto UnwindDest = I.getUnwindDest();
2190 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2191 BranchProbability UnwindDestProb =
2192 (BPI && UnwindDest)
2193 ? BPI->getEdgeProbability(Src: FuncInfo.MBB->getBasicBlock(), Dst: UnwindDest)
2194 : BranchProbability::getZero();
2195 findUnwindDestinations(FuncInfo, EHPadBB: UnwindDest, Prob: UnwindDestProb, UnwindDests);
2196 for (auto &UnwindDest : UnwindDests) {
2197 UnwindDest.first->setIsEHPad();
2198 addSuccessorWithProb(Src: FuncInfo.MBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
2199 }
2200 FuncInfo.MBB->normalizeSuccProbs();
2201
2202 // Create the terminator node.
2203 MachineBasicBlock *CleanupPadMBB =
2204 FuncInfo.getMBB(BB: I.getCleanupPad()->getParent());
2205 SDValue Ret = DAG.getNode(Opcode: ISD::CLEANUPRET, DL: getCurSDLoc(), VT: MVT::Other,
2206 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: CleanupPadMBB));
2207 DAG.setRoot(Ret);
2208}
2209
2210void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2211 report_fatal_error(reason: "visitCatchSwitch not yet implemented!");
2212}
2213
2214void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2215 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2216 auto &DL = DAG.getDataLayout();
2217 SDValue Chain = getControlRoot();
2218 SmallVector<ISD::OutputArg, 8> Outs;
2219 SmallVector<SDValue, 8> OutVals;
2220
2221 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2222 // lower
2223 //
2224 // %val = call <ty> @llvm.experimental.deoptimize()
2225 // ret <ty> %val
2226 //
2227 // differently.
2228 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2229 LowerDeoptimizingReturn();
2230 return;
2231 }
2232
2233 if (!FuncInfo.CanLowerReturn) {
2234 Register DemoteReg = FuncInfo.DemoteRegister;
2235
2236 // Emit a store of the return value through the virtual register.
2237 // Leave Outs empty so that LowerReturn won't try to load return
2238 // registers the usual way.
2239 MVT PtrValueVT = TLI.getPointerTy(DL, AS: DL.getAllocaAddrSpace());
2240 SDValue RetPtr =
2241 DAG.getCopyFromReg(Chain, dl: getCurSDLoc(), Reg: DemoteReg, VT: PtrValueVT);
2242 Type *RetTy = I.getOperand(i_nocapture: 0)->getType();
2243 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
2244 RetPtr =
2245 TLI.annotateStackObjectPointer(Ptr: RetPtr, DAG, DL: getCurSDLoc(), Alignment: BaseAlign);
2246 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2247
2248 SmallVector<EVT, 4> ValueVTs, MemVTs;
2249 SmallVector<uint64_t, 4> Offsets;
2250 ComputeValueVTs(TLI, DL, Ty: RetTy, ValueVTs, MemVTs: &MemVTs, FixedOffsets: &Offsets, StartingOffset: 0);
2251 unsigned NumValues = ValueVTs.size();
2252
2253 SmallVector<SDValue, 4> Chains(NumValues);
2254 for (unsigned i = 0; i != NumValues; ++i) {
2255 // An aggregate return value cannot wrap around the address space, so
2256 // offsets to its parts don't wrap either.
2257 SDValue Ptr = DAG.getObjectPtrOffset(SL: getCurSDLoc(), Ptr: RetPtr,
2258 Offset: TypeSize::getFixed(ExactSize: Offsets[i]));
2259
2260 SDValue Val = RetOp.getValue(R: RetOp.getResNo() + i);
2261 if (MemVTs[i] != ValueVTs[i])
2262 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: getCurSDLoc(), VT: MemVTs[i]);
2263 Chains[i] = DAG.getStore(
2264 Chain, dl: getCurSDLoc(), Val,
2265 // FIXME: better loc info would be nice.
2266 Ptr, PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
2267 Alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]));
2268 }
2269
2270 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: getCurSDLoc(),
2271 VT: MVT::Other, Ops: Chains);
2272 } else if (I.getNumOperands() != 0) {
2273 SmallVector<Type *, 4> Types;
2274 ComputeValueTypes(DL, Ty: I.getOperand(i_nocapture: 0)->getType(), Types);
2275 unsigned NumValues = Types.size();
2276 if (NumValues) {
2277 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2278
2279 const Function *F = I.getParent()->getParent();
2280
2281 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2282 Ty: I.getOperand(i_nocapture: 0)->getType(), CallConv: F->getCallingConv(),
2283 /*IsVarArg*/ isVarArg: false, DL);
2284
2285 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2286 if (F->getAttributes().hasRetAttr(Kind: Attribute::SExt))
2287 ExtendKind = ISD::SIGN_EXTEND;
2288 else if (F->getAttributes().hasRetAttr(Kind: Attribute::ZExt))
2289 ExtendKind = ISD::ZERO_EXTEND;
2290
2291 LLVMContext &Context = F->getContext();
2292 bool RetInReg = F->getAttributes().hasRetAttr(Kind: Attribute::InReg);
2293
2294 for (unsigned j = 0; j != NumValues; ++j) {
2295 EVT VT = TLI.getValueType(DL, Ty: Types[j]);
2296
2297 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2298 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2299
2300 CallingConv::ID CC = F->getCallingConv();
2301
2302 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2303 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2304 SmallVector<SDValue, 4> Parts(NumParts);
2305 getCopyToParts(DAG, DL: getCurSDLoc(),
2306 Val: SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2307 Parts: &Parts[0], NumParts, PartVT, V: &I, CallConv: CC, ExtendKind);
2308
2309 // 'inreg' on function refers to return value
2310 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2311 if (RetInReg)
2312 Flags.setInReg();
2313
2314 if (I.getOperand(i_nocapture: 0)->getType()->isPointerTy()) {
2315 Flags.setPointer();
2316 Flags.setPointerAddrSpace(
2317 cast<PointerType>(Val: I.getOperand(i_nocapture: 0)->getType())->getAddressSpace());
2318 }
2319
2320 if (NeedsRegBlock) {
2321 Flags.setInConsecutiveRegs();
2322 if (j == NumValues - 1)
2323 Flags.setInConsecutiveRegsLast();
2324 }
2325
2326 // Propagate extension type if any
2327 if (ExtendKind == ISD::SIGN_EXTEND)
2328 Flags.setSExt();
2329 else if (ExtendKind == ISD::ZERO_EXTEND)
2330 Flags.setZExt();
2331 else if (F->getAttributes().hasRetAttr(Kind: Attribute::NoExt))
2332 Flags.setNoExt();
2333
2334 for (unsigned i = 0; i < NumParts; ++i) {
2335 Outs.push_back(Elt: ISD::OutputArg(Flags,
2336 Parts[i].getValueType().getSimpleVT(),
2337 VT, Types[j], 0, 0));
2338 OutVals.push_back(Elt: Parts[i]);
2339 }
2340 }
2341 }
2342 }
2343
2344 // Push in swifterror virtual register as the last element of Outs. This makes
2345 // sure swifterror virtual register will be returned in the swifterror
2346 // physical register.
2347 const Function *F = I.getParent()->getParent();
2348 if (TLI.supportSwiftError() &&
2349 F->getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError)) {
2350 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2351 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2352 Flags.setSwiftError();
2353 Outs.push_back(Elt: ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
2354 /*argvt=*/EVT(TLI.getPointerTy(DL)),
2355 PointerType::getUnqual(C&: *DAG.getContext()),
2356 /*origidx=*/1, /*partOffs=*/0));
2357 // Create SDNode for the swifterror virtual register.
2358 OutVals.push_back(
2359 Elt: DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(
2360 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2361 VT: EVT(TLI.getPointerTy(DL))));
2362 }
2363
2364 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2365 CallingConv::ID CallConv =
2366 DAG.getMachineFunction().getFunction().getCallingConv();
2367 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2368 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2369
2370 // Verify that the target's LowerReturn behaved as expected.
2371 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2372 "LowerReturn didn't return a valid chain!");
2373
2374 // Update the DAG with the new chain value resulting from return lowering.
2375 DAG.setRoot(Chain);
2376}
2377
2378/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2379/// created for it, emit nodes to copy the value into the virtual
2380/// registers.
2381void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2382 // Skip empty types
2383 if (V->getType()->isEmptyTy())
2384 return;
2385
2386 auto VMI = FuncInfo.ValueMap.find(Val: V);
2387 if (VMI != FuncInfo.ValueMap.end()) {
2388 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2389 "Unused value assigned virtual registers!");
2390 CopyValueToVirtualRegister(V, Reg: VMI->second);
2391 }
2392}
2393
2394/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2395/// the current basic block, add it to ValueMap now so that we'll get a
2396/// CopyTo/FromReg.
2397void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2398 // No need to export constants.
2399 if (!isa<Instruction>(Val: V) && !isa<Argument>(Val: V)) return;
2400
2401 // Already exported?
2402 if (FuncInfo.isExportedInst(V)) return;
2403
2404 Register Reg = FuncInfo.InitializeRegForValue(V);
2405 CopyValueToVirtualRegister(V, Reg);
2406}
2407
2408bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2409 const BasicBlock *FromBB) {
2410 // The operands of the setcc have to be in this block. We don't know
2411 // how to export them from some other block.
2412 if (const Instruction *VI = dyn_cast<Instruction>(Val: V)) {
2413 // Can export from current BB.
2414 if (VI->getParent() == FromBB)
2415 return true;
2416
2417 // Is already exported, noop.
2418 return FuncInfo.isExportedInst(V);
2419 }
2420
2421 // If this is an argument, we can export it if the BB is the entry block or
2422 // if it is already exported.
2423 if (isa<Argument>(Val: V)) {
2424 if (FromBB->isEntryBlock())
2425 return true;
2426
2427 // Otherwise, can only export this if it is already exported.
2428 return FuncInfo.isExportedInst(V);
2429 }
2430
2431 // Otherwise, constants can always be exported.
2432 return true;
2433}
2434
2435/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2436BranchProbability
2437SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2438 const MachineBasicBlock *Dst) const {
2439 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2440 const BasicBlock *SrcBB = Src->getBasicBlock();
2441 const BasicBlock *DstBB = Dst->getBasicBlock();
2442 if (!BPI) {
2443 // If BPI is not available, set the default probability as 1 / N, where N is
2444 // the number of successors.
2445 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
2446 return BranchProbability(1, SuccSize);
2447 }
2448 return BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
2449}
2450
2451void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2452 MachineBasicBlock *Dst,
2453 BranchProbability Prob) {
2454 if (!FuncInfo.BPI)
2455 Src->addSuccessorWithoutProb(Succ: Dst);
2456 else {
2457 if (Prob.isUnknown())
2458 Prob = getEdgeProbability(Src, Dst);
2459 Src->addSuccessor(Succ: Dst, Prob);
2460 }
2461}
2462
2463static bool InBlock(const Value *V, const BasicBlock *BB) {
2464 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
2465 return I->getParent() == BB;
2466 return true;
2467}
2468
2469/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2470/// This function emits a branch and is used at the leaves of an OR or an
2471/// AND operator tree.
2472void
2473SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2474 MachineBasicBlock *TBB,
2475 MachineBasicBlock *FBB,
2476 MachineBasicBlock *CurBB,
2477 MachineBasicBlock *SwitchBB,
2478 BranchProbability TProb,
2479 BranchProbability FProb,
2480 bool InvertCond) {
2481 const BasicBlock *BB = CurBB->getBasicBlock();
2482
2483 // If the leaf of the tree is a comparison, merge the condition into
2484 // the caseblock.
2485 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
2486 // The operands of the cmp have to be in this block. We don't know
2487 // how to export them from some other block. If this is the first block
2488 // of the sequence, no exporting is needed.
2489 if (CurBB == SwitchBB ||
2490 (isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 0), FromBB: BB) &&
2491 isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 1), FromBB: BB))) {
2492 ISD::CondCode Condition;
2493 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
2494 ICmpInst::Predicate Pred =
2495 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2496 Condition = getICmpCondCode(Pred);
2497 } else {
2498 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
2499 FCmpInst::Predicate Pred =
2500 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2501 Condition = getFCmpCondCode(Pred);
2502 if (FC->hasNoNaNs() ||
2503 (isKnownNeverNaN(V: FC->getOperand(i_nocapture: 0),
2504 SQ: SimplifyQuery(DAG.getDataLayout(), FC)) &&
2505 isKnownNeverNaN(V: FC->getOperand(i_nocapture: 1),
2506 SQ: SimplifyQuery(DAG.getDataLayout(), FC))))
2507 Condition = getFCmpCodeWithoutNaN(CC: Condition);
2508 }
2509
2510 CaseBlock CB(Condition, BOp->getOperand(i_nocapture: 0), BOp->getOperand(i_nocapture: 1), nullptr,
2511 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2512 SL->SwitchCases.push_back(x: CB);
2513 return;
2514 }
2515 }
2516
2517 // Create a CaseBlock record representing this branch.
2518 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2519 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(Context&: *DAG.getContext()),
2520 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2521 SL->SwitchCases.push_back(x: CB);
2522}
2523
2524// Collect dependencies on V recursively. This is used for the cost analysis in
2525// `shouldKeepJumpConditionsTogether`.
2526static bool collectInstructionDeps(
2527 SmallMapVector<const Instruction *, bool, 8> *Deps, const Value *V,
2528 SmallMapVector<const Instruction *, bool, 8> *Necessary = nullptr,
2529 unsigned Depth = 0) {
2530 // Return false if we have an incomplete count.
2531 if (Depth >= SelectionDAG::MaxRecursionDepth)
2532 return false;
2533
2534 auto *I = dyn_cast<Instruction>(Val: V);
2535 if (I == nullptr)
2536 return true;
2537
2538 if (Necessary != nullptr) {
2539 // This instruction is necessary for the other side of the condition so
2540 // don't count it.
2541 if (Necessary->contains(Key: I))
2542 return true;
2543 }
2544
2545 // Already added this dep.
2546 if (!Deps->try_emplace(Key: I, Args: false).second)
2547 return true;
2548
2549 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2550 if (!collectInstructionDeps(Deps, V: I->getOperand(i: OpIdx), Necessary,
2551 Depth: Depth + 1))
2552 return false;
2553 return true;
2554}
2555
2556bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether(
2557 const FunctionLoweringInfo &FuncInfo, const CondBrInst &I,
2558 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2559 TargetLoweringBase::CondMergingParams Params) const {
2560 if (Params.BaseCost < 0)
2561 return false;
2562
2563 // Baseline cost.
2564 InstructionCost CostThresh = Params.BaseCost;
2565
2566 BranchProbabilityInfo *BPI = nullptr;
2567 if (Params.LikelyBias || Params.UnlikelyBias)
2568 BPI = FuncInfo.BPI;
2569 if (BPI != nullptr) {
2570 // See if we are either likely to get an early out or compute both lhs/rhs
2571 // of the condition.
2572 BasicBlock *IfFalse = I.getSuccessor(i: 0);
2573 BasicBlock *IfTrue = I.getSuccessor(i: 1);
2574
2575 std::optional<bool> Likely;
2576 if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfTrue))
2577 Likely = true;
2578 else if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfFalse))
2579 Likely = false;
2580
2581 if (Likely) {
2582 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2583 // Its likely we will have to compute both lhs and rhs of condition
2584 CostThresh += Params.LikelyBias;
2585 else {
2586 if (Params.UnlikelyBias < 0)
2587 return false;
2588 // Its likely we will get an early out.
2589 CostThresh -= Params.UnlikelyBias;
2590 }
2591 }
2592 }
2593
2594 if (CostThresh <= 0)
2595 return false;
2596
2597 // Collect "all" instructions that lhs condition is dependent on.
2598 // Use map for stable iteration (to avoid non-determanism of iteration of
2599 // SmallPtrSet). The `bool` value is just a dummy.
2600 SmallMapVector<const Instruction *, bool, 8> LhsDeps, RhsDeps;
2601 collectInstructionDeps(Deps: &LhsDeps, V: Lhs);
2602 // Collect "all" instructions that rhs condition is dependent on AND are
2603 // dependencies of lhs. This gives us an estimate on which instructions we
2604 // stand to save by splitting the condition.
2605 if (!collectInstructionDeps(Deps: &RhsDeps, V: Rhs, Necessary: &LhsDeps))
2606 return false;
2607 // Add the compare instruction itself unless its a dependency on the LHS.
2608 if (const auto *RhsI = dyn_cast<Instruction>(Val: Rhs))
2609 if (!LhsDeps.contains(Key: RhsI))
2610 RhsDeps.try_emplace(Key: RhsI, Args: false);
2611
2612 InstructionCost CostOfIncluding = 0;
2613 // See if this instruction will need to computed independently of whether RHS
2614 // is.
2615 Value *BrCond = I.getCondition();
2616 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2617 for (const auto *U : Ins->users()) {
2618 // If user is independent of RHS calculation we don't need to count it.
2619 if (auto *UIns = dyn_cast<Instruction>(Val: U))
2620 if (UIns != BrCond && !RhsDeps.contains(Key: UIns))
2621 return false;
2622 }
2623 return true;
2624 };
2625
2626 // Prune instructions from RHS Deps that are dependencies of unrelated
2627 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2628 // arbitrary and just meant to cap the how much time we spend in the pruning
2629 // loop. Its highly unlikely to come into affect.
2630 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2631 // Stop after a certain point. No incorrectness from including too many
2632 // instructions.
2633 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2634 const Instruction *ToDrop = nullptr;
2635 for (const auto &InsPair : RhsDeps) {
2636 if (!ShouldCountInsn(InsPair.first)) {
2637 ToDrop = InsPair.first;
2638 break;
2639 }
2640 }
2641 if (ToDrop == nullptr)
2642 break;
2643 RhsDeps.erase(Key: ToDrop);
2644 }
2645
2646 for (const auto &InsPair : RhsDeps) {
2647 // Finally accumulate latency that we can only attribute to computing the
2648 // RHS condition. Use latency because we are essentially trying to calculate
2649 // the cost of the dependency chain.
2650 // Possible TODO: We could try to estimate ILP and make this more precise.
2651 CostOfIncluding += TTI->getInstructionCost(
2652 U: InsPair.first, CostKind: TargetTransformInfo::TCK_Latency);
2653
2654 if (CostOfIncluding > CostThresh)
2655 return false;
2656 }
2657 return true;
2658}
2659
2660void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2661 MachineBasicBlock *TBB,
2662 MachineBasicBlock *FBB,
2663 MachineBasicBlock *CurBB,
2664 MachineBasicBlock *SwitchBB,
2665 Instruction::BinaryOps Opc,
2666 BranchProbability TProb,
2667 BranchProbability FProb,
2668 bool InvertCond) {
2669 // Skip over not part of the tree and remember to invert op and operands at
2670 // next level.
2671 Value *NotCond;
2672 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
2673 InBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
2674 FindMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2675 InvertCond: !InvertCond);
2676 return;
2677 }
2678
2679 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
2680 const Value *BOpOp0, *BOpOp1;
2681 // Compute the effective opcode for Cond, taking into account whether it needs
2682 // to be inverted, e.g.
2683 // and (not (or A, B)), C
2684 // gets lowered as
2685 // and (and (not A, not B), C)
2686 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2687 if (BOp) {
2688 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2689 ? Instruction::And
2690 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2691 ? Instruction::Or
2692 : (Instruction::BinaryOps)0);
2693 if (InvertCond) {
2694 if (BOpc == Instruction::And)
2695 BOpc = Instruction::Or;
2696 else if (BOpc == Instruction::Or)
2697 BOpc = Instruction::And;
2698 }
2699 }
2700
2701 // If this node is not part of the or/and tree, emit it as a branch.
2702 // Note that all nodes in the tree should have same opcode.
2703 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2704 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2705 !InBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
2706 !InBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
2707 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2708 TProb, FProb, InvertCond);
2709 return;
2710 }
2711
2712 // Create TmpBB after CurBB.
2713 MachineFunction::iterator BBI(CurBB);
2714 MachineFunction &MF = DAG.getMachineFunction();
2715 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
2716 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
2717
2718 if (Opc == Instruction::Or) {
2719 // Codegen X | Y as:
2720 // BB1:
2721 // jmp_if_X TBB
2722 // jmp TmpBB
2723 // TmpBB:
2724 // jmp_if_Y TBB
2725 // jmp FBB
2726 //
2727
2728 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2729 // The requirement is that
2730 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2731 // = TrueProb for original BB.
2732 // Assuming the original probabilities are A and B, one choice is to set
2733 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2734 // A/(1+B) and 2B/(1+B). This choice assumes that
2735 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2736 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2737 // TmpBB, but the math is more complicated.
2738
2739 auto NewTrueProb = TProb / 2;
2740 auto NewFalseProb = TProb / 2 + FProb;
2741 // Emit the LHS condition.
2742 FindMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2743 FProb: NewFalseProb, InvertCond);
2744
2745 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2746 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2747 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2748 // Emit the RHS condition into TmpBB.
2749 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2750 FProb: Probs[1], InvertCond);
2751 } else {
2752 assert(Opc == Instruction::And && "Unknown merge op!");
2753 // Codegen X & Y as:
2754 // BB1:
2755 // jmp_if_X TmpBB
2756 // jmp FBB
2757 // TmpBB:
2758 // jmp_if_Y TBB
2759 // jmp FBB
2760 //
2761 // This requires creation of TmpBB after CurBB.
2762
2763 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2764 // The requirement is that
2765 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2766 // = FalseProb for original BB.
2767 // Assuming the original probabilities are A and B, one choice is to set
2768 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2769 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2770 // TrueProb for BB1 * FalseProb for TmpBB.
2771
2772 auto NewTrueProb = TProb + FProb / 2;
2773 auto NewFalseProb = FProb / 2;
2774 // Emit the LHS condition.
2775 FindMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2776 FProb: NewFalseProb, InvertCond);
2777
2778 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2779 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2780 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2781 // Emit the RHS condition into TmpBB.
2782 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2783 FProb: Probs[1], InvertCond);
2784 }
2785}
2786
2787/// If the set of cases should be emitted as a series of branches, return true.
2788/// If we should emit this as a bunch of and/or'd together conditions, return
2789/// false.
2790bool
2791SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2792 if (Cases.size() != 2) return true;
2793
2794 // If this is two comparisons of the same values or'd or and'd together, they
2795 // will get folded into a single comparison, so don't emit two blocks.
2796 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2797 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2798 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2799 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2800 return false;
2801 }
2802
2803 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2804 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2805 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2806 Cases[0].CC == Cases[1].CC &&
2807 isa<Constant>(Val: Cases[0].CmpRHS) &&
2808 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
2809 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2810 return false;
2811 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2812 return false;
2813 }
2814
2815 return true;
2816}
2817
2818void SelectionDAGBuilder::visitUncondBr(const UncondBrInst &I) {
2819 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2820
2821 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2822
2823 // Update machine-CFG edges.
2824 BrMBB->addSuccessor(Succ: Succ0MBB);
2825
2826 // If this is not a fall-through branch or optimizations are switched off,
2827 // emit the branch.
2828 if (Succ0MBB != NextBlock(MBB: BrMBB) ||
2829 TM.getOptLevel() == CodeGenOptLevel::None) {
2830 auto Br = DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other, N1: getControlRoot(),
2831 N2: DAG.getBasicBlock(MBB: Succ0MBB));
2832 setValue(V: &I, NewN: Br);
2833 DAG.setRoot(Br);
2834 }
2835}
2836
2837void SelectionDAGBuilder::visitCondBr(const CondBrInst &I) {
2838 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2839
2840 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2841
2842 // If this condition is one of the special cases we handle, do special stuff
2843 // now.
2844 const Value *CondVal = I.getCondition();
2845 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 1));
2846
2847 // If this is a series of conditions that are or'd or and'd together, emit
2848 // this as a sequence of branches instead of setcc's with and/or operations.
2849 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2850 // unpredictable branches, and vector extracts because those jumps are likely
2851 // expensive for any target), this should improve performance.
2852 // For example, instead of something like:
2853 // cmp A, B
2854 // C = seteq
2855 // cmp D, E
2856 // F = setle
2857 // or C, F
2858 // jnz foo
2859 // Emit:
2860 // cmp A, B
2861 // je foo
2862 // cmp D, E
2863 // jle foo
2864 bool IsUnpredictable = I.hasMetadata(KindID: LLVMContext::MD_unpredictable);
2865 const Instruction *BOp = dyn_cast<Instruction>(Val: CondVal);
2866 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2867 BOp->hasOneUse() && !IsUnpredictable) {
2868 Value *Vec;
2869 const Value *BOp0, *BOp1;
2870 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2871 if (match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2872 Opcode = Instruction::And;
2873 else if (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2874 Opcode = Instruction::Or;
2875
2876 if (Opcode &&
2877 !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
2878 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value()))) &&
2879 !shouldKeepJumpConditionsTogether(
2880 FuncInfo, I, Opc: Opcode, Lhs: BOp0, Rhs: BOp1,
2881 Params: DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2882 Opcode, BOp0, BOp1, FuncInfo.Fn))) {
2883 FindMergedConditions(Cond: BOp, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: BrMBB, SwitchBB: BrMBB, Opc: Opcode,
2884 TProb: getEdgeProbability(Src: BrMBB, Dst: Succ0MBB),
2885 FProb: getEdgeProbability(Src: BrMBB, Dst: Succ1MBB),
2886 /*InvertCond=*/false);
2887 // If the compares in later blocks need to use values not currently
2888 // exported from this block, export them now. This block should always
2889 // be the first entry.
2890 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2891
2892 // Allow some cases to be rejected.
2893 if (ShouldEmitAsBranches(Cases: SL->SwitchCases)) {
2894 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2895 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpLHS);
2896 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpRHS);
2897 }
2898
2899 // Emit the branch for this block.
2900 visitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: BrMBB);
2901 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
2902 return;
2903 }
2904
2905 // Okay, we decided not to do this, remove any inserted MBB's and clear
2906 // SwitchCases.
2907 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2908 FuncInfo.MF->erase(MBBI: SL->SwitchCases[i].ThisBB);
2909
2910 SL->SwitchCases.clear();
2911 }
2912 }
2913
2914 // Create a CaseBlock record representing this branch.
2915 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(Context&: *DAG.getContext()),
2916 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2917 BranchProbability::getUnknown(), BranchProbability::getUnknown(),
2918 IsUnpredictable);
2919
2920 // Use visitSwitchCase to actually insert the fast branch sequence for this
2921 // cond branch.
2922 visitSwitchCase(CB, SwitchBB: BrMBB);
2923}
2924
2925/// visitSwitchCase - Emits the necessary code to represent a single node in
2926/// the binary search tree resulting from lowering a switch instruction.
2927void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2928 MachineBasicBlock *SwitchBB) {
2929 SDValue Cond;
2930 SDValue CondLHS = getValue(V: CB.CmpLHS);
2931 SDLoc dl = CB.DL;
2932
2933 if (CB.CC == ISD::SETTRUE) {
2934 // Branch or fall through to TrueBB.
2935 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
2936 SwitchBB->normalizeSuccProbs();
2937 if (CB.TrueBB != NextBlock(MBB: SwitchBB)) {
2938 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: getControlRoot(),
2939 N2: DAG.getBasicBlock(MBB: CB.TrueBB)));
2940 }
2941 return;
2942 }
2943
2944 auto &TLI = DAG.getTargetLoweringInfo();
2945 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: CB.CmpLHS->getType());
2946
2947 // Build the setcc now.
2948 if (!CB.CmpMHS) {
2949 // Fold "(X == true)" to X and "(X == false)" to !X to
2950 // handle common cases produced by branch lowering.
2951 if (CB.CmpRHS == ConstantInt::getTrue(Context&: *DAG.getContext()) &&
2952 CB.CC == ISD::SETEQ)
2953 Cond = CondLHS;
2954 else if (CB.CmpRHS == ConstantInt::getFalse(Context&: *DAG.getContext()) &&
2955 CB.CC == ISD::SETEQ) {
2956 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: CondLHS.getValueType());
2957 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CondLHS.getValueType(), N1: CondLHS, N2: True);
2958 } else {
2959 SDValue CondRHS = getValue(V: CB.CmpRHS);
2960
2961 // If a pointer's DAG type is larger than its memory type then the DAG
2962 // values are zero-extended. This breaks signed comparisons so truncate
2963 // back to the underlying type before doing the compare.
2964 if (CondLHS.getValueType() != MemVT) {
2965 CondLHS = DAG.getPtrExtOrTrunc(Op: CondLHS, DL: getCurSDLoc(), VT: MemVT);
2966 CondRHS = DAG.getPtrExtOrTrunc(Op: CondRHS, DL: getCurSDLoc(), VT: MemVT);
2967 }
2968 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: CondLHS, RHS: CondRHS, Cond: CB.CC);
2969 }
2970 } else {
2971 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2972
2973 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
2974 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
2975
2976 SDValue CmpOp = getValue(V: CB.CmpMHS);
2977 EVT VT = CmpOp.getValueType();
2978
2979 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
2980 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: CmpOp, RHS: DAG.getConstant(Val: High, DL: dl, VT),
2981 Cond: ISD::SETLE);
2982 } else {
2983 SDValue SUB = DAG.getNode(Opcode: ISD::SUB, DL: dl,
2984 VT, N1: CmpOp, N2: DAG.getConstant(Val: Low, DL: dl, VT));
2985 Cond = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: SUB,
2986 RHS: DAG.getConstant(Val: High-Low, DL: dl, VT), Cond: ISD::SETULE);
2987 }
2988 }
2989
2990 // Update successor info
2991 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
2992 // TrueBB and FalseBB are always different unless the incoming IR is
2993 // degenerate. This only happens when running llc on weird IR.
2994 if (CB.TrueBB != CB.FalseBB)
2995 addSuccessorWithProb(Src: SwitchBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
2996 SwitchBB->normalizeSuccProbs();
2997
2998 // If the lhs block is the next block, invert the condition so that we can
2999 // fall through to the lhs instead of the rhs block.
3000 if (CB.TrueBB == NextBlock(MBB: SwitchBB)) {
3001 std::swap(a&: CB.TrueBB, b&: CB.FalseBB);
3002 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: Cond.getValueType());
3003 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: Cond.getValueType(), N1: Cond, N2: True);
3004 }
3005
3006 SDNodeFlags Flags;
3007 Flags.setUnpredictable(CB.IsUnpredictable);
3008 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: getControlRoot(),
3009 N2: Cond, N3: DAG.getBasicBlock(MBB: CB.TrueBB), Flags);
3010
3011 setValue(V: CurInst, NewN: BrCond);
3012
3013 // Insert the false branch. Do this even if it's a fall through branch,
3014 // this makes it easier to do DAG optimizations which require inverting
3015 // the branch condition.
3016 BrCond = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrCond,
3017 N2: DAG.getBasicBlock(MBB: CB.FalseBB));
3018
3019 DAG.setRoot(BrCond);
3020}
3021
3022/// visitJumpTable - Emit JumpTable node in the current MBB
3023void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
3024 // Emit the code for the jump table
3025 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3026 assert(JT.Reg && "Should lower JT Header first!");
3027 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DL: DAG.getDataLayout());
3028 SDValue Index = DAG.getCopyFromReg(Chain: getControlRoot(), dl: *JT.SL, Reg: JT.Reg, VT: PTy);
3029 SDValue Table = DAG.getJumpTable(JTI: JT.JTI, VT: PTy);
3030 SDValue BrJumpTable = DAG.getNode(Opcode: ISD::BR_JT, DL: *JT.SL, VT: MVT::Other,
3031 N1: Index.getValue(R: 1), N2: Table, N3: Index);
3032 DAG.setRoot(BrJumpTable);
3033}
3034
3035/// visitJumpTableHeader - This function emits necessary code to produce index
3036/// in the JumpTable from switch case.
3037void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
3038 JumpTableHeader &JTH,
3039 MachineBasicBlock *SwitchBB) {
3040 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3041 const SDLoc &dl = *JT.SL;
3042
3043 // Subtract the lowest switch case value from the value being switched on.
3044 SDValue SwitchOp = getValue(V: JTH.SValue);
3045 EVT VT = SwitchOp.getValueType();
3046 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp,
3047 N2: DAG.getConstant(Val: JTH.First, DL: dl, VT));
3048
3049 // The SDNode we just created, which holds the value being switched on minus
3050 // the smallest case value, needs to be copied to a virtual register so it
3051 // can be used as an index into the jump table in a subsequent basic block.
3052 // This value may be smaller or larger than the target's pointer type, and
3053 // therefore require extension or truncating.
3054 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3055 SwitchOp =
3056 DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
3057
3058 Register JumpTableReg =
3059 FuncInfo.CreateReg(VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
3060 SDValue CopyTo =
3061 DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: JumpTableReg, N: SwitchOp);
3062 JT.Reg = JumpTableReg;
3063
3064 if (!JTH.FallthroughUnreachable) {
3065 // Emit the range check for the jump table, and branch to the default block
3066 // for the switch statement if the value being switched on exceeds the
3067 // largest case in the switch.
3068 SDValue CMP = DAG.getSetCC(
3069 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3070 VT: Sub.getValueType()),
3071 LHS: Sub, RHS: DAG.getConstant(Val: JTH.Last - JTH.First, DL: dl, VT), Cond: ISD::SETUGT);
3072
3073 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl,
3074 VT: MVT::Other, N1: CopyTo, N2: CMP,
3075 N3: DAG.getBasicBlock(MBB: JT.Default));
3076
3077 // Avoid emitting unnecessary branches to the next block.
3078 if (JT.MBB != NextBlock(MBB: SwitchBB))
3079 BrCond = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrCond,
3080 N2: DAG.getBasicBlock(MBB: JT.MBB));
3081
3082 DAG.setRoot(BrCond);
3083 } else {
3084 // Avoid emitting unnecessary branches to the next block.
3085 if (JT.MBB != NextBlock(MBB: SwitchBB))
3086 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: CopyTo,
3087 N2: DAG.getBasicBlock(MBB: JT.MBB)));
3088 else
3089 DAG.setRoot(CopyTo);
3090 }
3091}
3092
3093/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3094/// variable if there exists one.
3095static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
3096 SDValue &Chain) {
3097 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3098 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
3099 EVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout());
3100 MachineFunction &MF = DAG.getMachineFunction();
3101 Value *Global =
3102 TLI.getSDagStackGuard(M: *MF.getFunction().getParent(), Libcalls: DAG.getLibcalls());
3103 MachineSDNode *Node =
3104 DAG.getMachineNode(Opcode: TargetOpcode::LOAD_STACK_GUARD, dl: DL, VT: PtrTy, Op1: Chain);
3105 if (Global) {
3106 MachinePointerInfo MPInfo(Global);
3107 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
3108 MachineMemOperand::MODereferenceable;
3109 MachineMemOperand *MemRef = MF.getMachineMemOperand(
3110 PtrInfo: MPInfo, F: Flags, Size: PtrTy.getSizeInBits() / 8, BaseAlignment: DAG.getEVTAlign(MemoryVT: PtrTy));
3111 DAG.setNodeMemRefs(N: Node, NewMemRefs: {MemRef});
3112 }
3113 if (PtrTy != PtrMemTy)
3114 return DAG.getPtrExtOrTrunc(Op: SDValue(Node, 0), DL, VT: PtrMemTy);
3115 return SDValue(Node, 0);
3116}
3117
3118/// Codegen a new tail for a stack protector check ParentMBB which has had its
3119/// tail spliced into a stack protector check success bb.
3120///
3121/// For a high level explanation of how this fits into the stack protector
3122/// generation see the comment on the declaration of class
3123/// StackProtectorDescriptor.
3124void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
3125 MachineBasicBlock *ParentBB) {
3126
3127 // First create the loads to the guard/stack slot for the comparison.
3128 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3129 auto &DL = DAG.getDataLayout();
3130 EVT PtrTy = TLI.getFrameIndexTy(DL);
3131 EVT PtrMemTy = TLI.getPointerMemTy(DL, AS: DL.getAllocaAddrSpace());
3132
3133 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3134 int FI = MFI.getStackProtectorIndex();
3135
3136 SDValue Guard;
3137 SDLoc dl = getCurSDLoc();
3138 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3139 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3140 Align Align = DL.getPrefTypeAlign(
3141 Ty: PointerType::get(C&: M.getContext(), AddressSpace: DL.getAllocaAddrSpace()));
3142
3143 // Generate code to load the content of the guard slot.
3144 SDValue GuardVal = DAG.getLoad(
3145 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3146 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3147 MMOFlags: MachineMemOperand::MOVolatile);
3148
3149 // If cookie mixing is enabled, unmix the stored GuardVal to get back the
3150 // original cookie for comparison. The prologue stored (FP - Cookie) or
3151 // (FP XOR Cookie), so we apply the same operation again to unmix:
3152 // FP - (FP - Cookie) = Cookie, or (FP XOR Cookie) XOR FP = Cookie.
3153 if (TLI.useStackGuardMixFP())
3154 GuardVal = TLI.emitStackGuardMixFP(DAG, Val: GuardVal, DL: dl);
3155
3156 // If we're using function-based instrumentation, call the guard check
3157 // function
3158 if (SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3159 // Get the guard check function from the target and verify it exists since
3160 // we're using function-based instrumentation
3161 const Function *GuardCheckFn =
3162 TLI.getSSPStackGuardCheck(M, Libcalls: DAG.getLibcalls());
3163 assert(GuardCheckFn && "Guard check function is null");
3164
3165 // The target provides a guard check function to validate the guard value.
3166 // Generate a call to that function with the content of the guard slot as
3167 // argument.
3168 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3169 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3170
3171 TargetLowering::ArgListTy Args;
3172 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(i: 0));
3173 if (GuardCheckFn->hasParamAttribute(ArgNo: 0, Kind: Attribute::AttrKind::InReg))
3174 Entry.IsInReg = true;
3175 Args.push_back(x: Entry);
3176
3177 TargetLowering::CallLoweringInfo CLI(DAG);
3178 CLI.setDebugLoc(getCurSDLoc())
3179 .setChain(DAG.getEntryNode())
3180 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3181 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3182
3183 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3184 DAG.setRoot(Result.second);
3185 return;
3186 }
3187
3188 // Load the fresh guard value for comparison.
3189 // For targets that mix the cookie in LOAD_STACK_GUARD expansion, we need to
3190 // load directly without using LOAD_STACK_GUARD to avoid unwanted mixing.
3191 SDValue Chain = DAG.getEntryNode();
3192 if (TLI.useStackGuardMixFP()) {
3193 // Mixing targets: load cookie directly to avoid mixing in LOAD_STACK_GUARD
3194 if (const Value *IRGuard = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls())) {
3195 SDValue GuardPtr = getValue(V: IRGuard);
3196 Guard = DAG.getLoad(VT: PtrMemTy, dl, Chain, Ptr: GuardPtr,
3197 PtrInfo: MachinePointerInfo(IRGuard, 0), Alignment: Align,
3198 MMOFlags: MachineMemOperand::MOVolatile);
3199 } else {
3200 LLVMContext &Ctx = *DAG.getContext();
3201 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
3202 Guard = DAG.getPOISON(VT: PtrMemTy);
3203 }
3204 } else {
3205 // Non-mixing targets: use LOAD_STACK_GUARD or direct load as usual
3206 if (TLI.useLoadStackGuardNode(M)) {
3207 Guard = getLoadStackGuard(DAG, DL: dl, Chain);
3208 } else {
3209 if (const Value *IRGuard = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls())) {
3210 SDValue GuardPtr = getValue(V: IRGuard);
3211 Guard = DAG.getLoad(VT: PtrMemTy, dl, Chain, Ptr: GuardPtr,
3212 PtrInfo: MachinePointerInfo(IRGuard, 0), Alignment: Align,
3213 MMOFlags: MachineMemOperand::MOVolatile);
3214 } else {
3215 LLVMContext &Ctx = *DAG.getContext();
3216 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
3217 Guard = DAG.getPOISON(VT: PtrMemTy);
3218 }
3219 }
3220 }
3221
3222 // Now both Guard (fresh cookie) and GuardVal (unmixed from stored value)
3223 // contain unmixed cookie values that can be compared directly.
3224
3225 // Perform the comparison via a getsetcc.
3226 SDValue Cmp = DAG.getSetCC(
3227 DL: dl, VT: TLI.getSetCCResultType(DL, Context&: *DAG.getContext(), VT: Guard.getValueType()),
3228 LHS: Guard, RHS: GuardVal, Cond: ISD::SETNE);
3229
3230 // If the guard/stackslot do not equal, branch to failure MBB.
3231 SDValue BrCond = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: getControlRoot(),
3232 N2: Cmp, N3: DAG.getBasicBlock(MBB: SPD.getFailureMBB()));
3233 // Otherwise branch to success MBB.
3234 SDValue Br = DAG.getNode(Opcode: ISD::BR, DL: dl,
3235 VT: MVT::Other, N1: BrCond,
3236 N2: DAG.getBasicBlock(MBB: SPD.getSuccessMBB()));
3237
3238 DAG.setRoot(Br);
3239}
3240
3241/// Codegen the failure basic block for a stack protector check.
3242///
3243/// A failure stack protector machine basic block consists simply of a call to
3244/// __stack_chk_fail().
3245///
3246/// For a high level explanation of how this fits into the stack protector
3247/// generation see the comment on the declaration of class
3248/// StackProtectorDescriptor.
3249void SelectionDAGBuilder::visitSPDescriptorFailure(
3250 StackProtectorDescriptor &SPD) {
3251
3252 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3253 MachineBasicBlock *ParentBB = SPD.getParentMBB();
3254 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3255 SDValue Chain;
3256
3257 // For -Oz builds with a guard check function, we use function-based
3258 // instrumentation. Otherwise, if we have a guard check function, we call it
3259 // in the failure block.
3260 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M, Libcalls: DAG.getLibcalls());
3261 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3262 // First create the loads to the guard/stack slot for the comparison.
3263 auto &DL = DAG.getDataLayout();
3264 EVT PtrTy = TLI.getFrameIndexTy(DL);
3265 EVT PtrMemTy = TLI.getPointerMemTy(DL, AS: DL.getAllocaAddrSpace());
3266
3267 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3268 int FI = MFI.getStackProtectorIndex();
3269
3270 SDLoc dl = getCurSDLoc();
3271 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3272 Align Align = DL.getPrefTypeAlign(
3273 Ty: PointerType::get(C&: M.getContext(), AddressSpace: DL.getAllocaAddrSpace()));
3274
3275 // Generate code to load the content of the guard slot.
3276 SDValue GuardVal = DAG.getLoad(
3277 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3278 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3279 MMOFlags: MachineMemOperand::MOVolatile);
3280
3281 if (TLI.useStackGuardMixFP())
3282 GuardVal = TLI.emitStackGuardMixFP(DAG, Val: GuardVal, DL: dl);
3283
3284 // The target provides a guard check function to validate the guard value.
3285 // Generate a call to that function with the content of the guard slot as
3286 // argument.
3287 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3288 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3289
3290 TargetLowering::ArgListTy Args;
3291 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(i: 0));
3292 if (GuardCheckFn->hasParamAttribute(ArgNo: 0, Kind: Attribute::AttrKind::InReg))
3293 Entry.IsInReg = true;
3294 Args.push_back(x: Entry);
3295
3296 TargetLowering::CallLoweringInfo CLI(DAG);
3297 CLI.setDebugLoc(getCurSDLoc())
3298 .setChain(DAG.getEntryNode())
3299 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3300 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3301
3302 Chain = TLI.LowerCallTo(CLI).second;
3303 } else {
3304 TargetLowering::MakeLibCallOptions CallOptions;
3305 CallOptions.setDiscardResult(true);
3306 Chain = TLI.makeLibCall(DAG, LC: RTLIB::STACKPROTECTOR_CHECK_FAIL, RetVT: MVT::isVoid,
3307 Ops: {}, CallOptions, dl: getCurSDLoc())
3308 .second;
3309 }
3310
3311 // Emit a trap instruction if we are required to do so.
3312 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3313 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3314 Chain = DAG.getNode(Opcode: ISD::TRAP, DL: getCurSDLoc(), VT: MVT::Other, Operand: Chain);
3315
3316 DAG.setRoot(Chain);
3317}
3318
3319/// visitBitTestHeader - This function emits necessary code to produce value
3320/// suitable for "bit tests"
3321void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
3322 MachineBasicBlock *SwitchBB) {
3323 SDLoc dl = getCurSDLoc();
3324
3325 // Subtract the minimum value.
3326 SDValue SwitchOp = getValue(V: B.SValue);
3327 EVT VT = SwitchOp.getValueType();
3328 SDValue RangeSub =
3329 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp, N2: DAG.getConstant(Val: B.First, DL: dl, VT));
3330
3331 // Determine the type of the test operands.
3332 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3333 bool UsePtrType = false;
3334 if (!TLI.isTypeLegal(VT)) {
3335 UsePtrType = true;
3336 } else {
3337 for (const BitTestCase &Case : B.Cases)
3338 if (!isUIntN(N: VT.getSizeInBits(), x: Case.Mask)) {
3339 // Switch table case range are encoded into series of masks.
3340 // Just use pointer type, it's guaranteed to fit.
3341 UsePtrType = true;
3342 break;
3343 }
3344 }
3345 SDValue Sub = RangeSub;
3346 if (UsePtrType) {
3347 VT = TLI.getPointerTy(DL: DAG.getDataLayout());
3348 Sub = DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT);
3349 }
3350
3351 B.RegVT = VT.getSimpleVT();
3352 B.Reg = FuncInfo.CreateReg(VT: B.RegVT);
3353 SDValue CopyTo = DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: B.Reg, N: Sub);
3354
3355 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3356
3357 if (!B.FallthroughUnreachable)
3358 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
3359 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
3360 SwitchBB->normalizeSuccProbs();
3361
3362 SDValue Root = CopyTo;
3363 if (!B.FallthroughUnreachable) {
3364 // Conditional branch to the default block.
3365 SDValue RangeCmp = DAG.getSetCC(DL: dl,
3366 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3367 VT: RangeSub.getValueType()),
3368 LHS: RangeSub, RHS: DAG.getConstant(Val: B.Range, DL: dl, VT: RangeSub.getValueType()),
3369 Cond: ISD::SETUGT);
3370
3371 Root = DAG.getNode(Opcode: ISD::BRCOND, DL: dl, VT: MVT::Other, N1: Root, N2: RangeCmp,
3372 N3: DAG.getBasicBlock(MBB: B.Default));
3373 }
3374
3375 // Avoid emitting unnecessary branches to the next block.
3376 if (MBB != NextBlock(MBB: SwitchBB))
3377 Root = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: Root, N2: DAG.getBasicBlock(MBB));
3378
3379 DAG.setRoot(Root);
3380}
3381
3382/// visitBitTestCase - this function produces one "bit test"
3383void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
3384 MachineBasicBlock *NextMBB,
3385 BranchProbability BranchProbToNext,
3386 Register Reg, BitTestCase &B,
3387 MachineBasicBlock *SwitchBB) {
3388 SDLoc dl = getCurSDLoc();
3389 MVT VT = BB.RegVT;
3390 SDValue ShiftOp = DAG.getCopyFromReg(Chain: getControlRoot(), dl, Reg, VT);
3391 SDValue Cmp;
3392 unsigned PopCount = llvm::popcount(Value: B.Mask);
3393 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3394 if (PopCount == 1) {
3395 // Testing for a single bit; just compare the shift count with what it
3396 // would need to be to shift a 1 bit in that position.
3397 Cmp = DAG.getSetCC(
3398 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3399 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_zero(Val: B.Mask), DL: dl, VT),
3400 Cond: ISD::SETEQ);
3401 } else if (PopCount == BB.Range) {
3402 // There is only one zero bit in the range, test for it directly.
3403 Cmp = DAG.getSetCC(
3404 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3405 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_one(Value: B.Mask), DL: dl, VT), Cond: ISD::SETNE);
3406 } else {
3407 // Make desired shift
3408 SDValue SwitchVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT,
3409 N1: DAG.getConstant(Val: 1, DL: dl, VT), N2: ShiftOp);
3410
3411 // Emit bit tests and jumps
3412 SDValue AndOp = DAG.getNode(Opcode: ISD::AND, DL: dl,
3413 VT, N1: SwitchVal, N2: DAG.getConstant(Val: B.Mask, DL: dl, VT));
3414 Cmp = DAG.getSetCC(
3415 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3416 LHS: AndOp, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
3417 }
3418
3419 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3420 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
3421 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3422 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
3423 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3424 // one as they are relative probabilities (and thus work more like weights),
3425 // and hence we need to normalize them to let the sum of them become one.
3426 SwitchBB->normalizeSuccProbs();
3427
3428 SDValue BrAnd = DAG.getNode(Opcode: ISD::BRCOND, DL: dl,
3429 VT: MVT::Other, N1: getControlRoot(),
3430 N2: Cmp, N3: DAG.getBasicBlock(MBB: B.TargetBB));
3431
3432 // Avoid emitting unnecessary branches to the next block.
3433 if (NextMBB != NextBlock(MBB: SwitchBB))
3434 BrAnd = DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other, N1: BrAnd,
3435 N2: DAG.getBasicBlock(MBB: NextMBB));
3436
3437 DAG.setRoot(BrAnd);
3438}
3439
3440void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3441 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3442
3443 // Retrieve successors. Look through artificial IR level blocks like
3444 // catchswitch for successors.
3445 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
3446 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3447 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(BB: EHPadBB);
3448
3449 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3450 // have to do anything here to lower funclet bundles.
3451 failForInvalidBundles(I, Name: "invokes",
3452 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3453 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3454 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3455 LLVMContext::OB_clang_arc_attachedcall,
3456 LLVMContext::OB_kcfi});
3457
3458 const Value *Callee(I.getCalledOperand());
3459 const Function *Fn = dyn_cast<Function>(Val: Callee);
3460 if (isa<InlineAsm>(Val: Callee))
3461 visitInlineAsm(Call: I, EHPadBB);
3462 else if (Fn && Fn->isIntrinsic()) {
3463 switch (Fn->getIntrinsicID()) {
3464 default:
3465 llvm_unreachable("Cannot invoke this intrinsic");
3466 case Intrinsic::donothing:
3467 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3468 case Intrinsic::seh_try_begin:
3469 case Intrinsic::seh_scope_begin:
3470 case Intrinsic::seh_try_end:
3471 case Intrinsic::seh_scope_end:
3472 if (EHPadMBB)
3473 // a block referenced by EH table
3474 // so dtor-funclet not removed by opts
3475 EHPadMBB->setMachineBlockAddressTaken();
3476 break;
3477 case Intrinsic::experimental_patchpoint_void:
3478 case Intrinsic::experimental_patchpoint:
3479 visitPatchpoint(CB: I, EHPadBB);
3480 break;
3481 case Intrinsic::experimental_gc_statepoint:
3482 LowerStatepoint(I: cast<GCStatepointInst>(Val: I), EHPadBB);
3483 break;
3484 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3485 // but these intrinsics are special because they can be invoked, so we
3486 // manually lower it to a DAG node here.
3487 case Intrinsic::wasm_throw: {
3488 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3489 std::array<SDValue, 4> Ops = {
3490 getControlRoot(), // inchain for the terminator node
3491 DAG.getTargetConstant(Val: Intrinsic::wasm_throw, DL: getCurSDLoc(),
3492 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3493 getValue(V: I.getArgOperand(i: 0)), // tag
3494 getValue(V: I.getArgOperand(i: 1)) // thrown value
3495 };
3496 SDVTList VTs = DAG.getVTList(VTs: ArrayRef<EVT>({MVT::Other})); // outchain
3497 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3498 break;
3499 }
3500 case Intrinsic::wasm_rethrow: {
3501 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3502 std::array<SDValue, 2> Ops = {
3503 getControlRoot(), // inchain for the terminator node
3504 DAG.getTargetConstant(Val: Intrinsic::wasm_rethrow, DL: getCurSDLoc(),
3505 VT: TLI.getPointerTy(DL: DAG.getDataLayout()))};
3506 SDVTList VTs = DAG.getVTList(VTs: ArrayRef<EVT>({MVT::Other})); // outchain
3507 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3508 break;
3509 }
3510 }
3511 } else if (I.hasDeoptState()) {
3512 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3513 // Eventually we will support lowering the @llvm.experimental.deoptimize
3514 // intrinsic, and right now there are no plans to support other intrinsics
3515 // with deopt state.
3516 LowerCallSiteWithDeoptBundle(Call: &I, Callee: getValue(V: Callee), EHPadBB);
3517 } else if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
3518 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), EHPadBB);
3519 } else {
3520 LowerCallTo(CB: I, Callee: getValue(V: Callee), IsTailCall: false, IsMustTailCall: false, EHPadBB);
3521 }
3522
3523 // If the value of the invoke is used outside of its defining block, make it
3524 // available as a virtual register.
3525 // We already took care of the exported value for the statepoint instruction
3526 // during call to the LowerStatepoint.
3527 if (!isa<GCStatepointInst>(Val: I)) {
3528 CopyToExportRegsIfNeeded(V: &I);
3529 }
3530
3531 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3532 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3533 BranchProbability EHPadBBProb =
3534 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3535 : BranchProbability::getZero();
3536 findUnwindDestinations(FuncInfo, EHPadBB, Prob: EHPadBBProb, UnwindDests);
3537
3538 // Update successor info.
3539 addSuccessorWithProb(Src: InvokeMBB, Dst: Return);
3540 for (auto &UnwindDest : UnwindDests) {
3541 UnwindDest.first->setIsEHPad();
3542 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3543 }
3544 InvokeMBB->normalizeSuccProbs();
3545
3546 // Drop into normal successor.
3547 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other, N1: getControlRoot(),
3548 N2: DAG.getBasicBlock(MBB: Return)));
3549}
3550
3551/// The intrinsics currently supported by callbr are implicit control flow
3552/// intrinsics such as amdgcn.kill.
3553/// - they should be called (no "dontcall-" attributes)
3554/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
3555/// - they do not need custom argument handling (no
3556/// TLI.CollectTargetIntrinsicOperands())
3557void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
3558#ifndef NDEBUG
3559 SmallVector<TargetLowering::IntrinsicInfo, 2> Infos;
3560 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3561 Infos, I, DAG.getMachineFunction(), I.getIntrinsicID());
3562 assert(Infos.empty() && "Intrinsic touches memory");
3563#endif
3564
3565 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
3566
3567 SmallVector<SDValue, 8> Ops =
3568 getTargetIntrinsicOperands(I, HasChain, OnlyLoad);
3569 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
3570
3571 // Create the node.
3572 SDValue Result =
3573 getTargetNonMemIntrinsicNode(IntrinsicVT: *I.getType(), HasChain, Ops, VTs);
3574 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
3575
3576 setValue(V: &I, NewN: Result);
3577}
3578
3579void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3580 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3581
3582 if (I.isInlineAsm()) {
3583 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3584 // have to do anything here to lower funclet bundles.
3585 failForInvalidBundles(I, Name: "callbrs",
3586 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_funclet});
3587 visitInlineAsm(Call: I);
3588 } else {
3589 assert(!I.hasOperandBundles() &&
3590 "Can't have operand bundles for intrinsics");
3591 visitCallBrIntrinsic(I);
3592 }
3593 CopyToExportRegsIfNeeded(V: &I);
3594
3595 // Retrieve successors.
3596 SmallPtrSet<BasicBlock *, 8> Dests;
3597 Dests.insert(Ptr: I.getDefaultDest());
3598 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getDefaultDest());
3599
3600 // Update successor info.
3601 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3602 // TODO: For most of the cases where there is an intrinsic callbr, we're
3603 // having exactly one indirect target, which will be unreachable. As soon as
3604 // this changes, we might need to enhance
3605 // Target->setIsInlineAsmBrIndirectTarget or add something similar for
3606 // intrinsic indirect branches.
3607 if (I.isInlineAsm()) {
3608 for (BasicBlock *Dest : I.getIndirectDests()) {
3609 MachineBasicBlock *Target = FuncInfo.getMBB(BB: Dest);
3610 Target->setIsInlineAsmBrIndirectTarget();
3611 // If we introduce a type of asm goto statement that is permitted to use
3612 // an indirect call instruction to jump to its labels, then we should add
3613 // a call to Target->setMachineBlockAddressTaken() here, to mark the
3614 // target block as requiring a BTI.
3615
3616 Target->setLabelMustBeEmitted();
3617 // Don't add duplicate machine successors.
3618 if (Dests.insert(Ptr: Dest).second)
3619 addSuccessorWithProb(Src: CallBrMBB, Dst: Target, Prob: BranchProbability::getZero());
3620 }
3621 }
3622 CallBrMBB->normalizeSuccProbs();
3623
3624 // Drop into default successor.
3625 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(),
3626 VT: MVT::Other, N1: getControlRoot(),
3627 N2: DAG.getBasicBlock(MBB: Return)));
3628}
3629
3630void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3631 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3632}
3633
3634void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3635 assert(FuncInfo.MBB->isEHPad() &&
3636 "Call to landingpad not in landing pad!");
3637
3638 // If there aren't registers to copy the values into (e.g., during SjLj
3639 // exceptions), then don't bother to create these DAG nodes.
3640 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3641 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3642 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3643 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3644 return;
3645
3646 // If landingpad's return type is token type, we don't create DAG nodes
3647 // for its exception pointer and selector value. The extraction of exception
3648 // pointer or selector value from token type landingpads is not currently
3649 // supported.
3650 if (LP.getType()->isTokenTy())
3651 return;
3652
3653 SmallVector<EVT, 2> ValueVTs;
3654 SDLoc dl = getCurSDLoc();
3655 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: LP.getType(), ValueVTs);
3656 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3657
3658 // Get the two live-in registers as SDValues. The physregs have already been
3659 // copied into virtual registers.
3660 SDValue Ops[2];
3661 if (FuncInfo.ExceptionPointerVirtReg) {
3662 Ops[0] = DAG.getZExtOrTrunc(
3663 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3664 Reg: FuncInfo.ExceptionPointerVirtReg,
3665 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3666 DL: dl, VT: ValueVTs[0]);
3667 } else {
3668 Ops[0] = DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3669 }
3670 Ops[1] = DAG.getZExtOrTrunc(
3671 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3672 Reg: FuncInfo.ExceptionSelectorVirtReg,
3673 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3674 DL: dl, VT: ValueVTs[1]);
3675
3676 // Merge into one.
3677 SDValue Res = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
3678 VTList: DAG.getVTList(VTs: ValueVTs), Ops);
3679 setValue(V: &LP, NewN: Res);
3680}
3681
3682void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3683 MachineBasicBlock *Last) {
3684 // Update JTCases.
3685 for (JumpTableBlock &JTB : SL->JTCases)
3686 if (JTB.first.HeaderBB == First)
3687 JTB.first.HeaderBB = Last;
3688
3689 // Update BitTestCases.
3690 for (BitTestBlock &BTB : SL->BitTestCases)
3691 if (BTB.Parent == First)
3692 BTB.Parent = Last;
3693}
3694
3695void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3696 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3697
3698 // Update machine-CFG edges with unique successors.
3699 SmallPtrSet<BasicBlock *, 32> Done;
3700 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3701 BasicBlock *BB = I.getSuccessor(i);
3702 bool Inserted = Done.insert(Ptr: BB).second;
3703 if (!Inserted)
3704 continue;
3705
3706 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3707 addSuccessorWithProb(Src: IndirectBrMBB, Dst: Succ);
3708 }
3709 IndirectBrMBB->normalizeSuccProbs();
3710
3711 DAG.setRoot(DAG.getNode(Opcode: ISD::BRIND, DL: getCurSDLoc(),
3712 VT: MVT::Other, N1: getControlRoot(),
3713 N2: getValue(V: I.getAddress())));
3714}
3715
3716void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3717 if (!I.shouldLowerToTrap(TrapUnreachable: DAG.getTarget().Options.TrapUnreachable,
3718 NoTrapAfterNoreturn: DAG.getTarget().Options.NoTrapAfterNoreturn))
3719 return;
3720
3721 DAG.setRoot(DAG.getNode(Opcode: ISD::TRAP, DL: getCurSDLoc(), VT: MVT::Other, Operand: DAG.getRoot()));
3722}
3723
3724void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3725 SDNodeFlags Flags;
3726 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3727 Flags.copyFMF(FPMO: *FPOp);
3728
3729 SDValue Op = getValue(V: I.getOperand(i: 0));
3730 SDValue UnNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op.getValueType(),
3731 Operand: Op, Flags);
3732 setValue(V: &I, NewN: UnNodeValue);
3733}
3734
3735void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3736 SDNodeFlags Flags;
3737 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(Val: &I)) {
3738 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3739 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3740 }
3741 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(Val: &I))
3742 Flags.setExact(ExactOp->isExact());
3743 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(Val: &I))
3744 Flags.setDisjoint(DisjointOp->isDisjoint());
3745 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3746 Flags.copyFMF(FPMO: *FPOp);
3747
3748 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3749 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3750 SDValue BinNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(),
3751 N1: Op1, N2: Op2, Flags);
3752 setValue(V: &I, NewN: BinNodeValue);
3753}
3754
3755void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3756 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3757 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3758
3759 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3760 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
3761
3762 // Coerce the shift amount to the right type if we can. This exposes the
3763 // truncate or zext to optimization early.
3764 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3765 assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3766 "Unexpected shift type");
3767 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
3768 }
3769
3770 bool nuw = false;
3771 bool nsw = false;
3772 bool exact = false;
3773
3774 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3775
3776 if (const OverflowingBinaryOperator *OFBinOp =
3777 dyn_cast<const OverflowingBinaryOperator>(Val: &I)) {
3778 nuw = OFBinOp->hasNoUnsignedWrap();
3779 nsw = OFBinOp->hasNoSignedWrap();
3780 }
3781 if (const PossiblyExactOperator *ExactOp =
3782 dyn_cast<const PossiblyExactOperator>(Val: &I))
3783 exact = ExactOp->isExact();
3784 }
3785 SDNodeFlags Flags;
3786 Flags.setExact(exact);
3787 Flags.setNoSignedWrap(nsw);
3788 Flags.setNoUnsignedWrap(nuw);
3789 SDValue Res = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1, N2: Op2,
3790 Flags);
3791 setValue(V: &I, NewN: Res);
3792}
3793
3794void SelectionDAGBuilder::visitSDiv(const User &I) {
3795 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3796 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3797
3798 SDNodeFlags Flags;
3799 Flags.setExact(isa<PossiblyExactOperator>(Val: &I) &&
3800 cast<PossiblyExactOperator>(Val: &I)->isExact());
3801 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SDIV, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1,
3802 N2: Op2, Flags));
3803}
3804
3805void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3806 ICmpInst::Predicate predicate = I.getPredicate();
3807 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3808 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3809 ISD::CondCode Opcode = getICmpCondCode(Pred: predicate);
3810
3811 auto &TLI = DAG.getTargetLoweringInfo();
3812 EVT MemVT =
3813 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
3814
3815 // If a pointer's DAG type is larger than its memory type then the DAG values
3816 // are zero-extended. This breaks signed comparisons so truncate back to the
3817 // underlying type before doing the compare.
3818 if (Op1.getValueType() != MemVT) {
3819 Op1 = DAG.getPtrExtOrTrunc(Op: Op1, DL: getCurSDLoc(), VT: MemVT);
3820 Op2 = DAG.getPtrExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: MemVT);
3821 }
3822
3823 SDNodeFlags Flags;
3824 Flags.setSameSign(I.hasSameSign());
3825
3826 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3827 Ty: I.getType());
3828 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Opcode,
3829 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3830}
3831
3832void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3833 FCmpInst::Predicate predicate = I.getPredicate();
3834 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3835 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3836
3837 ISD::CondCode Condition = getFCmpCondCode(Pred: predicate);
3838 auto *FPMO = cast<FPMathOperator>(Val: &I);
3839 if (FPMO->hasNoNaNs() ||
3840 (DAG.isKnownNeverNaN(Op: Op1) && DAG.isKnownNeverNaN(Op: Op2)))
3841 Condition = getFCmpCodeWithoutNaN(CC: Condition);
3842
3843 SDNodeFlags Flags;
3844 Flags.copyFMF(FPMO: *FPMO);
3845
3846 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3847 Ty: I.getType());
3848 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Condition,
3849 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3850}
3851
3852// Check if the condition of the select has one use or two users that are both
3853// selects with the same condition.
3854static bool hasOnlySelectUsers(const Value *Cond) {
3855 return llvm::all_of(Range: Cond->users(), P: [](const Value *V) {
3856 return isa<SelectInst>(Val: V);
3857 });
3858}
3859
3860void SelectionDAGBuilder::visitSelect(const User &I) {
3861 SmallVector<EVT, 4> ValueVTs;
3862 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
3863 ValueVTs);
3864 unsigned NumValues = ValueVTs.size();
3865 if (NumValues == 0) return;
3866
3867 SmallVector<SDValue, 4> Values(NumValues);
3868 SDValue Cond = getValue(V: I.getOperand(i: 0));
3869 SDValue LHSVal = getValue(V: I.getOperand(i: 1));
3870 SDValue RHSVal = getValue(V: I.getOperand(i: 2));
3871 SmallVector<SDValue, 1> BaseOps(1, Cond);
3872 ISD::NodeType OpCode =
3873 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3874
3875 bool IsUnaryAbs = false;
3876 bool Negate = false;
3877
3878 SDNodeFlags Flags;
3879 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3880 Flags.copyFMF(FPMO: *FPOp);
3881
3882 Flags.setUnpredictable(
3883 cast<SelectInst>(Val: I).getMetadata(KindID: LLVMContext::MD_unpredictable));
3884
3885 // Min/max matching is only viable if all output VTs are the same.
3886 if (all_equal(Range&: ValueVTs)) {
3887 EVT VT = ValueVTs[0];
3888 LLVMContext &Ctx = *DAG.getContext();
3889 auto &TLI = DAG.getTargetLoweringInfo();
3890
3891 // We care about the legality of the operation after it has been type
3892 // legalized.
3893 while (TLI.getTypeAction(Context&: Ctx, VT) != TargetLoweringBase::TypeLegal)
3894 VT = TLI.getTypeToTransformTo(Context&: Ctx, VT);
3895
3896 // If the vselect is legal, assume we want to leave this as a vector setcc +
3897 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3898 // min/max is legal on the scalar type.
3899 bool UseScalarMinMax = VT.isVector() &&
3900 !TLI.isOperationLegalOrCustom(Op: ISD::VSELECT, VT);
3901
3902 // ValueTracking's select pattern matching does not account for -0.0,
3903 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3904 // -0.0 is less than +0.0.
3905 const Value *LHS, *RHS;
3906 auto SPR = matchSelectPattern(V: &I, LHS, RHS);
3907 ISD::NodeType Opc = ISD::DELETED_NODE;
3908 switch (SPR.Flavor) {
3909 case SPF_UMAX: Opc = ISD::UMAX; break;
3910 case SPF_UMIN: Opc = ISD::UMIN; break;
3911 case SPF_SMAX: Opc = ISD::SMAX; break;
3912 case SPF_SMIN: Opc = ISD::SMIN; break;
3913 case SPF_FMINNUM:
3914 if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
3915 break;
3916
3917 switch (SPR.NaNBehavior) {
3918 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3919 case SPNB_RETURNS_NAN: break;
3920 case SPNB_RETURNS_OTHER:
3921 Opc = ISD::FMINIMUMNUM;
3922 Flags.setNoSignedZeros(true);
3923 break;
3924 case SPNB_RETURNS_ANY:
3925 if (TLI.isOperationLegalOrCustom(Op: ISD::FMINNUM, VT) ||
3926 (UseScalarMinMax &&
3927 TLI.isOperationLegalOrCustom(Op: ISD::FMINNUM, VT: VT.getScalarType())))
3928 Opc = ISD::FMINNUM;
3929 break;
3930 }
3931 break;
3932 case SPF_FMAXNUM:
3933 if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
3934 break;
3935
3936 switch (SPR.NaNBehavior) {
3937 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3938 case SPNB_RETURNS_NAN: break;
3939 case SPNB_RETURNS_OTHER:
3940 Opc = ISD::FMAXIMUMNUM;
3941 Flags.setNoSignedZeros(true);
3942 break;
3943 case SPNB_RETURNS_ANY:
3944 if (TLI.isOperationLegalOrCustom(Op: ISD::FMAXNUM, VT) ||
3945 (UseScalarMinMax &&
3946 TLI.isOperationLegalOrCustom(Op: ISD::FMAXNUM, VT: VT.getScalarType())))
3947 Opc = ISD::FMAXNUM;
3948 break;
3949 }
3950 break;
3951 case SPF_NABS:
3952 Negate = true;
3953 [[fallthrough]];
3954 case SPF_ABS:
3955 IsUnaryAbs = true;
3956 Opc = ISD::ABS;
3957 break;
3958 default: break;
3959 }
3960
3961 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3962 (TLI.isOperationLegalOrCustom(Op: Opc, VT) ||
3963 (UseScalarMinMax &&
3964 TLI.isOperationLegalOrCustom(Op: Opc, VT: VT.getScalarType()))) &&
3965 // If the underlying comparison instruction is used by any other
3966 // instruction, the consumed instructions won't be destroyed, so it is
3967 // not profitable to convert to a min/max.
3968 hasOnlySelectUsers(Cond: cast<SelectInst>(Val: I).getCondition())) {
3969 OpCode = Opc;
3970 LHSVal = getValue(V: LHS);
3971 RHSVal = getValue(V: RHS);
3972 BaseOps.clear();
3973 }
3974
3975 if (IsUnaryAbs) {
3976 OpCode = Opc;
3977 LHSVal = getValue(V: LHS);
3978 BaseOps.clear();
3979 }
3980 }
3981
3982 if (IsUnaryAbs) {
3983 for (unsigned i = 0; i != NumValues; ++i) {
3984 SDLoc dl = getCurSDLoc();
3985 EVT VT = LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i);
3986 Values[i] =
3987 DAG.getNode(Opcode: OpCode, DL: dl, VT, Operand: LHSVal.getValue(R: LHSVal.getResNo() + i));
3988 if (Negate)
3989 Values[i] = DAG.getNegative(Val: Values[i], DL: dl, VT);
3990 }
3991 } else {
3992 for (unsigned i = 0; i != NumValues; ++i) {
3993 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3994 Ops.push_back(Elt: SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3995 Ops.push_back(Elt: SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3996 Values[i] = DAG.getNode(
3997 Opcode: OpCode, DL: getCurSDLoc(),
3998 VT: LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i), Ops, Flags);
3999 }
4000 }
4001
4002 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4003 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
4004}
4005
4006void SelectionDAGBuilder::visitTrunc(const User &I) {
4007 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
4008 SDValue N = getValue(V: I.getOperand(i: 0));
4009 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4010 Ty: I.getType());
4011 SDNodeFlags Flags;
4012 if (auto *Trunc = dyn_cast<TruncInst>(Val: &I)) {
4013 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
4014 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
4015 }
4016
4017 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4018}
4019
4020void SelectionDAGBuilder::visitZExt(const User &I) {
4021 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4022 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
4023 SDValue N = getValue(V: I.getOperand(i: 0));
4024 auto &TLI = DAG.getTargetLoweringInfo();
4025 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4026
4027 SDNodeFlags Flags;
4028 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(Val: &I))
4029 Flags.setNonNeg(PNI->hasNonNeg());
4030
4031 // Eagerly use nonneg information to canonicalize towards sign_extend if
4032 // that is the target's preference.
4033 // TODO: Let the target do this later.
4034 if (Flags.hasNonNeg() &&
4035 TLI.isSExtCheaperThanZExt(FromTy: N.getValueType(), ToTy: DestVT)) {
4036 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4037 return;
4038 }
4039
4040 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4041}
4042
4043void SelectionDAGBuilder::visitSExt(const User &I) {
4044 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4045 // SExt also can't be a cast to bool for same reason. So, nothing much to do
4046 SDValue N = getValue(V: I.getOperand(i: 0));
4047 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4048 Ty: I.getType());
4049 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4050}
4051
4052void SelectionDAGBuilder::visitFPTrunc(const User &I) {
4053 // FPTrunc is never a no-op cast, no need to check
4054 SDValue N = getValue(V: I.getOperand(i: 0));
4055 SDLoc dl = getCurSDLoc();
4056 SDNodeFlags Flags;
4057 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
4058 Flags.copyFMF(FPMO: *FPOp);
4059 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4060 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4061 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: DestVT, N1: N,
4062 N2: DAG.getTargetConstant(
4063 Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4064 Flags));
4065}
4066
4067void SelectionDAGBuilder::visitFPExt(const User &I) {
4068 // FPExt is never a no-op cast, no need to check
4069 SDValue N = getValue(V: I.getOperand(i: 0));
4070 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4071 Ty: I.getType());
4072 SDNodeFlags Flags;
4073 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
4074 Flags.copyFMF(FPMO: *FPOp);
4075 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4076}
4077
4078void SelectionDAGBuilder::visitFPToUI(const User &I) {
4079 // FPToUI is never a no-op cast, no need to check
4080 SDValue N = getValue(V: I.getOperand(i: 0));
4081 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4082 Ty: I.getType());
4083 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4084}
4085
4086void SelectionDAGBuilder::visitFPToSI(const User &I) {
4087 // FPToSI is never a no-op cast, no need to check
4088 SDValue N = getValue(V: I.getOperand(i: 0));
4089 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4090 Ty: I.getType());
4091 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
4092}
4093
4094void SelectionDAGBuilder::visitUIToFP(const User &I) {
4095 // UIToFP is never a no-op cast, no need to check
4096 SDValue N = getValue(V: I.getOperand(i: 0));
4097 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4098 Ty: I.getType());
4099 SDNodeFlags Flags;
4100 Flags.setNonNeg(cast<PossiblyNonNegInst>(Val: &I)->hasNonNeg());
4101 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
4102
4103 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4104}
4105
4106void SelectionDAGBuilder::visitSIToFP(const User &I) {
4107 // SIToFP is never a no-op cast, no need to check
4108 SDValue N = getValue(V: I.getOperand(i: 0));
4109 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4110 Ty: I.getType());
4111 SDNodeFlags Flags;
4112 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
4113
4114 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
4115}
4116
4117void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
4118 SDValue N = getValue(V: I.getOperand(i: 0));
4119 // By definition the type of the ptrtoaddr must be equal to the address type.
4120 const auto &TLI = DAG.getTargetLoweringInfo();
4121 EVT AddrVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4122 // The address width must be smaller or equal to the pointer representation
4123 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).
4124 N = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: AddrVT, Operand: N);
4125 setValue(V: &I, NewN: N);
4126}
4127
4128void SelectionDAGBuilder::visitPtrToInt(const User &I) {
4129 // What to do depends on the size of the integer and the size of the pointer.
4130 // We can either truncate, zero extend, or no-op, accordingly.
4131 SDValue N = getValue(V: I.getOperand(i: 0));
4132 auto &TLI = DAG.getTargetLoweringInfo();
4133 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4134 Ty: I.getType());
4135 EVT PtrMemVT =
4136 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i: 0)->getType());
4137 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
4138 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
4139 setValue(V: &I, NewN: N);
4140}
4141
4142void SelectionDAGBuilder::visitIntToPtr(const User &I) {
4143 // What to do depends on the size of the integer and the size of the pointer.
4144 // We can either truncate, zero extend, or no-op, accordingly.
4145 SDValue N = getValue(V: I.getOperand(i: 0));
4146 auto &TLI = DAG.getTargetLoweringInfo();
4147 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4148 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4149 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
4150 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
4151 setValue(V: &I, NewN: N);
4152}
4153
4154void SelectionDAGBuilder::visitBitCast(const User &I) {
4155 SDValue N = getValue(V: I.getOperand(i: 0));
4156 SDLoc dl = getCurSDLoc();
4157 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
4158 Ty: I.getType());
4159
4160 // BitCast assures us that source and destination are the same size so this is
4161 // either a BITCAST or a no-op.
4162 if (DestVT != N.getValueType())
4163 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITCAST, DL: dl,
4164 VT: DestVT, Operand: N)); // convert types.
4165 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
4166 // might fold any kind of constant expression to an integer constant and that
4167 // is not what we are looking for. Only recognize a bitcast of a genuine
4168 // constant integer as an opaque constant.
4169 else if(ConstantInt *C = dyn_cast<ConstantInt>(Val: I.getOperand(i: 0)))
4170 setValue(V: &I, NewN: DAG.getConstant(Val: C->getValue(), DL: dl, VT: DestVT, /*isTarget=*/false,
4171 /*isOpaque*/true));
4172 else
4173 setValue(V: &I, NewN: N); // noop cast.
4174}
4175
4176void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
4177 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4178 const Value *SV = I.getOperand(i: 0);
4179 SDValue N = getValue(V: SV);
4180 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4181
4182 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
4183 unsigned DestAS = I.getType()->getPointerAddressSpace();
4184
4185 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
4186 N = DAG.getAddrSpaceCast(dl: getCurSDLoc(), VT: DestVT, Ptr: N, SrcAS, DestAS);
4187
4188 setValue(V: &I, NewN: N);
4189}
4190
4191void SelectionDAGBuilder::visitInsertElement(const User &I) {
4192 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4193 SDValue InVec = getValue(V: I.getOperand(i: 0));
4194 SDValue InVal = getValue(V: I.getOperand(i: 1));
4195 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 2)), DL: getCurSDLoc(),
4196 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
4197 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: getCurSDLoc(),
4198 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
4199 N1: InVec, N2: InVal, N3: InIdx));
4200}
4201
4202void SelectionDAGBuilder::visitExtractElement(const User &I) {
4203 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4204 SDValue InVec = getValue(V: I.getOperand(i: 0));
4205 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 1)), DL: getCurSDLoc(),
4206 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
4207 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: getCurSDLoc(),
4208 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
4209 N1: InVec, N2: InIdx));
4210}
4211
4212void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4213 SDValue Src1 = getValue(V: I.getOperand(i: 0));
4214 SDValue Src2 = getValue(V: I.getOperand(i: 1));
4215 ArrayRef<int> Mask;
4216 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &I))
4217 Mask = SVI->getShuffleMask();
4218 else
4219 Mask = cast<ConstantExpr>(Val: I).getShuffleMask();
4220 SDLoc DL = getCurSDLoc();
4221 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4222 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4223 EVT SrcVT = Src1.getValueType();
4224
4225 if (all_of(Range&: Mask, P: equal_to(Arg: 0)) && VT.isScalableVector()) {
4226 // Canonical splat form of first element of first input vector.
4227 SDValue FirstElt =
4228 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SrcVT.getScalarType(), N1: Src1,
4229 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4230 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: FirstElt));
4231 return;
4232 }
4233
4234 // For now, we only handle splats for scalable vectors.
4235 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4236 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4237 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4238
4239 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4240 unsigned MaskNumElts = Mask.size();
4241
4242 if (SrcNumElts == MaskNumElts) {
4243 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask));
4244 return;
4245 }
4246
4247 // Normalize the shuffle vector since mask and vector length don't match.
4248 if (SrcNumElts < MaskNumElts) {
4249 // Mask is longer than the source vectors. We can use concatenate vector to
4250 // make the mask and vectors lengths match.
4251
4252 if (MaskNumElts % SrcNumElts == 0) {
4253 // Mask length is a multiple of the source vector length.
4254 // Check if the shuffle is some kind of concatenation of the input
4255 // vectors.
4256 unsigned NumConcat = MaskNumElts / SrcNumElts;
4257 bool IsConcat = true;
4258 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4259 for (unsigned i = 0; i != MaskNumElts; ++i) {
4260 int Idx = Mask[i];
4261 if (Idx < 0)
4262 continue;
4263 // Ensure the indices in each SrcVT sized piece are sequential and that
4264 // the same source is used for the whole piece.
4265 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4266 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4267 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4268 IsConcat = false;
4269 break;
4270 }
4271 // Remember which source this index came from.
4272 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4273 }
4274
4275 // The shuffle is concatenating multiple vectors together. Just emit
4276 // a CONCAT_VECTORS operation.
4277 if (IsConcat) {
4278 SmallVector<SDValue, 8> ConcatOps;
4279 for (auto Src : ConcatSrcs) {
4280 if (Src < 0)
4281 ConcatOps.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
4282 else if (Src == 0)
4283 ConcatOps.push_back(Elt: Src1);
4284 else
4285 ConcatOps.push_back(Elt: Src2);
4286 }
4287 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps));
4288 return;
4289 }
4290 }
4291
4292 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
4293 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4294 EVT PaddedVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(),
4295 NumElements: PaddedMaskNumElts);
4296
4297 // Pad both vectors with undefs to make them the same length as the mask.
4298 SDValue UndefVal = DAG.getUNDEF(VT: SrcVT);
4299
4300 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4301 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4302 MOps1[0] = Src1;
4303 MOps2[0] = Src2;
4304
4305 Src1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps1);
4306 Src2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps2);
4307
4308 // Readjust mask for new input vector length.
4309 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4310 for (unsigned i = 0; i != MaskNumElts; ++i) {
4311 int Idx = Mask[i];
4312 if (Idx >= (int)SrcNumElts)
4313 Idx -= SrcNumElts - PaddedMaskNumElts;
4314 MappedOps[i] = Idx;
4315 }
4316
4317 SDValue Result = DAG.getVectorShuffle(VT: PaddedVT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps);
4318
4319 // If the concatenated vector was padded, extract a subvector with the
4320 // correct number of elements.
4321 if (MaskNumElts != PaddedMaskNumElts)
4322 Result = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Result,
4323 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4324
4325 setValue(V: &I, NewN: Result);
4326 return;
4327 }
4328
4329 assert(SrcNumElts > MaskNumElts);
4330
4331 // Analyze the access pattern of the vector to see if we can extract
4332 // two subvectors and do the shuffle.
4333 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4334 bool CanExtract = true;
4335 for (int Idx : Mask) {
4336 unsigned Input = 0;
4337 if (Idx < 0)
4338 continue;
4339
4340 if (Idx >= (int)SrcNumElts) {
4341 Input = 1;
4342 Idx -= SrcNumElts;
4343 }
4344
4345 // If all the indices come from the same MaskNumElts sized portion of
4346 // the sources we can use extract. Also make sure the extract wouldn't
4347 // extract past the end of the source.
4348 int NewStartIdx = alignDown(Value: Idx, Align: MaskNumElts);
4349 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4350 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4351 CanExtract = false;
4352 // Make sure we always update StartIdx as we use it to track if all
4353 // elements are undef.
4354 StartIdx[Input] = NewStartIdx;
4355 }
4356
4357 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4358 setValue(V: &I, NewN: DAG.getUNDEF(VT)); // Vectors are not used.
4359 return;
4360 }
4361 if (CanExtract) {
4362 // Extract appropriate subvector and generate a vector shuffle
4363 for (unsigned Input = 0; Input < 2; ++Input) {
4364 SDValue &Src = Input == 0 ? Src1 : Src2;
4365 if (StartIdx[Input] < 0)
4366 Src = DAG.getUNDEF(VT);
4367 else {
4368 Src = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Src,
4369 N2: DAG.getVectorIdxConstant(Val: StartIdx[Input], DL));
4370 }
4371 }
4372
4373 // Calculate new mask.
4374 SmallVector<int, 8> MappedOps(Mask);
4375 for (int &Idx : MappedOps) {
4376 if (Idx >= (int)SrcNumElts)
4377 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4378 else if (Idx >= 0)
4379 Idx -= StartIdx[0];
4380 }
4381
4382 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps));
4383 return;
4384 }
4385
4386 // We can't use either concat vectors or extract subvectors so fall back to
4387 // replacing the shuffle with extract and build vector.
4388 // to insert and build vector.
4389 EVT EltVT = VT.getVectorElementType();
4390 SmallVector<SDValue,8> Ops;
4391 for (int Idx : Mask) {
4392 SDValue Res;
4393
4394 if (Idx < 0) {
4395 Res = DAG.getUNDEF(VT: EltVT);
4396 } else {
4397 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4398 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4399
4400 Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Src,
4401 N2: DAG.getVectorIdxConstant(Val: Idx, DL));
4402 }
4403
4404 Ops.push_back(Elt: Res);
4405 }
4406
4407 setValue(V: &I, NewN: DAG.getBuildVector(VT, DL, Ops));
4408}
4409
4410void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4411 ArrayRef<unsigned> Indices = I.getIndices();
4412 const Value *Op0 = I.getOperand(i_nocapture: 0);
4413 const Value *Op1 = I.getOperand(i_nocapture: 1);
4414 Type *AggTy = I.getType();
4415 Type *ValTy = Op1->getType();
4416 bool IntoUndef = isa<UndefValue>(Val: Op0);
4417 bool FromUndef = isa<UndefValue>(Val: Op1);
4418
4419 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4420
4421 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4422 SmallVector<EVT, 4> AggValueVTs;
4423 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: AggTy, ValueVTs&: AggValueVTs);
4424 SmallVector<EVT, 4> ValValueVTs;
4425 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4426
4427 unsigned NumAggValues = AggValueVTs.size();
4428 unsigned NumValValues = ValValueVTs.size();
4429 SmallVector<SDValue, 4> Values(NumAggValues);
4430
4431 // Ignore an insertvalue that produces an empty object
4432 if (!NumAggValues) {
4433 setValue(V: &I, NewN: DAG.getUNDEF(VT: MVT(MVT::Other)));
4434 return;
4435 }
4436
4437 SDValue Agg = getValue(V: Op0);
4438 unsigned i = 0;
4439 // Copy the beginning value(s) from the original aggregate.
4440 for (; i != LinearIndex; ++i)
4441 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4442 SDValue(Agg.getNode(), Agg.getResNo() + i);
4443 // Copy values from the inserted value(s).
4444 if (NumValValues) {
4445 SDValue Val = getValue(V: Op1);
4446 for (; i != LinearIndex + NumValValues; ++i)
4447 Values[i] = FromUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4448 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4449 }
4450 // Copy remaining value(s) from the original aggregate.
4451 for (; i != NumAggValues; ++i)
4452 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4453 SDValue(Agg.getNode(), Agg.getResNo() + i);
4454
4455 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4456 VTList: DAG.getVTList(VTs: AggValueVTs), Ops: Values));
4457}
4458
4459void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4460 ArrayRef<unsigned> Indices = I.getIndices();
4461 const Value *Op0 = I.getOperand(i_nocapture: 0);
4462 Type *AggTy = Op0->getType();
4463 Type *ValTy = I.getType();
4464 bool OutOfUndef = isa<UndefValue>(Val: Op0);
4465
4466 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4467
4468 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4469 SmallVector<EVT, 4> ValValueVTs;
4470 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4471
4472 unsigned NumValValues = ValValueVTs.size();
4473
4474 // Ignore a extractvalue that produces an empty object
4475 if (!NumValValues) {
4476 setValue(V: &I, NewN: DAG.getUNDEF(VT: MVT(MVT::Other)));
4477 return;
4478 }
4479
4480 SmallVector<SDValue, 4> Values(NumValValues);
4481
4482 SDValue Agg = getValue(V: Op0);
4483 // Copy out the selected value(s).
4484 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4485 Values[i - LinearIndex] =
4486 OutOfUndef ?
4487 DAG.getUNDEF(VT: Agg.getNode()->getValueType(ResNo: Agg.getResNo() + i)) :
4488 SDValue(Agg.getNode(), Agg.getResNo() + i);
4489
4490 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4491 VTList: DAG.getVTList(VTs: ValValueVTs), Ops: Values));
4492}
4493
4494void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4495 Value *Op0 = I.getOperand(i: 0);
4496 // Note that the pointer operand may be a vector of pointers. Take the scalar
4497 // element which holds a pointer.
4498 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4499 SDValue N = getValue(V: Op0);
4500 SDLoc dl = getCurSDLoc();
4501 auto &TLI = DAG.getTargetLoweringInfo();
4502 GEPNoWrapFlags NW = cast<GEPOperator>(Val: I).getNoWrapFlags();
4503
4504 // For a vector GEP, keep the prefix scalar as long as possible, then
4505 // convert any scalars encountered after the first vector operand to vectors.
4506 bool IsVectorGEP = I.getType()->isVectorTy();
4507 ElementCount VectorElementCount =
4508 IsVectorGEP ? cast<VectorType>(Val: I.getType())->getElementCount()
4509 : ElementCount::getFixed(MinVal: 0);
4510
4511 for (gep_type_iterator GTI = gep_type_begin(GEP: &I), E = gep_type_end(GEP: &I);
4512 GTI != E; ++GTI) {
4513 const Value *Idx = GTI.getOperand();
4514 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4515 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
4516 if (Field) {
4517 // N = N + Offset
4518 uint64_t Offset =
4519 DAG.getDataLayout().getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
4520
4521 // In an inbounds GEP with an offset that is nonnegative even when
4522 // interpreted as signed, assume there is no unsigned overflow.
4523 SDNodeFlags Flags;
4524 if (NW.hasNoUnsignedWrap() ||
4525 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4526 Flags |= SDNodeFlags::NoUnsignedWrap;
4527 Flags.setInBounds(NW.isInBounds());
4528
4529 N = DAG.getMemBasePlusOffset(
4530 Base: N, Offset: DAG.getConstant(Val: Offset, DL: dl, VT: N.getValueType()), DL: dl, Flags);
4531 }
4532 } else {
4533 // IdxSize is the width of the arithmetic according to IR semantics.
4534 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4535 // (and fix up the result later).
4536 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4537 MVT IdxTy = MVT::getIntegerVT(BitWidth: IdxSize);
4538 TypeSize ElementSize =
4539 GTI.getSequentialElementStride(DL: DAG.getDataLayout());
4540 // We intentionally mask away the high bits here; ElementSize may not
4541 // fit in IdxTy.
4542 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4543 /*isSigned=*/false, /*implicitTrunc=*/true);
4544 bool ElementScalable = ElementSize.isScalable();
4545
4546 // If this is a scalar constant or a splat vector of constants,
4547 // handle it quickly.
4548 const auto *C = dyn_cast<Constant>(Val: Idx);
4549 if (C && isa<VectorType>(Val: C->getType()))
4550 C = C->getSplatValue();
4551
4552 const auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
4553 if (CI && CI->isZero())
4554 continue;
4555 if (CI && !ElementScalable) {
4556 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(width: IdxSize);
4557 LLVMContext &Context = *DAG.getContext();
4558 SDValue OffsVal;
4559 if (N.getValueType().isVector())
4560 OffsVal = DAG.getConstant(
4561 Val: Offs, DL: dl, VT: EVT::getVectorVT(Context, VT: IdxTy, EC: VectorElementCount));
4562 else
4563 OffsVal = DAG.getConstant(Val: Offs, DL: dl, VT: IdxTy);
4564
4565 // In an inbounds GEP with an offset that is nonnegative even when
4566 // interpreted as signed, assume there is no unsigned overflow.
4567 SDNodeFlags Flags;
4568 if (NW.hasNoUnsignedWrap() ||
4569 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4570 Flags.setNoUnsignedWrap(true);
4571 Flags.setInBounds(NW.isInBounds());
4572
4573 OffsVal = DAG.getSExtOrTrunc(Op: OffsVal, DL: dl, VT: N.getValueType());
4574
4575 N = DAG.getMemBasePlusOffset(Base: N, Offset: OffsVal, DL: dl, Flags);
4576 continue;
4577 }
4578
4579 // N = N + Idx * ElementMul;
4580 SDValue IdxN = getValue(V: Idx);
4581
4582 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
4583 if (N.getValueType().isVector()) {
4584 EVT VT = EVT::getVectorVT(Context&: *Context, VT: IdxN.getValueType(),
4585 EC: VectorElementCount);
4586 IdxN = DAG.getSplat(VT, DL: dl, Op: IdxN);
4587 } else {
4588 EVT VT =
4589 EVT::getVectorVT(Context&: *Context, VT: N.getValueType(), EC: VectorElementCount);
4590 N = DAG.getSplat(VT, DL: dl, Op: N);
4591 }
4592 }
4593
4594 // If the index is smaller or larger than intptr_t, truncate or extend
4595 // it.
4596 IdxN = DAG.getSExtOrTrunc(Op: IdxN, DL: dl, VT: N.getValueType());
4597
4598 SDNodeFlags ScaleFlags;
4599 // The multiplication of an index by the type size does not wrap the
4600 // pointer index type in a signed sense (mul nsw).
4601 ScaleFlags.setNoSignedWrap(NW.hasNoUnsignedSignedWrap());
4602
4603 // The multiplication of an index by the type size does not wrap the
4604 // pointer index type in an unsigned sense (mul nuw).
4605 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4606
4607 if (ElementScalable) {
4608 EVT VScaleTy = N.getValueType().getScalarType();
4609 SDValue VScale = DAG.getNode(
4610 Opcode: ISD::VSCALE, DL: dl, VT: VScaleTy,
4611 Operand: DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl, VT: VScaleTy));
4612 if (N.getValueType().isVector())
4613 VScale = DAG.getSplatVector(VT: N.getValueType(), DL: dl, Op: VScale);
4614 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: VScale,
4615 Flags: ScaleFlags);
4616 } else {
4617 // If this is a multiply by a power of two, turn it into a shl
4618 // immediately. This is a very common case.
4619 if (ElementMul != 1) {
4620 if (ElementMul.isPowerOf2()) {
4621 unsigned Amt = ElementMul.logBase2();
4622 IdxN = DAG.getNode(
4623 Opcode: ISD::SHL, DL: dl, VT: N.getValueType(), N1: IdxN,
4624 N2: DAG.getShiftAmountConstant(Val: Amt, VT: N.getValueType(), DL: dl),
4625 Flags: ScaleFlags);
4626 } else {
4627 SDValue Scale = DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl,
4628 VT: IdxN.getValueType());
4629 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: Scale,
4630 Flags: ScaleFlags);
4631 }
4632 }
4633 }
4634
4635 // The successive addition of the current address, truncated to the
4636 // pointer index type and interpreted as an unsigned number, and each
4637 // offset, also interpreted as an unsigned number, does not wrap the
4638 // pointer index type (add nuw).
4639 SDNodeFlags AddFlags;
4640 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4641 AddFlags.setInBounds(NW.isInBounds());
4642
4643 N = DAG.getMemBasePlusOffset(Base: N, Offset: IdxN, DL: dl, Flags: AddFlags);
4644 }
4645 }
4646
4647 if (IsVectorGEP && !N.getValueType().isVector()) {
4648 EVT VT = EVT::getVectorVT(Context&: *Context, VT: N.getValueType(), EC: VectorElementCount);
4649 N = DAG.getSplat(VT, DL: dl, Op: N);
4650 }
4651
4652 MVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout(), AS);
4653 MVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout(), AS);
4654 if (IsVectorGEP) {
4655 PtrTy = MVT::getVectorVT(VT: PtrTy, EC: VectorElementCount);
4656 PtrMemTy = MVT::getVectorVT(VT: PtrMemTy, EC: VectorElementCount);
4657 }
4658
4659 if (PtrMemTy != PtrTy && !cast<GEPOperator>(Val: I).isInBounds())
4660 N = DAG.getPtrExtendInReg(Op: N, DL: dl, VT: PtrMemTy);
4661
4662 setValue(V: &I, NewN: N);
4663}
4664
4665void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4666 // If this is a fixed sized alloca in the entry block of the function,
4667 // allocate it statically on the stack.
4668 if (FuncInfo.StaticAllocaMap.count(Val: &I))
4669 return; // getValue will auto-populate this.
4670
4671 SDLoc dl = getCurSDLoc();
4672 Type *Ty = I.getAllocatedType();
4673 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4674 auto &DL = DAG.getDataLayout();
4675 TypeSize TySize = DL.getTypeAllocSize(Ty);
4676 MaybeAlign Alignment = I.getAlign();
4677
4678 SDValue AllocSize = getValue(V: I.getArraySize());
4679
4680 EVT IntPtr = TLI.getPointerTy(DL, AS: I.getAddressSpace());
4681 if (AllocSize.getValueType() != IntPtr)
4682 AllocSize = DAG.getZExtOrTrunc(Op: AllocSize, DL: dl, VT: IntPtr);
4683
4684 AllocSize = DAG.getNode(
4685 Opcode: ISD::MUL, DL: dl, VT: IntPtr, N1: AllocSize,
4686 N2: DAG.getZExtOrTrunc(Op: DAG.getTypeSize(DL: dl, VT: MVT::i64, TS: TySize), DL: dl, VT: IntPtr));
4687
4688 // Handle alignment. If the requested alignment is less than or equal to
4689 // the stack alignment, ignore it. If the size is greater than or equal to
4690 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4691 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4692 if (*Alignment <= StackAlign)
4693 Alignment = std::nullopt;
4694
4695 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4696 // Round the size of the allocation up to the stack alignment size
4697 // by add SA-1 to the size. This doesn't overflow because we're computing
4698 // an address inside an alloca.
4699 AllocSize = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4700 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT: IntPtr),
4701 Flags: SDNodeFlags::NoUnsignedWrap);
4702
4703 // Mask out the low bits for alignment purposes.
4704 AllocSize = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4705 N2: DAG.getSignedConstant(Val: ~StackAlignMask, DL: dl, VT: IntPtr));
4706
4707 SDValue Ops[] = {
4708 getRoot(), AllocSize,
4709 DAG.getConstant(Val: Alignment ? Alignment->value() : 0, DL: dl, VT: IntPtr)};
4710 SDVTList VTs = DAG.getVTList(VT1: AllocSize.getValueType(), VT2: MVT::Other);
4711 SDValue DSA = DAG.getNode(Opcode: ISD::DYNAMIC_STACKALLOC, DL: dl, VTList: VTs, Ops);
4712 setValue(V: &I, NewN: DSA);
4713 DAG.setRoot(DSA.getValue(R: 1));
4714
4715 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4716}
4717
4718static const MDNode *getRangeMetadata(const Instruction &I) {
4719 return I.getMetadata(KindID: LLVMContext::MD_range);
4720}
4721
4722static std::optional<ConstantRange> getRange(const Instruction &I) {
4723 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4724 if (std::optional<ConstantRange> CR = CB->getRange())
4725 return CR;
4726 if (const MDNode *Range = getRangeMetadata(I))
4727 return getConstantRangeFromMetadata(RangeMD: *Range);
4728 return std::nullopt;
4729}
4730
4731static FPClassTest getNoFPClass(const Instruction &I) {
4732 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4733 return CB->getRetNoFPClass();
4734 return fcNone;
4735}
4736
4737void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4738 if (I.isAtomic())
4739 return visitAtomicLoad(I);
4740
4741 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4742 const Value *SV = I.getOperand(i_nocapture: 0);
4743 if (TLI.supportSwiftError()) {
4744 // Swifterror values can come from either a function parameter with
4745 // swifterror attribute or an alloca with swifterror attribute.
4746 if (const Argument *Arg = dyn_cast<Argument>(Val: SV)) {
4747 if (Arg->hasSwiftErrorAttr())
4748 return visitLoadFromSwiftError(I);
4749 }
4750
4751 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: SV)) {
4752 if (Alloca->isSwiftError())
4753 return visitLoadFromSwiftError(I);
4754 }
4755 }
4756
4757 SDValue Ptr = getValue(V: SV);
4758
4759 Type *Ty = I.getType();
4760 SmallVector<EVT, 4> ValueVTs, MemVTs;
4761 SmallVector<TypeSize, 4> Offsets;
4762 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty, ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
4763 unsigned NumValues = ValueVTs.size();
4764 if (NumValues == 0)
4765 return;
4766
4767 Align Alignment = I.getAlign();
4768 AAMDNodes AAInfo = I.getAAMetadata();
4769 const MDNode *Ranges = getRangeMetadata(I);
4770 bool isVolatile = I.isVolatile();
4771 MachineMemOperand::Flags MMOFlags =
4772 TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
4773
4774 SDValue Root;
4775 bool ConstantMemory = false;
4776 if (isVolatile)
4777 // Serialize volatile loads with other side effects.
4778 Root = getRoot();
4779 else if (NumValues > MaxParallelChains)
4780 Root = getMemoryRoot();
4781 else if (BatchAA &&
4782 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(
4783 SV,
4784 LocationSize::precise(Value: DAG.getDataLayout().getTypeStoreSize(Ty)),
4785 AAInfo))) {
4786 // Do not serialize (non-volatile) loads of constant memory with anything.
4787 Root = DAG.getEntryNode();
4788 ConstantMemory = true;
4789 MMOFlags |= MachineMemOperand::MOInvariant;
4790 } else {
4791 // Do not serialize non-volatile loads against each other.
4792 Root = DAG.getRoot();
4793 }
4794
4795 SDLoc dl = getCurSDLoc();
4796
4797 if (isVolatile)
4798 Root = TLI.prepareVolatileOrAtomicLoad(Chain: Root, DL: dl, DAG);
4799
4800 SmallVector<SDValue, 4> Values(NumValues);
4801 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
4802
4803 unsigned ChainI = 0;
4804 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4805 // Serializing loads here may result in excessive register pressure, and
4806 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4807 // could recover a bit by hoisting nodes upward in the chain by recognizing
4808 // they are side-effect free or do not alias. The optimizer should really
4809 // avoid this case by converting large object/array copies to llvm.memcpy
4810 // (MaxParallelChains should always remain as failsafe).
4811 if (ChainI == MaxParallelChains) {
4812 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4813 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4814 Ops: ArrayRef(Chains.data(), ChainI));
4815 Root = Chain;
4816 ChainI = 0;
4817 }
4818
4819 // TODO: MachinePointerInfo only supports a fixed length offset.
4820 MachinePointerInfo PtrInfo =
4821 !Offsets[i].isScalable() || Offsets[i].isZero()
4822 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4823 : MachinePointerInfo();
4824
4825 SDValue A = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
4826 SDValue L = DAG.getLoad(VT: MemVTs[i], dl, Chain: Root, Ptr: A, PtrInfo, Alignment,
4827 MMOFlags, AAInfo, Ranges);
4828 Chains[ChainI] = L.getValue(R: 1);
4829
4830 if (MemVTs[i] != ValueVTs[i])
4831 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT: ValueVTs[i]);
4832
4833 if (MDNode *NoFPClassMD = I.getMetadata(KindID: LLVMContext::MD_nofpclass)) {
4834 uint64_t FPTestInt =
4835 cast<ConstantInt>(
4836 Val: cast<ConstantAsMetadata>(Val: NoFPClassMD->getOperand(I: 0))->getValue())
4837 ->getZExtValue();
4838 if (FPTestInt != fcNone) {
4839 SDValue FPTestConst =
4840 DAG.getTargetConstant(Val: FPTestInt, DL: SDLoc(), VT: MVT::i32);
4841 L = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: L.getValueType(), N1: L,
4842 N2: FPTestConst);
4843 }
4844 }
4845 Values[i] = L;
4846 }
4847
4848 if (!ConstantMemory) {
4849 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4850 Ops: ArrayRef(Chains.data(), ChainI));
4851 if (isVolatile)
4852 DAG.setRoot(Chain);
4853 else
4854 PendingLoads.push_back(Elt: Chain);
4855 }
4856
4857 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
4858 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
4859}
4860
4861void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4862 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4863 "call visitStoreToSwiftError when backend supports swifterror");
4864
4865 SmallVector<EVT, 4> ValueVTs;
4866 SmallVector<uint64_t, 4> Offsets;
4867 const Value *SrcV = I.getOperand(i_nocapture: 0);
4868 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
4869 Ty: SrcV->getType(), ValueVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
4870 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4871 "expect a single EVT for swifterror");
4872
4873 SDValue Src = getValue(V: SrcV);
4874 // Create a virtual register, then update the virtual register.
4875 Register VReg =
4876 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4877 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4878 // Chain can be getRoot or getControlRoot.
4879 SDValue CopyNode = DAG.getCopyToReg(Chain: getRoot(), dl: getCurSDLoc(), Reg: VReg,
4880 N: SDValue(Src.getNode(), Src.getResNo()));
4881 DAG.setRoot(CopyNode);
4882}
4883
4884void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4885 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4886 "call visitLoadFromSwiftError when backend supports swifterror");
4887
4888 assert(!I.isVolatile() &&
4889 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4890 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4891 "Support volatile, non temporal, invariant for load_from_swift_error");
4892
4893 const Value *SV = I.getOperand(i_nocapture: 0);
4894 Type *Ty = I.getType();
4895 assert(
4896 (!BatchAA ||
4897 !BatchAA->pointsToConstantMemory(MemoryLocation(
4898 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4899 I.getAAMetadata()))) &&
4900 "load_from_swift_error should not be constant memory");
4901
4902 SmallVector<EVT, 4> ValueVTs;
4903 SmallVector<uint64_t, 4> Offsets;
4904 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty,
4905 ValueVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
4906 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4907 "expect a single EVT for swifterror");
4908
4909 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4910 SDValue L = DAG.getCopyFromReg(
4911 Chain: getRoot(), dl: getCurSDLoc(),
4912 Reg: SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), VT: ValueVTs[0]);
4913
4914 setValue(V: &I, NewN: L);
4915}
4916
4917void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4918 if (I.isAtomic())
4919 return visitAtomicStore(I);
4920
4921 const Value *SrcV = I.getOperand(i_nocapture: 0);
4922 const Value *PtrV = I.getOperand(i_nocapture: 1);
4923
4924 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4925 if (TLI.supportSwiftError()) {
4926 // Swifterror values can come from either a function parameter with
4927 // swifterror attribute or an alloca with swifterror attribute.
4928 if (const Argument *Arg = dyn_cast<Argument>(Val: PtrV)) {
4929 if (Arg->hasSwiftErrorAttr())
4930 return visitStoreToSwiftError(I);
4931 }
4932
4933 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: PtrV)) {
4934 if (Alloca->isSwiftError())
4935 return visitStoreToSwiftError(I);
4936 }
4937 }
4938
4939 SmallVector<EVT, 4> ValueVTs, MemVTs;
4940 SmallVector<TypeSize, 4> Offsets;
4941 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
4942 Ty: SrcV->getType(), ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
4943 unsigned NumValues = ValueVTs.size();
4944 if (NumValues == 0)
4945 return;
4946
4947 // Get the lowered operands. Note that we do this after
4948 // checking if NumResults is zero, because with zero results
4949 // the operands won't have values in the map.
4950 SDValue Src = getValue(V: SrcV);
4951 SDValue Ptr = getValue(V: PtrV);
4952
4953 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4954 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
4955 SDLoc dl = getCurSDLoc();
4956 Align Alignment = I.getAlign();
4957 AAMDNodes AAInfo = I.getAAMetadata();
4958
4959 auto MMOFlags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
4960
4961 unsigned ChainI = 0;
4962 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4963 // See visitLoad comments.
4964 if (ChainI == MaxParallelChains) {
4965 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4966 Ops: ArrayRef(Chains.data(), ChainI));
4967 Root = Chain;
4968 ChainI = 0;
4969 }
4970
4971 // TODO: MachinePointerInfo only supports a fixed length offset.
4972 MachinePointerInfo PtrInfo =
4973 !Offsets[i].isScalable() || Offsets[i].isZero()
4974 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4975 : MachinePointerInfo();
4976
4977 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
4978 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4979 if (MemVTs[i] != ValueVTs[i])
4980 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVTs[i]);
4981 SDValue St =
4982 DAG.getStore(Chain: Root, dl, Val, Ptr: Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4983 Chains[ChainI] = St;
4984 }
4985
4986 SDValue StoreNode = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
4987 Ops: ArrayRef(Chains.data(), ChainI));
4988 setValue(V: &I, NewN: StoreNode);
4989 DAG.setRoot(StoreNode);
4990}
4991
4992void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4993 bool IsCompressing) {
4994 SDLoc sdl = getCurSDLoc();
4995
4996 Value *Src0Operand = I.getArgOperand(i: 0);
4997 Value *PtrOperand = I.getArgOperand(i: 1);
4998 Value *MaskOperand = I.getArgOperand(i: 2);
4999 Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
5000
5001 SDValue Ptr = getValue(V: PtrOperand);
5002 SDValue Src0 = getValue(V: Src0Operand);
5003 SDValue Mask = getValue(V: MaskOperand);
5004 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
5005
5006 EVT VT = Src0.getValueType();
5007
5008 auto MMOFlags = MachineMemOperand::MOStore;
5009 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5010 MMOFlags |= MachineMemOperand::MONonTemporal;
5011
5012 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5013 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5014 Size: LocationSize::upperBound(Value: VT.getStoreSize()), BaseAlignment: Alignment,
5015 AAInfo: I.getAAMetadata());
5016
5017 const auto &TLI = DAG.getTargetLoweringInfo();
5018
5019 SDValue StoreNode =
5020 !IsCompressing && TTI->hasConditionalLoadStoreForType(
5021 Ty: I.getArgOperand(i: 0)->getType(), /*IsStore=*/true)
5022 ? TLI.visitMaskedStore(DAG, DL: sdl, Chain: getMemoryRoot(), MMO, Ptr, Val: Src0,
5023 Mask)
5024 : DAG.getMaskedStore(Chain: getMemoryRoot(), dl: sdl, Val: Src0, Base: Ptr, Offset, Mask,
5025 MemVT: VT, MMO, AM: ISD::UNINDEXED, /*Truncating=*/IsTruncating: false,
5026 IsCompressing);
5027 DAG.setRoot(StoreNode);
5028 setValue(V: &I, NewN: StoreNode);
5029}
5030
5031// Get a uniform base for the Gather/Scatter intrinsic.
5032// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
5033// We try to represent it as a base pointer + vector of indices.
5034// Usually, the vector of pointers comes from a 'getelementptr' instruction.
5035// The first operand of the GEP may be a single pointer or a vector of pointers
5036// Example:
5037// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
5038// or
5039// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
5040// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
5041//
5042// When the first GEP operand is a single pointer - it is the uniform base we
5043// are looking for. If first operand of the GEP is a splat vector - we
5044// extract the splat value and use it as a uniform base.
5045// In all other cases the function returns 'false'.
5046static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
5047 SDValue &Scale, SelectionDAGBuilder *SDB,
5048 const BasicBlock *CurBB, uint64_t ElemSize) {
5049 SelectionDAG& DAG = SDB->DAG;
5050 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5051 const DataLayout &DL = DAG.getDataLayout();
5052
5053 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
5054
5055 // Handle splat constant pointer.
5056 if (auto *C = dyn_cast<Constant>(Val: Ptr)) {
5057 C = C->getSplatValue();
5058 if (!C)
5059 return false;
5060
5061 Base = SDB->getValue(V: C);
5062
5063 ElementCount NumElts = cast<VectorType>(Val: Ptr->getType())->getElementCount();
5064 EVT VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TLI.getPointerTy(DL), EC: NumElts);
5065 Index = DAG.getConstant(Val: 0, DL: SDB->getCurSDLoc(), VT);
5066 Scale = DAG.getTargetConstant(Val: 1, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
5067 return true;
5068 }
5069
5070 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
5071 if (!GEP || GEP->getParent() != CurBB)
5072 return false;
5073
5074 if (GEP->getNumOperands() != 2)
5075 return false;
5076
5077 const Value *BasePtr = GEP->getPointerOperand();
5078 const Value *IndexVal = GEP->getOperand(i_nocapture: GEP->getNumOperands() - 1);
5079
5080 // Make sure the base is scalar and the index is a vector.
5081 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
5082 return false;
5083
5084 TypeSize ScaleVal = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
5085 if (ScaleVal.isScalable())
5086 return false;
5087
5088 // Target may not support the required addressing mode.
5089 if (ScaleVal != 1 &&
5090 !TLI.isLegalScaleForGatherScatter(Scale: ScaleVal.getFixedValue(), ElemSize))
5091 return false;
5092
5093 Base = SDB->getValue(V: BasePtr);
5094 Index = SDB->getValue(V: IndexVal);
5095
5096 Scale =
5097 DAG.getTargetConstant(Val: ScaleVal, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
5098 return true;
5099}
5100
5101void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
5102 SDLoc sdl = getCurSDLoc();
5103
5104 // llvm.masked.scatter.*(Src0, Ptrs, Mask)
5105 const Value *Ptr = I.getArgOperand(i: 1);
5106 SDValue Src0 = getValue(V: I.getArgOperand(i: 0));
5107 SDValue Mask = getValue(V: I.getArgOperand(i: 2));
5108 EVT VT = Src0.getValueType();
5109 Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
5110 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5111
5112 SDValue Base;
5113 SDValue Index;
5114 SDValue Scale;
5115 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
5116 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5117
5118 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5119 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5120 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOStore,
5121 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: I.getAAMetadata());
5122 if (!UniformBase) {
5123 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5124 Index = getValue(V: Ptr);
5125 Scale =
5126 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5127 }
5128
5129 EVT IdxVT = Index.getValueType();
5130 EVT EltTy = IdxVT.getVectorElementType();
5131 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5132 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
5133 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5134 }
5135
5136 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
5137 SDValue Scatter = DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: VT, dl: sdl,
5138 Ops, MMO, IndexType: ISD::SIGNED_SCALED, IsTruncating: false);
5139 DAG.setRoot(Scatter);
5140 setValue(V: &I, NewN: Scatter);
5141}
5142
5143void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
5144 SDLoc sdl = getCurSDLoc();
5145
5146 Value *PtrOperand = I.getArgOperand(i: 0);
5147 Value *MaskOperand = I.getArgOperand(i: 1);
5148 Value *Src0Operand = I.getArgOperand(i: 2);
5149 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5150
5151 SDValue Ptr = getValue(V: PtrOperand);
5152 SDValue Src0 = getValue(V: Src0Operand);
5153 SDValue Mask = getValue(V: MaskOperand);
5154 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
5155
5156 EVT VT = Src0.getValueType();
5157 AAMDNodes AAInfo = I.getAAMetadata();
5158 const MDNode *Ranges = getRangeMetadata(I);
5159
5160 // Do not serialize masked loads of constant memory with anything.
5161 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
5162 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
5163
5164 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5165
5166 auto MMOFlags = MachineMemOperand::MOLoad;
5167 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
5168 MMOFlags |= MachineMemOperand::MONonTemporal;
5169 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
5170 MMOFlags |= MachineMemOperand::MOInvariant;
5171
5172 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5173 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
5174 Size: LocationSize::upperBound(Value: VT.getStoreSize()), BaseAlignment: Alignment, AAInfo, Ranges);
5175
5176 const auto &TLI = DAG.getTargetLoweringInfo();
5177
5178 // The Load/Res may point to different values and both of them are output
5179 // variables.
5180 SDValue Load;
5181 SDValue Res;
5182 if (!IsExpanding &&
5183 TTI->hasConditionalLoadStoreForType(Ty: Src0Operand->getType(),
5184 /*IsStore=*/false))
5185 Res = TLI.visitMaskedLoad(DAG, DL: sdl, Chain: InChain, MMO, NewLoad&: Load, Ptr, PassThru: Src0, Mask);
5186 else
5187 Res = Load =
5188 DAG.getMaskedLoad(VT, dl: sdl, Chain: InChain, Base: Ptr, Offset, Mask, Src0, MemVT: VT, MMO,
5189 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5190 if (AddToChain)
5191 PendingLoads.push_back(Elt: Load.getValue(R: 1));
5192 setValue(V: &I, NewN: Res);
5193}
5194
5195void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5196 SDLoc sdl = getCurSDLoc();
5197
5198 // @llvm.masked.gather.*(Ptrs, Mask, Src0)
5199 const Value *Ptr = I.getArgOperand(i: 0);
5200 SDValue Src0 = getValue(V: I.getArgOperand(i: 2));
5201 SDValue Mask = getValue(V: I.getArgOperand(i: 1));
5202
5203 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5204 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5205 Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
5206
5207 const MDNode *Ranges = getRangeMetadata(I);
5208
5209 SDValue Root = DAG.getRoot();
5210 SDValue Base;
5211 SDValue Index;
5212 SDValue Scale;
5213 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
5214 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5215 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5216 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5217 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOLoad,
5218 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: I.getAAMetadata(),
5219 Ranges);
5220
5221 if (!UniformBase) {
5222 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5223 Index = getValue(V: Ptr);
5224 Scale =
5225 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5226 }
5227
5228 EVT IdxVT = Index.getValueType();
5229 EVT EltTy = IdxVT.getVectorElementType();
5230 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5231 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
5232 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5233 }
5234
5235 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5236 SDValue Gather =
5237 DAG.getMaskedGather(VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), MemVT: VT, dl: sdl, Ops, MMO,
5238 IndexType: ISD::SIGNED_SCALED, ExtTy: ISD::NON_EXTLOAD);
5239
5240 PendingLoads.push_back(Elt: Gather.getValue(R: 1));
5241 setValue(V: &I, NewN: Gather);
5242}
5243
5244void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5245 SDLoc dl = getCurSDLoc();
5246 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5247 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5248 SyncScope::ID SSID = I.getSyncScopeID();
5249
5250 SDValue InChain = getRoot();
5251
5252 MVT MemVT = getValue(V: I.getCompareOperand()).getSimpleValueType();
5253 SDVTList VTs = DAG.getVTList(VT1: MemVT, VT2: MVT::i1, VT3: MVT::Other);
5254
5255 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5256 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5257
5258 MachineFunction &MF = DAG.getMachineFunction();
5259 MachineMemOperand *MMO =
5260 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags,
5261 Size: MemVT.getStoreSize(), BaseAlignment: I.getAlign(), AAInfo: AAMDNodes(),
5262 Ranges: nullptr, SSID, Ordering: SuccessOrdering, FailureOrdering);
5263
5264 SDValue L = DAG.getAtomicCmpSwap(Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
5265 dl, MemVT, VTs, Chain: InChain,
5266 Ptr: getValue(V: I.getPointerOperand()),
5267 Cmp: getValue(V: I.getCompareOperand()),
5268 Swp: getValue(V: I.getNewValOperand()), MMO);
5269
5270 SDValue OutChain = L.getValue(R: 2);
5271
5272 setValue(V: &I, NewN: L);
5273 DAG.setRoot(OutChain);
5274}
5275
5276void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5277 SDLoc dl = getCurSDLoc();
5278 ISD::NodeType NT;
5279 switch (I.getOperation()) {
5280 default: llvm_unreachable("Unknown atomicrmw operation");
5281 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
5282 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
5283 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
5284 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
5285 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
5286 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
5287 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
5288 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
5289 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
5290 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
5291 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
5292 case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
5293 case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
5294 case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
5295 case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
5296 case AtomicRMWInst::FMaximum:
5297 NT = ISD::ATOMIC_LOAD_FMAXIMUM;
5298 break;
5299 case AtomicRMWInst::FMinimum:
5300 NT = ISD::ATOMIC_LOAD_FMINIMUM;
5301 break;
5302 case AtomicRMWInst::FMaximumNum:
5303 NT = ISD::ATOMIC_LOAD_FMAXIMUMNUM;
5304 break;
5305 case AtomicRMWInst::FMinimumNum:
5306 NT = ISD::ATOMIC_LOAD_FMINIMUMNUM;
5307 break;
5308 case AtomicRMWInst::UIncWrap:
5309 NT = ISD::ATOMIC_LOAD_UINC_WRAP;
5310 break;
5311 case AtomicRMWInst::UDecWrap:
5312 NT = ISD::ATOMIC_LOAD_UDEC_WRAP;
5313 break;
5314 case AtomicRMWInst::USubCond:
5315 NT = ISD::ATOMIC_LOAD_USUB_COND;
5316 break;
5317 case AtomicRMWInst::USubSat:
5318 NT = ISD::ATOMIC_LOAD_USUB_SAT;
5319 break;
5320 }
5321 AtomicOrdering Ordering = I.getOrdering();
5322 SyncScope::ID SSID = I.getSyncScopeID();
5323
5324 SDValue InChain = getRoot();
5325
5326 auto MemVT = getValue(V: I.getValOperand()).getSimpleValueType();
5327 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5328 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5329
5330 MachineFunction &MF = DAG.getMachineFunction();
5331 MachineMemOperand *MMO = MF.getMachineMemOperand(
5332 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5333 BaseAlignment: I.getAlign(), AAInfo: AAMDNodes(), Ranges: nullptr, SSID, Ordering);
5334
5335 SDValue L =
5336 DAG.getAtomic(Opcode: NT, dl, MemVT, Chain: InChain,
5337 Ptr: getValue(V: I.getPointerOperand()), Val: getValue(V: I.getValOperand()),
5338 MMO);
5339
5340 SDValue OutChain = L.getValue(R: 1);
5341
5342 setValue(V: &I, NewN: L);
5343 DAG.setRoot(OutChain);
5344}
5345
5346void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5347 SDLoc dl = getCurSDLoc();
5348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5349 SDValue Ops[3];
5350 Ops[0] = getRoot();
5351 Ops[1] = DAG.getTargetConstant(Val: (unsigned)I.getOrdering(), DL: dl,
5352 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5353 Ops[2] = DAG.getTargetConstant(Val: I.getSyncScopeID(), DL: dl,
5354 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5355 SDValue N = DAG.getNode(Opcode: ISD::ATOMIC_FENCE, DL: dl, VT: MVT::Other, Ops);
5356 setValue(V: &I, NewN: N);
5357 DAG.setRoot(N);
5358}
5359
5360void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5361 SDLoc dl = getCurSDLoc();
5362 AtomicOrdering Order = I.getOrdering();
5363 SyncScope::ID SSID = I.getSyncScopeID();
5364
5365 SDValue InChain = getRoot();
5366
5367 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5368 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5369 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5370
5371 if (!TLI.supportsUnalignedAtomics() &&
5372 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5373 report_fatal_error(reason: "Cannot generate unaligned atomic load");
5374
5375 auto Flags = TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
5376
5377 const MDNode *Ranges = getRangeMetadata(I);
5378 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5379 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5380 BaseAlignment: I.getAlign(), AAInfo: AAMDNodes(), Ranges, SSID, Ordering: Order);
5381
5382 InChain = TLI.prepareVolatileOrAtomicLoad(Chain: InChain, DL: dl, DAG);
5383
5384 SDValue Ptr = getValue(V: I.getPointerOperand());
5385 SDValue L =
5386 DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT, VT: MemVT, Chain: InChain, Ptr, MMO);
5387
5388 SDValue OutChain = L.getValue(R: 1);
5389 if (MemVT != VT)
5390 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT);
5391
5392 setValue(V: &I, NewN: L);
5393 DAG.setRoot(OutChain);
5394}
5395
5396void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5397 SDLoc dl = getCurSDLoc();
5398
5399 AtomicOrdering Ordering = I.getOrdering();
5400 SyncScope::ID SSID = I.getSyncScopeID();
5401
5402 SDValue InChain = getRoot();
5403
5404 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5405 EVT MemVT =
5406 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getValueOperand()->getType());
5407
5408 if (!TLI.supportsUnalignedAtomics() &&
5409 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5410 report_fatal_error(reason: "Cannot generate unaligned atomic store");
5411
5412 auto Flags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
5413
5414 MachineFunction &MF = DAG.getMachineFunction();
5415 MachineMemOperand *MMO = MF.getMachineMemOperand(
5416 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5417 BaseAlignment: I.getAlign(), AAInfo: AAMDNodes(), Ranges: nullptr, SSID, Ordering);
5418
5419 SDValue Val = getValue(V: I.getValueOperand());
5420 if (Val.getValueType() != MemVT)
5421 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVT);
5422 SDValue Ptr = getValue(V: I.getPointerOperand());
5423
5424 SDValue OutChain =
5425 DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT, Chain: InChain, Ptr: Val, Val: Ptr, MMO);
5426
5427 setValue(V: &I, NewN: OutChain);
5428 DAG.setRoot(OutChain);
5429}
5430
5431/// Check if this intrinsic call depends on the chain (1st return value)
5432/// and if it only *loads* memory.
5433/// Ignore the callsite's attributes. A specific call site may be marked with
5434/// readnone, but the lowering code will expect the chain based on the
5435/// definition.
5436std::pair<bool, bool>
5437SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {
5438 const Function *F = I.getCalledFunction();
5439 bool HasChain = !F->doesNotAccessMemory();
5440 bool OnlyLoad =
5441 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5442
5443 return {HasChain, OnlyLoad};
5444}
5445
5446SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
5447 const CallBase &I, bool HasChain, bool OnlyLoad,
5448 TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
5449 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5450
5451 // Build the operand list.
5452 SmallVector<SDValue, 8> Ops;
5453 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5454 if (OnlyLoad) {
5455 // We don't need to serialize loads against other loads.
5456 Ops.push_back(Elt: DAG.getRoot());
5457 } else {
5458 Ops.push_back(Elt: getRoot());
5459 }
5460 }
5461
5462 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5463 if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||
5464 TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)
5465 Ops.push_back(Elt: DAG.getTargetConstant(Val: I.getIntrinsicID(), DL: getCurSDLoc(),
5466 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
5467
5468 // Add all operands of the call to the operand list.
5469 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5470 const Value *Arg = I.getArgOperand(i);
5471 if (!I.paramHasAttr(ArgNo: i, Kind: Attribute::ImmArg)) {
5472 Ops.push_back(Elt: getValue(V: Arg));
5473 continue;
5474 }
5475
5476 // Use TargetConstant instead of a regular constant for immarg.
5477 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: Arg->getType(), AllowUnknown: true);
5478 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg)) {
5479 assert(CI->getBitWidth() <= 64 &&
5480 "large intrinsic immediates not handled");
5481 Ops.push_back(Elt: DAG.getTargetConstant(Val: *CI, DL: SDLoc(), VT));
5482 } else {
5483 Ops.push_back(
5484 Elt: DAG.getTargetConstantFP(Val: *cast<ConstantFP>(Val: Arg), DL: SDLoc(), VT));
5485 }
5486 }
5487
5488 if (std::optional<OperandBundleUse> Bundle =
5489 I.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
5490 auto *Sym = Bundle->Inputs[0].get();
5491 SDValue SDSym = getValue(V: Sym);
5492 SDSym = DAG.getDeactivationSymbol(GV: cast<GlobalValue>(Val: Sym));
5493 Ops.push_back(Elt: SDSym);
5494 }
5495
5496 if (std::optional<OperandBundleUse> Bundle =
5497 I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
5498 Value *Token = Bundle->Inputs[0].get();
5499 SDValue ConvControlToken = getValue(V: Token);
5500 assert(Ops.back().getValueType() != MVT::Glue &&
5501 "Did not expect another glue node here.");
5502 ConvControlToken =
5503 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: {}, VT: MVT::Glue, Operand: ConvControlToken);
5504 Ops.push_back(Elt: ConvControlToken);
5505 }
5506
5507 return Ops;
5508}
5509
5510SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
5511 bool HasChain) {
5512 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5513
5514 SmallVector<EVT, 4> ValueVTs;
5515 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
5516
5517 if (HasChain)
5518 ValueVTs.push_back(Elt: MVT::Other);
5519
5520 return DAG.getVTList(VTs: ValueVTs);
5521}
5522
5523/// Get an INTRINSIC node for a target intrinsic which does not touch memory.
5524SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5525 const Type &IntrinsicVT, bool HasChain, ArrayRef<SDValue> Ops,
5526 const SDVTList &VTs) {
5527 if (!HasChain)
5528 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5529 if (!IntrinsicVT.isVoidTy())
5530 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5531 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops);
5532}
5533
5534/// Set root, convert return type if necessary and check alignment.
5535SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
5536 bool HasChain,
5537 bool OnlyLoad,
5538 SDValue Result) {
5539 if (HasChain) {
5540 SDValue Chain = Result.getValue(R: Result.getNode()->getNumValues() - 1);
5541 if (OnlyLoad)
5542 PendingLoads.push_back(Elt: Chain);
5543 else
5544 DAG.setRoot(Chain);
5545 }
5546
5547 if (I.getType()->isVoidTy())
5548 return Result;
5549
5550 if (MaybeAlign Alignment = I.getRetAlign(); InsertAssertAlign && Alignment) {
5551 // Insert `assertalign` node if there's an alignment.
5552 Result = DAG.getAssertAlign(DL: getCurSDLoc(), V: Result, A: Alignment.valueOrOne());
5553 } else if (!isa<VectorType>(Val: I.getType())) {
5554 Result = lowerRangeToAssertZExt(DAG, I, Op: Result);
5555 }
5556
5557 return Result;
5558}
5559
5560/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5561/// node.
5562void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5563 unsigned Intrinsic) {
5564 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
5565 Intrinsic::ID IntrinsicID = static_cast<Intrinsic::ID>(Intrinsic);
5566
5567 if (!DAG.getMachineFunction().getSubtarget().isIntrinsicSupported(
5568 IntrinsicID: Intrinsic)) {
5569 SDLoc DL = getCurSDLoc();
5570 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupportedTargetIntrinsic(
5571 *I.getFunction(), IntrinsicID, DL.getDebugLoc()));
5572
5573 // The intrinsic is not available on this subtarget. Preserve the chain for
5574 // side-effecting intrinsics and lower any result to poison so that
5575 // compilation can continue and collect further diagnostics.
5576 if (HasChain && !OnlyLoad)
5577 DAG.setRoot(getRoot());
5578
5579 setValueToPoison(V: &I, dl: DL);
5580 return;
5581 }
5582
5583 // Infos is set by getTgtMemIntrinsic.
5584 SmallVector<TargetLowering::IntrinsicInfo> Infos;
5585 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5586 TLI.getTgtMemIntrinsic(Infos, I, MF&: DAG.getMachineFunction(), Intrinsic);
5587 // Use the first (primary) info determines the node opcode.
5588 TargetLowering::IntrinsicInfo *Info = !Infos.empty() ? &Infos[0] : nullptr;
5589
5590 SmallVector<SDValue, 8> Ops =
5591 getTargetIntrinsicOperands(I, HasChain, OnlyLoad, TgtMemIntrinsicInfo: Info);
5592 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
5593
5594 // Propagate fast-math-flags from IR to node(s).
5595 SDNodeFlags Flags;
5596 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
5597 Flags.copyFMF(FPMO: *FPMO);
5598 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5599
5600 // Create the node.
5601 SDValue Result;
5602
5603 // In some cases, custom collection of operands from CallInst I may be needed.
5604 TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);
5605 if (!Infos.empty()) {
5606 // This is target intrinsic that touches memory
5607 // Create MachineMemOperands for each memory access described by the target.
5608 MachineFunction &MF = DAG.getMachineFunction();
5609 SmallVector<MachineMemOperand *> MMOs;
5610 for (const auto &Info : Infos) {
5611 // TODO: We currently just fallback to address space 0 if
5612 // getTgtMemIntrinsic didn't yield anything useful.
5613 MachinePointerInfo MPI;
5614 if (Info.ptrVal)
5615 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5616 else if (Info.fallbackAddressSpace)
5617 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5618 EVT MemVT = Info.memVT;
5619 LocationSize Size = LocationSize::precise(Value: Info.size);
5620 if (Size.hasValue() && !Size.getValue())
5621 Size = LocationSize::precise(Value: MemVT.getStoreSize());
5622 Align Alignment = Info.align.value_or(u: DAG.getEVTAlign(MemoryVT: MemVT));
5623 MachineMemOperand *MMO = MF.getMachineMemOperand(
5624 PtrInfo: MPI, F: Info.flags, Size, BaseAlignment: Alignment, AAInfo: I.getAAMetadata(),
5625 /*Ranges=*/nullptr, SSID: Info.ssid, Ordering: Info.order, FailureOrdering: Info.failureOrder);
5626 MMOs.push_back(Elt: MMO);
5627 }
5628
5629 Result = DAG.getMemIntrinsicNode(Opcode: Info->opc, dl: getCurSDLoc(), VTList: VTs, Ops,
5630 MemVT: Info->memVT, MMOs);
5631 } else {
5632 Result = getTargetNonMemIntrinsicNode(IntrinsicVT: *I.getType(), HasChain, Ops, VTs);
5633 }
5634
5635 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
5636
5637 setValue(V: &I, NewN: Result);
5638}
5639
5640/// GetSignificand - Get the significand and build it into a floating-point
5641/// number with exponent of 1:
5642///
5643/// Op = (Op & 0x007fffff) | 0x3f800000;
5644///
5645/// where Op is the hexadecimal representation of floating point value.
5646static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
5647 SDValue t1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Op,
5648 N2: DAG.getConstant(Val: 0x007fffff, DL: dl, VT: MVT::i32));
5649 SDValue t2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: t1,
5650 N2: DAG.getConstant(Val: 0x3f800000, DL: dl, VT: MVT::i32));
5651 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: t2);
5652}
5653
5654/// GetExponent - Get the exponent:
5655///
5656/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5657///
5658/// where Op is the hexadecimal representation of floating point value.
5659static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
5660 const TargetLowering &TLI, const SDLoc &dl) {
5661 SDValue t0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Op,
5662 N2: DAG.getConstant(Val: 0x7f800000, DL: dl, VT: MVT::i32));
5663 SDValue t1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: t0,
5664 N2: DAG.getShiftAmountConstant(Val: 23, VT: MVT::i32, DL: dl));
5665 SDValue t2 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: t1,
5666 N2: DAG.getConstant(Val: 127, DL: dl, VT: MVT::i32));
5667 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::f32, Operand: t2);
5668}
5669
5670/// getF32Constant - Get 32-bit floating point constant.
5671static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
5672 const SDLoc &dl) {
5673 return DAG.getConstantFP(Val: APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), DL: dl,
5674 VT: MVT::f32);
5675}
5676
5677static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
5678 SelectionDAG &DAG) {
5679 // TODO: What fast-math-flags should be set on the floating-point nodes?
5680
5681 // IntegerPartOfX = ((int32_t)(t0);
5682 SDValue IntegerPartOfX = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: t0);
5683
5684 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5685 SDValue t1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::f32, Operand: IntegerPartOfX);
5686 SDValue X = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0, N2: t1);
5687
5688 // IntegerPartOfX <<= 23;
5689 IntegerPartOfX = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: IntegerPartOfX,
5690 N2: DAG.getShiftAmountConstant(Val: 23, VT: MVT::i32, DL: dl));
5691
5692 SDValue TwoToFractionalPartOfX;
5693 if (LimitFloatPrecision <= 6) {
5694 // For floating-point precision of 6:
5695 //
5696 // TwoToFractionalPartOfX =
5697 // 0.997535578f +
5698 // (0.735607626f + 0.252464424f * x) * x;
5699 //
5700 // error 0.0144103317, which is 6 bits
5701 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5702 N2: getF32Constant(DAG, Flt: 0x3e814304, dl));
5703 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5704 N2: getF32Constant(DAG, Flt: 0x3f3c50c8, dl));
5705 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5706 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5707 N2: getF32Constant(DAG, Flt: 0x3f7f5e7e, dl));
5708 } else if (LimitFloatPrecision <= 12) {
5709 // For floating-point precision of 12:
5710 //
5711 // TwoToFractionalPartOfX =
5712 // 0.999892986f +
5713 // (0.696457318f +
5714 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5715 //
5716 // error 0.000107046256, which is 13 to 14 bits
5717 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5718 N2: getF32Constant(DAG, Flt: 0x3da235e3, dl));
5719 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5720 N2: getF32Constant(DAG, Flt: 0x3e65b8f3, dl));
5721 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5722 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5723 N2: getF32Constant(DAG, Flt: 0x3f324b07, dl));
5724 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5725 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5726 N2: getF32Constant(DAG, Flt: 0x3f7ff8fd, dl));
5727 } else { // LimitFloatPrecision <= 18
5728 // For floating-point precision of 18:
5729 //
5730 // TwoToFractionalPartOfX =
5731 // 0.999999982f +
5732 // (0.693148872f +
5733 // (0.240227044f +
5734 // (0.554906021e-1f +
5735 // (0.961591928e-2f +
5736 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5737 // error 2.47208000*10^(-7), which is better than 18 bits
5738 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5739 N2: getF32Constant(DAG, Flt: 0x3924b03e, dl));
5740 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
5741 N2: getF32Constant(DAG, Flt: 0x3ab24b87, dl));
5742 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5743 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5744 N2: getF32Constant(DAG, Flt: 0x3c1d8c17, dl));
5745 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5746 SDValue t7 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
5747 N2: getF32Constant(DAG, Flt: 0x3d634a1d, dl));
5748 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5749 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
5750 N2: getF32Constant(DAG, Flt: 0x3e75fe14, dl));
5751 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
5752 SDValue t11 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t10,
5753 N2: getF32Constant(DAG, Flt: 0x3f317234, dl));
5754 SDValue t12 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t11, N2: X);
5755 TwoToFractionalPartOfX = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t12,
5756 N2: getF32Constant(DAG, Flt: 0x3f800000, dl));
5757 }
5758
5759 // Add the exponent into the result in integer domain.
5760 SDValue t13 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: TwoToFractionalPartOfX);
5761 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32,
5762 Operand: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: t13, N2: IntegerPartOfX));
5763}
5764
5765/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5766/// limited-precision mode.
5767static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5768 const TargetLowering &TLI, SDNodeFlags Flags) {
5769 if (Op.getValueType() == MVT::f32 &&
5770 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5771
5772 // Put the exponent in the right bit position for later addition to the
5773 // final result:
5774 //
5775 // t0 = Op * log2(e)
5776
5777 // TODO: What fast-math-flags should be set here?
5778 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Op,
5779 N2: DAG.getConstantFP(Val: numbers::log2ef, DL: dl, VT: MVT::f32));
5780 return getLimitedPrecisionExp2(t0, dl, DAG);
5781 }
5782
5783 // No special expansion.
5784 return DAG.getNode(Opcode: ISD::FEXP, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5785}
5786
5787/// expandLog - Lower a log intrinsic. Handles the special sequences for
5788/// limited-precision mode.
5789static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5790 const TargetLowering &TLI, SDNodeFlags Flags) {
5791 // TODO: What fast-math-flags should be set on the floating-point nodes?
5792
5793 if (Op.getValueType() == MVT::f32 &&
5794 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5795 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
5796
5797 // Scale the exponent by log(2).
5798 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
5799 SDValue LogOfExponent =
5800 DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Exp,
5801 N2: DAG.getConstantFP(Val: numbers::ln2f, DL: dl, VT: MVT::f32));
5802
5803 // Get the significand and build it into a floating-point number with
5804 // exponent of 1.
5805 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5806
5807 SDValue LogOfMantissa;
5808 if (LimitFloatPrecision <= 6) {
5809 // For floating-point precision of 6:
5810 //
5811 // LogofMantissa =
5812 // -1.1609546f +
5813 // (1.4034025f - 0.23903021f * x) * x;
5814 //
5815 // error 0.0034276066, which is better than 8 bits
5816 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5817 N2: getF32Constant(DAG, Flt: 0xbe74c456, dl));
5818 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5819 N2: getF32Constant(DAG, Flt: 0x3fb3a2b1, dl));
5820 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5821 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5822 N2: getF32Constant(DAG, Flt: 0x3f949a29, dl));
5823 } else if (LimitFloatPrecision <= 12) {
5824 // For floating-point precision of 12:
5825 //
5826 // LogOfMantissa =
5827 // -1.7417939f +
5828 // (2.8212026f +
5829 // (-1.4699568f +
5830 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5831 //
5832 // error 0.000061011436, which is 14 bits
5833 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5834 N2: getF32Constant(DAG, Flt: 0xbd67b6d6, dl));
5835 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5836 N2: getF32Constant(DAG, Flt: 0x3ee4f4b8, dl));
5837 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5838 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5839 N2: getF32Constant(DAG, Flt: 0x3fbc278b, dl));
5840 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5841 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5842 N2: getF32Constant(DAG, Flt: 0x40348e95, dl));
5843 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5844 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
5845 N2: getF32Constant(DAG, Flt: 0x3fdef31a, dl));
5846 } else { // LimitFloatPrecision <= 18
5847 // For floating-point precision of 18:
5848 //
5849 // LogOfMantissa =
5850 // -2.1072184f +
5851 // (4.2372794f +
5852 // (-3.7029485f +
5853 // (2.2781945f +
5854 // (-0.87823314f +
5855 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5856 //
5857 // error 0.0000023660568, which is better than 18 bits
5858 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5859 N2: getF32Constant(DAG, Flt: 0xbc91e5ac, dl));
5860 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5861 N2: getF32Constant(DAG, Flt: 0x3e4350aa, dl));
5862 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5863 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5864 N2: getF32Constant(DAG, Flt: 0x3f60d3e3, dl));
5865 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5866 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5867 N2: getF32Constant(DAG, Flt: 0x4011cdf0, dl));
5868 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5869 SDValue t7 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
5870 N2: getF32Constant(DAG, Flt: 0x406cfd1c, dl));
5871 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5872 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
5873 N2: getF32Constant(DAG, Flt: 0x408797cb, dl));
5874 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
5875 LogOfMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t10,
5876 N2: getF32Constant(DAG, Flt: 0x4006dcab, dl));
5877 }
5878
5879 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: LogOfMantissa);
5880 }
5881
5882 // No special expansion.
5883 return DAG.getNode(Opcode: ISD::FLOG, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5884}
5885
5886/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5887/// limited-precision mode.
5888static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5889 const TargetLowering &TLI, SDNodeFlags Flags) {
5890 // TODO: What fast-math-flags should be set on the floating-point nodes?
5891
5892 if (Op.getValueType() == MVT::f32 &&
5893 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5894 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
5895
5896 // Get the exponent.
5897 SDValue LogOfExponent = GetExponent(DAG, Op: Op1, TLI, dl);
5898
5899 // Get the significand and build it into a floating-point number with
5900 // exponent of 1.
5901 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5902
5903 // Different possible minimax approximations of significand in
5904 // floating-point for various degrees of accuracy over [1,2].
5905 SDValue Log2ofMantissa;
5906 if (LimitFloatPrecision <= 6) {
5907 // For floating-point precision of 6:
5908 //
5909 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5910 //
5911 // error 0.0049451742, which is more than 7 bits
5912 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5913 N2: getF32Constant(DAG, Flt: 0xbeb08fe0, dl));
5914 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5915 N2: getF32Constant(DAG, Flt: 0x40019463, dl));
5916 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5917 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5918 N2: getF32Constant(DAG, Flt: 0x3fd6633d, dl));
5919 } else if (LimitFloatPrecision <= 12) {
5920 // For floating-point precision of 12:
5921 //
5922 // Log2ofMantissa =
5923 // -2.51285454f +
5924 // (4.07009056f +
5925 // (-2.12067489f +
5926 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5927 //
5928 // error 0.0000876136000, which is better than 13 bits
5929 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5930 N2: getF32Constant(DAG, Flt: 0xbda7262e, dl));
5931 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5932 N2: getF32Constant(DAG, Flt: 0x3f25280b, dl));
5933 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5934 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5935 N2: getF32Constant(DAG, Flt: 0x4007b923, dl));
5936 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5937 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5938 N2: getF32Constant(DAG, Flt: 0x40823e2f, dl));
5939 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5940 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
5941 N2: getF32Constant(DAG, Flt: 0x4020d29c, dl));
5942 } else { // LimitFloatPrecision <= 18
5943 // For floating-point precision of 18:
5944 //
5945 // Log2ofMantissa =
5946 // -3.0400495f +
5947 // (6.1129976f +
5948 // (-5.3420409f +
5949 // (3.2865683f +
5950 // (-1.2669343f +
5951 // (0.27515199f -
5952 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5953 //
5954 // error 0.0000018516, which is better than 18 bits
5955 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
5956 N2: getF32Constant(DAG, Flt: 0xbcd2769e, dl));
5957 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
5958 N2: getF32Constant(DAG, Flt: 0x3e8ce0b9, dl));
5959 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
5960 SDValue t3 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
5961 N2: getF32Constant(DAG, Flt: 0x3fa22ae7, dl));
5962 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
5963 SDValue t5 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t4,
5964 N2: getF32Constant(DAG, Flt: 0x40525723, dl));
5965 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
5966 SDValue t7 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t6,
5967 N2: getF32Constant(DAG, Flt: 0x40aaf200, dl));
5968 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
5969 SDValue t9 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t8,
5970 N2: getF32Constant(DAG, Flt: 0x40c39dad, dl));
5971 SDValue t10 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t9, N2: X);
5972 Log2ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t10,
5973 N2: getF32Constant(DAG, Flt: 0x4042902c, dl));
5974 }
5975
5976 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: Log2ofMantissa);
5977 }
5978
5979 // No special expansion.
5980 return DAG.getNode(Opcode: ISD::FLOG2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5981}
5982
5983/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5984/// limited-precision mode.
5985static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5986 const TargetLowering &TLI, SDNodeFlags Flags) {
5987 // TODO: What fast-math-flags should be set on the floating-point nodes?
5988
5989 if (Op.getValueType() == MVT::f32 &&
5990 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5991 SDValue Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op);
5992
5993 // Scale the exponent by log10(2) [0.30102999f].
5994 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
5995 SDValue LogOfExponent = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: Exp,
5996 N2: getF32Constant(DAG, Flt: 0x3e9a209a, dl));
5997
5998 // Get the significand and build it into a floating-point number with
5999 // exponent of 1.
6000 SDValue X = GetSignificand(DAG, Op: Op1, dl);
6001
6002 SDValue Log10ofMantissa;
6003 if (LimitFloatPrecision <= 6) {
6004 // For floating-point precision of 6:
6005 //
6006 // Log10ofMantissa =
6007 // -0.50419619f +
6008 // (0.60948995f - 0.10380950f * x) * x;
6009 //
6010 // error 0.0014886165, which is 6 bits
6011 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6012 N2: getF32Constant(DAG, Flt: 0xbdd49a13, dl));
6013 SDValue t1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t0,
6014 N2: getF32Constant(DAG, Flt: 0x3f1c0789, dl));
6015 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6016 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t2,
6017 N2: getF32Constant(DAG, Flt: 0x3f011300, dl));
6018 } else if (LimitFloatPrecision <= 12) {
6019 // For floating-point precision of 12:
6020 //
6021 // Log10ofMantissa =
6022 // -0.64831180f +
6023 // (0.91751397f +
6024 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
6025 //
6026 // error 0.00019228036, which is better than 12 bits
6027 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6028 N2: getF32Constant(DAG, Flt: 0x3d431f31, dl));
6029 SDValue t1 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0,
6030 N2: getF32Constant(DAG, Flt: 0x3ea21fb2, dl));
6031 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6032 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
6033 N2: getF32Constant(DAG, Flt: 0x3f6ae232, dl));
6034 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6035 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t4,
6036 N2: getF32Constant(DAG, Flt: 0x3f25f7c3, dl));
6037 } else { // LimitFloatPrecision <= 18
6038 // For floating-point precision of 18:
6039 //
6040 // Log10ofMantissa =
6041 // -0.84299375f +
6042 // (1.5327582f +
6043 // (-1.0688956f +
6044 // (0.49102474f +
6045 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
6046 //
6047 // error 0.0000037995730, which is better than 18 bits
6048 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: X,
6049 N2: getF32Constant(DAG, Flt: 0x3c5d51ce, dl));
6050 SDValue t1 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t0,
6051 N2: getF32Constant(DAG, Flt: 0x3e00685a, dl));
6052 SDValue t2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t1, N2: X);
6053 SDValue t3 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t2,
6054 N2: getF32Constant(DAG, Flt: 0x3efb6798, dl));
6055 SDValue t4 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t3, N2: X);
6056 SDValue t5 = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t4,
6057 N2: getF32Constant(DAG, Flt: 0x3f88d192, dl));
6058 SDValue t6 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t5, N2: X);
6059 SDValue t7 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: t6,
6060 N2: getF32Constant(DAG, Flt: 0x3fc4316c, dl));
6061 SDValue t8 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: t7, N2: X);
6062 Log10ofMantissa = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f32, N1: t8,
6063 N2: getF32Constant(DAG, Flt: 0x3f57ce70, dl));
6064 }
6065
6066 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f32, N1: LogOfExponent, N2: Log10ofMantissa);
6067 }
6068
6069 // No special expansion.
6070 return DAG.getNode(Opcode: ISD::FLOG10, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6071}
6072
6073/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
6074/// limited-precision mode.
6075static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
6076 const TargetLowering &TLI, SDNodeFlags Flags) {
6077 if (Op.getValueType() == MVT::f32 &&
6078 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
6079 return getLimitedPrecisionExp2(t0: Op, dl, DAG);
6080
6081 // No special expansion.
6082 return DAG.getNode(Opcode: ISD::FEXP2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
6083}
6084
6085/// visitPow - Lower a pow intrinsic. Handles the special sequences for
6086/// limited-precision mode with x == 10.0f.
6087static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
6088 SelectionDAG &DAG, const TargetLowering &TLI,
6089 SDNodeFlags Flags) {
6090 bool IsExp10 = false;
6091 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
6092 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
6093 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
6094 APFloat Ten(10.0f);
6095 IsExp10 = LHSC->isExactlyValue(V: Ten);
6096 }
6097 }
6098
6099 // TODO: What fast-math-flags should be set on the FMUL node?
6100 if (IsExp10) {
6101 // Put the exponent in the right bit position for later addition to the
6102 // final result:
6103 //
6104 // #define LOG2OF10 3.3219281f
6105 // t0 = Op * LOG2OF10;
6106 SDValue t0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f32, N1: RHS,
6107 N2: getF32Constant(DAG, Flt: 0x40549a78, dl));
6108 return getLimitedPrecisionExp2(t0, dl, DAG);
6109 }
6110
6111 // No special expansion.
6112 return DAG.getNode(Opcode: ISD::FPOW, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags);
6113}
6114
6115/// ExpandPowI - Expand a llvm.powi intrinsic.
6116static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
6117 SelectionDAG &DAG) {
6118 // If RHS is a constant, we can expand this out to a multiplication tree if
6119 // it's beneficial on the target, otherwise we end up lowering to a call to
6120 // __powidf2 (for example).
6121 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
6122 unsigned Val = RHSC->getSExtValue();
6123
6124 // powi(x, 0) -> 1.0
6125 if (Val == 0)
6126 return DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType());
6127
6128 if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
6129 Exponent: Val, OptForSize: DAG.shouldOptForSize())) {
6130 // Get the exponent as a positive value.
6131 if ((int)Val < 0)
6132 Val = -Val;
6133 // We use the simple binary decomposition method to generate the multiply
6134 // sequence. There are more optimal ways to do this (for example,
6135 // powi(x,15) generates one more multiply than it should), but this has
6136 // the benefit of being both really simple and much better than a libcall.
6137 SDValue Res; // Logically starts equal to 1.0
6138 SDValue CurSquare = LHS;
6139 // TODO: Intrinsics should have fast-math-flags that propagate to these
6140 // nodes.
6141 while (Val) {
6142 if (Val & 1) {
6143 if (Res.getNode())
6144 Res =
6145 DAG.getNode(Opcode: ISD::FMUL, DL, VT: Res.getValueType(), N1: Res, N2: CurSquare);
6146 else
6147 Res = CurSquare; // 1.0*CurSquare.
6148 }
6149
6150 CurSquare = DAG.getNode(Opcode: ISD::FMUL, DL, VT: CurSquare.getValueType(),
6151 N1: CurSquare, N2: CurSquare);
6152 Val >>= 1;
6153 }
6154
6155 // If the original was negative, invert the result, producing 1/(x*x*x).
6156 if (RHSC->getSExtValue() < 0)
6157 Res = DAG.getNode(Opcode: ISD::FDIV, DL, VT: LHS.getValueType(),
6158 N1: DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType()), N2: Res);
6159 return Res;
6160 }
6161 }
6162
6163 // Otherwise, expand to a libcall.
6164 return DAG.getNode(Opcode: ISD::FPOWI, DL, VT: LHS.getValueType(), N1: LHS, N2: RHS);
6165}
6166
6167static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
6168 SDValue LHS, SDValue RHS, SDValue Scale,
6169 SelectionDAG &DAG, const TargetLowering &TLI) {
6170 EVT VT = LHS.getValueType();
6171 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
6172 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
6173 LLVMContext &Ctx = *DAG.getContext();
6174
6175 // If the type is legal but the operation isn't, this node might survive all
6176 // the way to operation legalization. If we end up there and we do not have
6177 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
6178 // node.
6179
6180 // Coax the legalizer into expanding the node during type legalization instead
6181 // by bumping the size by one bit. This will force it to Promote, enabling the
6182 // early expansion and avoiding the need to expand later.
6183
6184 // We don't have to do this if Scale is 0; that can always be expanded, unless
6185 // it's a saturating signed operation. Those can experience true integer
6186 // division overflow, a case which we must avoid.
6187
6188 // FIXME: We wouldn't have to do this (or any of the early
6189 // expansion/promotion) if it was possible to expand a libcall of an
6190 // illegal type during operation legalization. But it's not, so things
6191 // get a bit hacky.
6192 unsigned ScaleInt = Scale->getAsZExtVal();
6193 if ((ScaleInt > 0 || (Saturating && Signed)) &&
6194 (TLI.isTypeLegal(VT) ||
6195 (VT.isVector() && TLI.isTypeLegal(VT: VT.getVectorElementType())))) {
6196 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
6197 Op: Opcode, VT, Scale: ScaleInt);
6198 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
6199 EVT PromVT;
6200 if (VT.isScalarInteger())
6201 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits() + 1);
6202 else if (VT.isVector()) {
6203 PromVT = VT.getVectorElementType();
6204 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PromVT.getSizeInBits() + 1);
6205 PromVT = EVT::getVectorVT(Context&: Ctx, VT: PromVT, EC: VT.getVectorElementCount());
6206 } else
6207 llvm_unreachable("Wrong VT for DIVFIX?");
6208 LHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL, VT: PromVT);
6209 RHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL, VT: PromVT);
6210 EVT ShiftTy = TLI.getShiftAmountTy(LHSTy: PromVT, DL: DAG.getDataLayout());
6211 // For saturating operations, we need to shift up the LHS to get the
6212 // proper saturation width, and then shift down again afterwards.
6213 if (Saturating)
6214 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: PromVT, N1: LHS,
6215 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
6216 SDValue Res = DAG.getNode(Opcode, DL, VT: PromVT, N1: LHS, N2: RHS, N3: Scale);
6217 if (Saturating)
6218 Res = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: PromVT, N1: Res,
6219 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
6220 return DAG.getZExtOrTrunc(Op: Res, DL, VT);
6221 }
6222 }
6223
6224 return DAG.getNode(Opcode, DL, VT, N1: LHS, N2: RHS, N3: Scale);
6225}
6226
6227// getUnderlyingArgRegs - Find underlying registers used for a truncated,
6228// bitcasted, or split argument. Returns a list of <Register, size in bits>
6229static void
6230getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
6231 const SDValue &N) {
6232 switch (N.getOpcode()) {
6233 case ISD::CopyFromReg: {
6234 SDValue Op = N.getOperand(i: 1);
6235 Regs.emplace_back(Args: cast<RegisterSDNode>(Val&: Op)->getReg(),
6236 Args: Op.getValueType().getSizeInBits());
6237 return;
6238 }
6239 case ISD::BITCAST:
6240 case ISD::AssertZext:
6241 case ISD::AssertSext:
6242 case ISD::TRUNCATE:
6243 getUnderlyingArgRegs(Regs, N: N.getOperand(i: 0));
6244 return;
6245 case ISD::BUILD_PAIR:
6246 case ISD::BUILD_VECTOR:
6247 case ISD::CONCAT_VECTORS:
6248 for (SDValue Op : N->op_values())
6249 getUnderlyingArgRegs(Regs, N: Op);
6250 return;
6251 default:
6252 return;
6253 }
6254}
6255
6256/// If the DbgValueInst is a dbg_value of a function argument, create the
6257/// corresponding DBG_VALUE machine instruction for it now. At the end of
6258/// instruction selection, they will be inserted to the entry BB.
6259/// We don't currently support this for variadic dbg_values, as they shouldn't
6260/// appear for function arguments or in the prologue.
6261bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6262 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6263 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6264 const Argument *Arg = dyn_cast<Argument>(Val: V);
6265 if (!Arg)
6266 return false;
6267
6268 MachineFunction &MF = DAG.getMachineFunction();
6269 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6270
6271 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6272 // we've been asked to pursue.
6273 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6274 bool Indirect) {
6275 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6276 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6277 // pointing at the VReg, which will be patched up later.
6278 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_INSTR_REF);
6279 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
6280 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6281 /* isKill */ false, /* isDead */ false,
6282 /* isUndef */ false, /* isEarlyClobber */ false,
6283 /* SubReg */ 0, /* isDebug */ true)});
6284
6285 auto *NewDIExpr = FragExpr;
6286 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6287 // the DIExpression.
6288 if (Indirect)
6289 NewDIExpr = DIExpression::prepend(Expr: FragExpr, Flags: DIExpression::DerefBefore);
6290 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
6291 NewDIExpr = DIExpression::prependOpcodes(Expr: NewDIExpr, Ops);
6292 return BuildMI(MF, DL, MCID: Inst, IsIndirect: false, MOs, Variable, Expr: NewDIExpr);
6293 } else {
6294 // Create a completely standard DBG_VALUE.
6295 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_VALUE);
6296 return BuildMI(MF, DL, MCID: Inst, IsIndirect: Indirect, Reg, Variable, Expr: FragExpr);
6297 }
6298 };
6299
6300 if (Kind == FuncArgumentDbgValueKind::Value) {
6301 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6302 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6303 // the entry block.
6304 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6305 if (!IsInEntryBlock)
6306 return false;
6307
6308 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6309 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6310 // variable that also is a param.
6311 //
6312 // Although, if we are at the top of the entry block already, we can still
6313 // emit using ArgDbgValue. This might catch some situations when the
6314 // dbg.value refers to an argument that isn't used in the entry block, so
6315 // any CopyToReg node would be optimized out and the only way to express
6316 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6317 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6318 // we should only emit as ArgDbgValue if the Variable is an argument to the
6319 // current function, and the dbg.value intrinsic is found in the entry
6320 // block.
6321 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6322 !DL->getInlinedAt();
6323 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6324 if (!IsInPrologue && !VariableIsFunctionInputArg)
6325 return false;
6326
6327 // Here we assume that a function argument on IR level only can be used to
6328 // describe one input parameter on source level. If we for example have
6329 // source code like this
6330 //
6331 // struct A { long x, y; };
6332 // void foo(struct A a, long b) {
6333 // ...
6334 // b = a.x;
6335 // ...
6336 // }
6337 //
6338 // and IR like this
6339 //
6340 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6341 // entry:
6342 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6343 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6344 // call void @llvm.dbg.value(metadata i32 %b, "b",
6345 // ...
6346 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6347 // ...
6348 //
6349 // then the last dbg.value is describing a parameter "b" using a value that
6350 // is an argument. But since we already has used %a1 to describe a parameter
6351 // we should not handle that last dbg.value here (that would result in an
6352 // incorrect hoisting of the DBG_VALUE to the function entry).
6353 // Notice that we allow one dbg.value per IR level argument, to accommodate
6354 // for the situation with fragments above.
6355 // If there is no node for the value being handled, we return true to skip
6356 // the normal generation of debug info, as it would kill existing debug
6357 // info for the parameter in case of duplicates.
6358 if (VariableIsFunctionInputArg) {
6359 unsigned ArgNo = Arg->getArgNo();
6360 if (ArgNo >= FuncInfo.DescribedArgs.size())
6361 FuncInfo.DescribedArgs.resize(N: ArgNo + 1, t: false);
6362 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(Idx: ArgNo))
6363 return !NodeMap[V].getNode();
6364 FuncInfo.DescribedArgs.set(ArgNo);
6365 }
6366 }
6367
6368 bool IsIndirect = false;
6369 std::optional<MachineOperand> Op;
6370 // Some arguments' frame index is recorded during argument lowering.
6371 int FI = FuncInfo.getArgumentFrameIndex(A: Arg);
6372 if (FI != std::numeric_limits<int>::max())
6373 Op = MachineOperand::CreateFI(Idx: FI);
6374
6375 SmallVector<std::pair<Register, TypeSize>, 8> ArgRegsAndSizes;
6376 if (!Op && N.getNode()) {
6377 getUnderlyingArgRegs(Regs&: ArgRegsAndSizes, N);
6378 Register Reg;
6379 if (ArgRegsAndSizes.size() == 1)
6380 Reg = ArgRegsAndSizes.front().first;
6381
6382 if (Reg && Reg.isVirtual()) {
6383 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6384 Register PR = RegInfo.getLiveInPhysReg(VReg: Reg);
6385 if (PR)
6386 Reg = PR;
6387 }
6388 if (Reg) {
6389 Op = MachineOperand::CreateReg(Reg, isDef: false);
6390 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6391 }
6392 }
6393
6394 if (!Op && N.getNode()) {
6395 // Check if frame index is available.
6396 SDValue LCandidate = peekThroughBitcasts(V: N);
6397 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(Val: LCandidate.getNode()))
6398 if (FrameIndexSDNode *FINode =
6399 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
6400 Op = MachineOperand::CreateFI(Idx: FINode->getIndex());
6401 }
6402
6403 if (!Op) {
6404 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6405 auto splitMultiRegDbgValue =
6406 [&](ArrayRef<std::pair<Register, TypeSize>> SplitRegs) -> bool {
6407 unsigned Offset = 0;
6408 for (const auto &[Reg, RegSizeInBits] : SplitRegs) {
6409 // FIXME: Scalable sizes are not supported in fragment expressions.
6410 if (RegSizeInBits.isScalable())
6411 return false;
6412
6413 // If the expression is already a fragment, the current register
6414 // offset+size might extend beyond the fragment. In this case, only
6415 // the register bits that are inside the fragment are relevant.
6416 int RegFragmentSizeInBits = RegSizeInBits.getFixedValue();
6417 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6418 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6419 // The register is entirely outside the expression fragment,
6420 // so is irrelevant for debug info.
6421 if (Offset >= ExprFragmentSizeInBits)
6422 break;
6423 // The register is partially outside the expression fragment, only
6424 // the low bits within the fragment are relevant for debug info.
6425 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6426 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6427 }
6428 }
6429
6430 auto FragmentExpr = DIExpression::createFragmentExpression(
6431 Expr, OffsetInBits: Offset, SizeInBits: RegFragmentSizeInBits);
6432 Offset += RegSizeInBits.getFixedValue();
6433 // If a valid fragment expression cannot be created, the variable's
6434 // correct value cannot be determined and so it is set as poison.
6435 if (!FragmentExpr) {
6436 SDDbgValue *SDV = DAG.getConstantDbgValue(
6437 Var: Variable, Expr, C: PoisonValue::get(T: V->getType()), DL, O: SDNodeOrder);
6438 DAG.AddDbgValue(DB: SDV, isParameter: false);
6439 continue;
6440 }
6441 MachineInstr *NewMI = MakeVRegDbgValue(
6442 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6443 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6444 }
6445
6446 return true;
6447 };
6448
6449 // Check if ValueMap has reg number.
6450 DenseMap<const Value *, Register>::const_iterator
6451 VMI = FuncInfo.ValueMap.find(Val: V);
6452 if (VMI != FuncInfo.ValueMap.end()) {
6453 const auto &TLI = DAG.getTargetLoweringInfo();
6454 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6455 V->getType(), std::nullopt);
6456 if (RFV.occupiesMultipleRegs())
6457 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6458
6459 Op = MachineOperand::CreateReg(Reg: VMI->second, isDef: false);
6460 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6461 } else if (ArgRegsAndSizes.size() > 1) {
6462 // This was split due to the calling convention, and no virtual register
6463 // mapping exists for the value.
6464 return splitMultiRegDbgValue(ArgRegsAndSizes);
6465 }
6466 }
6467
6468 if (!Op)
6469 return false;
6470
6471 assert(Variable->isValidLocationForIntrinsic(DL) &&
6472 "Expected inlined-at fields to agree");
6473 MachineInstr *NewMI = nullptr;
6474
6475 if (Op->isReg())
6476 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6477 else
6478 NewMI = BuildMI(MF, DL, MCID: TII->get(Opcode: TargetOpcode::DBG_VALUE), IsIndirect: true, MOs: *Op,
6479 Variable, Expr);
6480
6481 // Otherwise, use ArgDbgValues.
6482 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6483 return true;
6484}
6485
6486/// Return the appropriate SDDbgValue based on N.
6487SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6488 DILocalVariable *Variable,
6489 DIExpression *Expr,
6490 const DebugLoc &dl,
6491 unsigned DbgSDNodeOrder) {
6492 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
6493 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6494 // stack slot locations.
6495 //
6496 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6497 // debug values here after optimization:
6498 //
6499 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6500 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6501 //
6502 // Both describe the direct values of their associated variables.
6503 return DAG.getFrameIndexDbgValue(Var: Variable, Expr, FI: FISDN->getIndex(),
6504 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6505 }
6506 return DAG.getDbgValue(Var: Variable, Expr, N: N.getNode(), R: N.getResNo(),
6507 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6508}
6509
6510static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6511 switch (Intrinsic) {
6512 case Intrinsic::smul_fix:
6513 return ISD::SMULFIX;
6514 case Intrinsic::umul_fix:
6515 return ISD::UMULFIX;
6516 case Intrinsic::smul_fix_sat:
6517 return ISD::SMULFIXSAT;
6518 case Intrinsic::umul_fix_sat:
6519 return ISD::UMULFIXSAT;
6520 case Intrinsic::sdiv_fix:
6521 return ISD::SDIVFIX;
6522 case Intrinsic::udiv_fix:
6523 return ISD::UDIVFIX;
6524 case Intrinsic::sdiv_fix_sat:
6525 return ISD::SDIVFIXSAT;
6526 case Intrinsic::udiv_fix_sat:
6527 return ISD::UDIVFIXSAT;
6528 default:
6529 llvm_unreachable("Unhandled fixed point intrinsic");
6530 }
6531}
6532
6533/// Given a @llvm.call.preallocated.setup, return the corresponding
6534/// preallocated call.
6535static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6536 assert(cast<CallBase>(PreallocatedSetup)
6537 ->getCalledFunction()
6538 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6539 "expected call_preallocated_setup Value");
6540 for (const auto *U : PreallocatedSetup->users()) {
6541 auto *UseCall = cast<CallBase>(Val: U);
6542 const Function *Fn = UseCall->getCalledFunction();
6543 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6544 return UseCall;
6545 }
6546 }
6547 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6548}
6549
6550/// If DI is a debug value with an EntryValue expression, lower it using the
6551/// corresponding physical register of the associated Argument value
6552/// (guaranteed to exist by the verifier).
6553bool SelectionDAGBuilder::visitEntryValueDbgValue(
6554 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6555 DIExpression *Expr, DebugLoc DbgLoc) {
6556 if (!Expr->isEntryValue() || !hasSingleElement(C&: Values))
6557 return false;
6558
6559 // These properties are guaranteed by the verifier.
6560 const Argument *Arg = cast<Argument>(Val: Values[0]);
6561 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6562
6563 auto ArgIt = FuncInfo.ValueMap.find(Val: Arg);
6564 if (ArgIt == FuncInfo.ValueMap.end()) {
6565 LLVM_DEBUG(
6566 dbgs() << "Dropping dbg.value: expression is entry_value but "
6567 "couldn't find an associated register for the Argument\n");
6568 return true;
6569 }
6570 Register ArgVReg = ArgIt->getSecond();
6571
6572 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6573 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6574 SDDbgValue *SDV = DAG.getVRegDbgValue(
6575 Var: Variable, Expr, VReg: PhysReg, IsIndirect: false /*IsIndidrect*/, DL: DbgLoc, O: SDNodeOrder);
6576 DAG.AddDbgValue(DB: SDV, isParameter: false /*treat as dbg.declare byval parameter*/);
6577 return true;
6578 }
6579 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6580 "couldn't find a physical register\n");
6581 return true;
6582}
6583
6584/// Lower the call to the specified intrinsic function.
6585void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6586 unsigned Intrinsic) {
6587 SDLoc sdl = getCurSDLoc();
6588 switch (Intrinsic) {
6589 case Intrinsic::experimental_convergence_anchor:
6590 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_ANCHOR, DL: sdl, VT: MVT::Untyped));
6591 break;
6592 case Intrinsic::experimental_convergence_entry:
6593 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_ENTRY, DL: sdl, VT: MVT::Untyped));
6594 break;
6595 case Intrinsic::experimental_convergence_loop: {
6596 auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
6597 auto *Token = Bundle->Inputs[0].get();
6598 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_LOOP, DL: sdl, VT: MVT::Untyped,
6599 Operand: getValue(V: Token)));
6600 break;
6601 }
6602 }
6603}
6604
6605void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6606 unsigned IntrinsicID) {
6607 // For now, we're only lowering an 'add' histogram.
6608 // We can add others later, e.g. saturating adds, min/max.
6609 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6610 "Tried to lower unsupported histogram type");
6611 SDLoc sdl = getCurSDLoc();
6612 Value *Ptr = I.getOperand(i_nocapture: 0);
6613 SDValue Inc = getValue(V: I.getOperand(i_nocapture: 1));
6614 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
6615
6616 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6617 DataLayout TargetDL = DAG.getDataLayout();
6618 EVT VT = Inc.getValueType();
6619 Align Alignment = DAG.getEVTAlign(MemoryVT: VT);
6620
6621 const MDNode *Ranges = getRangeMetadata(I);
6622
6623 SDValue Root = DAG.getRoot();
6624 SDValue Base;
6625 SDValue Index;
6626 SDValue Scale;
6627 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, SDB: this,
6628 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
6629
6630 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6631
6632 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6633 PtrInfo: MachinePointerInfo(AS),
6634 F: MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
6635 Size: MemoryLocation::UnknownSize, BaseAlignment: Alignment, AAInfo: I.getAAMetadata(), Ranges);
6636
6637 if (!UniformBase) {
6638 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6639 Index = getValue(V: Ptr);
6640 Scale =
6641 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6642 }
6643
6644 EVT IdxVT = Index.getValueType();
6645
6646 // Avoid using e.g. i32 as index type when the increment must be performed
6647 // on i64's.
6648 bool MustExtendIndex = VT.getScalarSizeInBits() > IdxVT.getScalarSizeInBits();
6649 EVT EltTy = MustExtendIndex ? VT : IdxVT.getVectorElementType();
6650 if (MustExtendIndex || TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
6651 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
6652 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
6653 }
6654
6655 SDValue ID = DAG.getTargetConstant(Val: IntrinsicID, DL: sdl, VT: MVT::i32);
6656
6657 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6658 SDValue Histogram = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT: VT, dl: sdl,
6659 Ops, MMO, IndexType: ISD::SIGNED_SCALED);
6660
6661 setValue(V: &I, NewN: Histogram);
6662 DAG.setRoot(Histogram);
6663}
6664
6665void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6666 unsigned Intrinsic) {
6667 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6668 "Tried lowering invalid vector extract last");
6669 SDLoc sdl = getCurSDLoc();
6670 const DataLayout &Layout = DAG.getDataLayout();
6671 SDValue Data = getValue(V: I.getOperand(i_nocapture: 0));
6672 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
6673
6674 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6675 EVT ResVT = TLI.getValueType(DL: Layout, Ty: I.getType());
6676
6677 EVT ExtVT = TLI.getVectorIdxTy(DL: Layout);
6678 SDValue Idx = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL: sdl, VT: ExtVT, Operand: Mask);
6679 SDValue Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: sdl, VT: ResVT, N1: Data, N2: Idx);
6680
6681 Value *Default = I.getOperand(i_nocapture: 2);
6682 if (!isa<PoisonValue>(Val: Default) && !isa<UndefValue>(Val: Default)) {
6683 SDValue PassThru = getValue(V: Default);
6684 EVT BoolVT = Mask.getValueType().getScalarType();
6685 SDValue AnyActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: sdl, VT: BoolVT, Operand: Mask);
6686 Result = DAG.getSelect(DL: sdl, VT: ResVT, Cond: AnyActive, LHS: Result, RHS: PassThru);
6687 }
6688
6689 setValue(V: &I, NewN: Result);
6690}
6691
6692/// Lower the call to the specified intrinsic function.
6693void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6694 unsigned Intrinsic) {
6695 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6696 SDLoc sdl = getCurSDLoc();
6697 DebugLoc dl = getCurDebugLoc();
6698 SDValue Res;
6699
6700 SDNodeFlags Flags;
6701 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
6702 Flags.copyFMF(FPMO: *FPOp);
6703
6704 switch (Intrinsic) {
6705 default:
6706 // By default, turn this into a target intrinsic node.
6707 visitTargetIntrinsic(I, Intrinsic);
6708 return;
6709 case Intrinsic::vscale: {
6710 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6711 setValue(V: &I, NewN: DAG.getVScale(DL: sdl, VT, MulImm: APInt(VT.getSizeInBits(), 1)));
6712 return;
6713 }
6714 case Intrinsic::vastart: visitVAStart(I); return;
6715 case Intrinsic::vaend: visitVAEnd(I); return;
6716 case Intrinsic::vacopy: visitVACopy(I); return;
6717 case Intrinsic::returnaddress:
6718 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::RETURNADDR, DL: sdl,
6719 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
6720 Operand: getValue(V: I.getArgOperand(i: 0))));
6721 return;
6722 case Intrinsic::addressofreturnaddress:
6723 setValue(V: &I,
6724 NewN: DAG.getNode(Opcode: ISD::ADDROFRETURNADDR, DL: sdl,
6725 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6726 return;
6727 case Intrinsic::sponentry:
6728 setValue(V: &I,
6729 NewN: DAG.getNode(Opcode: ISD::SPONENTRY, DL: sdl,
6730 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6731 return;
6732 case Intrinsic::frameaddress:
6733 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FRAMEADDR, DL: sdl,
6734 VT: TLI.getFrameIndexTy(DL: DAG.getDataLayout()),
6735 Operand: getValue(V: I.getArgOperand(i: 0))));
6736 return;
6737 case Intrinsic::read_volatile_register:
6738 case Intrinsic::read_register: {
6739 Value *Reg = I.getArgOperand(i: 0);
6740 SDValue Chain = getRoot();
6741 SDValue RegName =
6742 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6743 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6744 Res = DAG.getNode(Opcode: ISD::READ_REGISTER, DL: sdl,
6745 VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), N1: Chain, N2: RegName);
6746 setValue(V: &I, NewN: Res);
6747 DAG.setRoot(Res.getValue(R: 1));
6748 return;
6749 }
6750 case Intrinsic::write_register: {
6751 Value *Reg = I.getArgOperand(i: 0);
6752 Value *RegValue = I.getArgOperand(i: 1);
6753 SDValue Chain = getRoot();
6754 SDValue RegName =
6755 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6756 DAG.setRoot(DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL: sdl, VT: MVT::Other, N1: Chain,
6757 N2: RegName, N3: getValue(V: RegValue)));
6758 return;
6759 }
6760 case Intrinsic::write_volatile_register: {
6761 Value *Reg = I.getArgOperand(i: 0);
6762 Value *RegValue = I.getArgOperand(i: 1);
6763 SDValue Chain = getRoot();
6764 const MDNode *MD = cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata());
6765 SDValue RegName = DAG.getMDNode(MD);
6766 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: RegValue->getType());
6767 SDValue WriteChain = DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL: sdl, VT: MVT::Other,
6768 N1: Chain, N2: RegName, N3: getValue(V: RegValue));
6769 // FAKE_USE of the physical register marks it live after the WRITE_REGISTER,
6770 // preventing the backend from dead-eliminating the write. This is
6771 // preferred over READ_REGISTER, which would emit extra register copies
6772 // (e.g. fmov xN, dN for FP/SIMD registers).
6773 const MDString *RegStr = cast<MDString>(Val: MD->getOperand(I: 0));
6774 LLT Ty = VT.isSimple() ? getLLTForMVT(Ty: VT.getSimpleVT()) : LLT();
6775 const MachineFunction &MF = DAG.getMachineFunction();
6776 Register PhysReg =
6777 TLI.getRegisterByName(RegName: RegStr->getString().data(), Ty, MF);
6778 if (PhysReg.isValid()) {
6779 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6780 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: PhysReg);
6781 MVT RegVT = *TRI->legalclasstypes_begin(RC: *RC);
6782 DAG.setRoot(DAG.getNode(Opcode: ISD::FAKE_USE, DL: sdl, VT: MVT::Other,
6783 Ops: {WriteChain, DAG.getRegister(Reg: PhysReg, VT: RegVT)}));
6784 } else {
6785 DAG.setRoot(WriteChain);
6786 }
6787 return;
6788 }
6789 case Intrinsic::memcpy:
6790 case Intrinsic::memcpy_inline: {
6791 const auto &MCI = cast<MemCpyInst>(Val: I);
6792 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6793 SDValue Src = getValue(V: I.getArgOperand(i: 1));
6794 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6795 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6796 "memcpy_inline needs constant size");
6797 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6798 Align DstAlign = MCI.getDestAlign().valueOrOne();
6799 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6800 bool isVol = MCI.isVolatile();
6801 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6802 SDValue MC = DAG.getMemcpy(Chain: Root, dl: sdl, Dst, Src, Size, DstAlign, SrcAlign,
6803 isVol, AlwaysInline: MCI.isForceInlined(), CI: &I, OverrideTailCall: std::nullopt,
6804 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
6805 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)),
6806 AAInfo: I.getAAMetadata(), BatchAA);
6807 updateDAGForMaybeTailCall(MaybeTC: MC);
6808 return;
6809 }
6810 case Intrinsic::memset:
6811 case Intrinsic::memset_inline: {
6812 const auto &MSII = cast<MemSetInst>(Val: I);
6813 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6814 SDValue Value = getValue(V: I.getArgOperand(i: 1));
6815 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6816 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6817 "memset_inline needs constant size");
6818 // @llvm.memset defines 0 and 1 to both mean no alignment.
6819 Align DstAlign = MSII.getDestAlign().valueOrOne();
6820 bool isVol = MSII.isVolatile();
6821 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6822 SDValue MC = DAG.getMemset(
6823 Chain: Root, dl: sdl, Dst, Src: Value, Size, Alignment: DstAlign, isVol, AlwaysInline: MSII.isForceInlined(),
6824 CI: &I, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)), AAInfo: I.getAAMetadata());
6825 updateDAGForMaybeTailCall(MaybeTC: MC);
6826 return;
6827 }
6828 case Intrinsic::memmove: {
6829 const auto &MMI = cast<MemMoveInst>(Val: I);
6830 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
6831 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
6832 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
6833 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6834 Align DstAlign = MMI.getDestAlign().valueOrOne();
6835 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6836 bool isVol = MMI.isVolatile();
6837 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6838 SDValue MM = DAG.getMemmove(
6839 Chain: Root, dl: sdl, Dst: Op1, Src: Op2, Size: Op3, DstAlign, SrcAlign, isVol, CI: &I,
6840 /* OverrideTailCall */ std::nullopt,
6841 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
6842 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)), AAInfo: I.getAAMetadata(), BatchAA);
6843 updateDAGForMaybeTailCall(MaybeTC: MM);
6844 return;
6845 }
6846 case Intrinsic::memcpy_element_unordered_atomic: {
6847 auto &MI = cast<AnyMemCpyInst>(Val: I);
6848 SDValue Dst = getValue(V: MI.getRawDest());
6849 SDValue Src = getValue(V: MI.getRawSource());
6850 SDValue Length = getValue(V: MI.getLength());
6851
6852 Type *LengthTy = MI.getLength()->getType();
6853 unsigned ElemSz = MI.getElementSizeInBytes();
6854 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6855 SDValue MC =
6856 DAG.getAtomicMemcpy(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
6857 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
6858 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
6859 updateDAGForMaybeTailCall(MaybeTC: MC);
6860 return;
6861 }
6862 case Intrinsic::memmove_element_unordered_atomic: {
6863 auto &MI = cast<AnyMemMoveInst>(Val: I);
6864 SDValue Dst = getValue(V: MI.getRawDest());
6865 SDValue Src = getValue(V: MI.getRawSource());
6866 SDValue Length = getValue(V: MI.getLength());
6867
6868 Type *LengthTy = MI.getLength()->getType();
6869 unsigned ElemSz = MI.getElementSizeInBytes();
6870 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6871 SDValue MC =
6872 DAG.getAtomicMemmove(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
6873 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
6874 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
6875 updateDAGForMaybeTailCall(MaybeTC: MC);
6876 return;
6877 }
6878 case Intrinsic::memset_element_unordered_atomic: {
6879 auto &MI = cast<AnyMemSetInst>(Val: I);
6880 SDValue Dst = getValue(V: MI.getRawDest());
6881 SDValue Val = getValue(V: MI.getValue());
6882 SDValue Length = getValue(V: MI.getLength());
6883
6884 Type *LengthTy = MI.getLength()->getType();
6885 unsigned ElemSz = MI.getElementSizeInBytes();
6886 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6887 SDValue MC =
6888 DAG.getAtomicMemset(Chain: getRoot(), dl: sdl, Dst, Value: Val, Size: Length, SizeTy: LengthTy, ElemSz,
6889 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()));
6890 updateDAGForMaybeTailCall(MaybeTC: MC);
6891 return;
6892 }
6893 case Intrinsic::call_preallocated_setup: {
6894 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: &I);
6895 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
6896 SDValue Res = DAG.getNode(Opcode: ISD::PREALLOCATED_SETUP, DL: sdl, VT: MVT::Other,
6897 N1: getRoot(), N2: SrcValue);
6898 setValue(V: &I, NewN: Res);
6899 DAG.setRoot(Res);
6900 return;
6901 }
6902 case Intrinsic::call_preallocated_arg: {
6903 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: I.getOperand(i_nocapture: 0));
6904 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
6905 SDValue Ops[3];
6906 Ops[0] = getRoot();
6907 Ops[1] = SrcValue;
6908 Ops[2] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 1)), DL: sdl,
6909 VT: MVT::i32); // arg index
6910 SDValue Res = DAG.getNode(
6911 Opcode: ISD::PREALLOCATED_ARG, DL: sdl,
6912 VTList: DAG.getVTList(VT1: TLI.getPointerTy(DL: DAG.getDataLayout()), VT2: MVT::Other), Ops);
6913 setValue(V: &I, NewN: Res);
6914 DAG.setRoot(Res.getValue(R: 1));
6915 return;
6916 }
6917
6918 case Intrinsic::eh_typeid_for: {
6919 // Find the type id for the given typeinfo.
6920 GlobalValue *GV = ExtractTypeInfo(V: I.getArgOperand(i: 0));
6921 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(TI: GV);
6922 Res = DAG.getConstant(Val: TypeID, DL: sdl, VT: MVT::i32);
6923 setValue(V: &I, NewN: Res);
6924 return;
6925 }
6926
6927 case Intrinsic::eh_return_i32:
6928 case Intrinsic::eh_return_i64:
6929 DAG.getMachineFunction().setCallsEHReturn(true);
6930 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_RETURN, DL: sdl,
6931 VT: MVT::Other,
6932 N1: getControlRoot(),
6933 N2: getValue(V: I.getArgOperand(i: 0)),
6934 N3: getValue(V: I.getArgOperand(i: 1))));
6935 return;
6936 case Intrinsic::eh_unwind_init:
6937 DAG.getMachineFunction().setCallsUnwindInit(true);
6938 return;
6939 case Intrinsic::eh_dwarf_cfa:
6940 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EH_DWARF_CFA, DL: sdl,
6941 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
6942 Operand: getValue(V: I.getArgOperand(i: 0))));
6943 return;
6944 case Intrinsic::eh_sjlj_callsite: {
6945 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 0));
6946 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
6947
6948 FuncInfo.setCurrentCallSite(CI->getZExtValue());
6949 return;
6950 }
6951 case Intrinsic::eh_sjlj_functioncontext: {
6952 // Get and store the index of the function context.
6953 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6954 AllocaInst *FnCtx =
6955 cast<AllocaInst>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
6956 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6957 MFI.setFunctionContextIndex(FI);
6958 return;
6959 }
6960 case Intrinsic::eh_sjlj_setjmp: {
6961 SDValue Ops[2];
6962 Ops[0] = getRoot();
6963 Ops[1] = getValue(V: I.getArgOperand(i: 0));
6964 SDValue Op = DAG.getNode(Opcode: ISD::EH_SJLJ_SETJMP, DL: sdl,
6965 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
6966 setValue(V: &I, NewN: Op.getValue(R: 0));
6967 DAG.setRoot(Op.getValue(R: 1));
6968 return;
6969 }
6970 case Intrinsic::eh_sjlj_longjmp:
6971 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_SJLJ_LONGJMP, DL: sdl, VT: MVT::Other,
6972 N1: getRoot(), N2: getValue(V: I.getArgOperand(i: 0))));
6973 return;
6974 case Intrinsic::eh_sjlj_setup_dispatch:
6975 DAG.setRoot(DAG.getNode(Opcode: ISD::EH_SJLJ_SETUP_DISPATCH, DL: sdl, VT: MVT::Other,
6976 Operand: getRoot()));
6977 return;
6978 case Intrinsic::masked_gather:
6979 visitMaskedGather(I);
6980 return;
6981 case Intrinsic::masked_load:
6982 visitMaskedLoad(I);
6983 return;
6984 case Intrinsic::masked_scatter:
6985 visitMaskedScatter(I);
6986 return;
6987 case Intrinsic::masked_store:
6988 visitMaskedStore(I);
6989 return;
6990 case Intrinsic::masked_expandload:
6991 visitMaskedLoad(I, IsExpanding: true /* IsExpanding */);
6992 return;
6993 case Intrinsic::masked_compressstore:
6994 visitMaskedStore(I, IsCompressing: true /* IsCompressing */);
6995 return;
6996 case Intrinsic::powi:
6997 setValue(V: &I, NewN: ExpandPowI(DL: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
6998 RHS: getValue(V: I.getArgOperand(i: 1)), DAG));
6999 return;
7000 case Intrinsic::log:
7001 setValue(V: &I, NewN: expandLog(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7002 return;
7003 case Intrinsic::log2:
7004 setValue(V: &I,
7005 NewN: expandLog2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7006 return;
7007 case Intrinsic::log10:
7008 setValue(V: &I,
7009 NewN: expandLog10(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7010 return;
7011 case Intrinsic::exp:
7012 setValue(V: &I, NewN: expandExp(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7013 return;
7014 case Intrinsic::exp2:
7015 setValue(V: &I,
7016 NewN: expandExp2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
7017 return;
7018 case Intrinsic::pow:
7019 setValue(V: &I, NewN: expandPow(dl: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
7020 RHS: getValue(V: I.getArgOperand(i: 1)), DAG, TLI, Flags));
7021 return;
7022 case Intrinsic::sqrt:
7023 case Intrinsic::fabs:
7024 case Intrinsic::sin:
7025 case Intrinsic::cos:
7026 case Intrinsic::tan:
7027 case Intrinsic::asin:
7028 case Intrinsic::acos:
7029 case Intrinsic::atan:
7030 case Intrinsic::sinh:
7031 case Intrinsic::cosh:
7032 case Intrinsic::tanh:
7033 case Intrinsic::exp10:
7034 case Intrinsic::floor:
7035 case Intrinsic::ceil:
7036 case Intrinsic::trunc:
7037 case Intrinsic::rint:
7038 case Intrinsic::nearbyint:
7039 case Intrinsic::round:
7040 case Intrinsic::roundeven:
7041 case Intrinsic::canonicalize: {
7042 unsigned Opcode;
7043 // clang-format off
7044 switch (Intrinsic) {
7045 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7046 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
7047 case Intrinsic::fabs: Opcode = ISD::FABS; break;
7048 case Intrinsic::sin: Opcode = ISD::FSIN; break;
7049 case Intrinsic::cos: Opcode = ISD::FCOS; break;
7050 case Intrinsic::tan: Opcode = ISD::FTAN; break;
7051 case Intrinsic::asin: Opcode = ISD::FASIN; break;
7052 case Intrinsic::acos: Opcode = ISD::FACOS; break;
7053 case Intrinsic::atan: Opcode = ISD::FATAN; break;
7054 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
7055 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
7056 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
7057 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
7058 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
7059 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
7060 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
7061 case Intrinsic::rint: Opcode = ISD::FRINT; break;
7062 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
7063 case Intrinsic::round: Opcode = ISD::FROUND; break;
7064 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
7065 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
7066 }
7067 // clang-format on
7068
7069 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl,
7070 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7071 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
7072 return;
7073 }
7074 case Intrinsic::atan2:
7075 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FATAN2, DL: sdl,
7076 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7077 N1: getValue(V: I.getArgOperand(i: 0)),
7078 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7079 return;
7080 case Intrinsic::lround:
7081 case Intrinsic::llround:
7082 case Intrinsic::lrint:
7083 case Intrinsic::llrint: {
7084 unsigned Opcode;
7085 // clang-format off
7086 switch (Intrinsic) {
7087 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7088 case Intrinsic::lround: Opcode = ISD::LROUND; break;
7089 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
7090 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
7091 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
7092 }
7093 // clang-format on
7094
7095 EVT RetVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7096 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VT: RetVT,
7097 Operand: getValue(V: I.getArgOperand(i: 0))));
7098 return;
7099 }
7100 case Intrinsic::minnum:
7101 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINNUM, DL: sdl,
7102 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7103 N1: getValue(V: I.getArgOperand(i: 0)),
7104 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7105 return;
7106 case Intrinsic::maxnum:
7107 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXNUM, DL: sdl,
7108 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7109 N1: getValue(V: I.getArgOperand(i: 0)),
7110 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7111 return;
7112 case Intrinsic::minimum:
7113 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUM, DL: sdl,
7114 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7115 N1: getValue(V: I.getArgOperand(i: 0)),
7116 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7117 return;
7118 case Intrinsic::maximum:
7119 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUM, DL: sdl,
7120 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7121 N1: getValue(V: I.getArgOperand(i: 0)),
7122 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7123 return;
7124 case Intrinsic::minimumnum:
7125 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUMNUM, DL: sdl,
7126 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7127 N1: getValue(V: I.getArgOperand(i: 0)),
7128 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7129 return;
7130 case Intrinsic::maximumnum:
7131 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUMNUM, DL: sdl,
7132 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7133 N1: getValue(V: I.getArgOperand(i: 0)),
7134 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7135 return;
7136 case Intrinsic::copysign:
7137 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: sdl,
7138 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7139 N1: getValue(V: I.getArgOperand(i: 0)),
7140 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7141 return;
7142 case Intrinsic::ldexp:
7143 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FLDEXP, DL: sdl,
7144 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7145 N1: getValue(V: I.getArgOperand(i: 0)),
7146 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
7147 return;
7148 case Intrinsic::modf:
7149 case Intrinsic::sincos:
7150 case Intrinsic::sincospi:
7151 case Intrinsic::frexp: {
7152 unsigned Opcode;
7153 switch (Intrinsic) {
7154 default:
7155 llvm_unreachable("unexpected intrinsic");
7156 case Intrinsic::sincos:
7157 Opcode = ISD::FSINCOS;
7158 break;
7159 case Intrinsic::sincospi:
7160 Opcode = ISD::FSINCOSPI;
7161 break;
7162 case Intrinsic::modf:
7163 Opcode = ISD::FMODF;
7164 break;
7165 case Intrinsic::frexp:
7166 Opcode = ISD::FFREXP;
7167 break;
7168 }
7169 SmallVector<EVT, 2> ValueVTs;
7170 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
7171 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
7172 setValue(
7173 V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: getValue(V: I.getArgOperand(i: 0)), Flags));
7174 return;
7175 }
7176 case Intrinsic::arithmetic_fence: {
7177 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: sdl,
7178 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7179 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
7180 return;
7181 }
7182 case Intrinsic::fma:
7183 setValue(V: &I, NewN: DAG.getNode(
7184 Opcode: ISD::FMA, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7185 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)),
7186 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7187 return;
7188#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7189 case Intrinsic::INTRINSIC:
7190#include "llvm/IR/ConstrainedOps.def"
7191 visitConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: I));
7192 return;
7193#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7194#include "llvm/IR/VPIntrinsics.def"
7195 visitVectorPredicationIntrinsic(VPIntrin: cast<VPIntrinsic>(Val: I));
7196 return;
7197 case Intrinsic::fptrunc_round: {
7198 // Get the last argument, the metadata and convert it to an integer in the
7199 // call
7200 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7201 std::optional<RoundingMode> RoundMode =
7202 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
7203
7204 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7205
7206 // Propagate fast-math-flags from IR to node(s).
7207 SDNodeFlags Flags;
7208 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
7209 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
7210
7211 SDValue Result;
7212 Result = DAG.getNode(
7213 Opcode: ISD::FPTRUNC_ROUND, DL: sdl, VT, N1: getValue(V: I.getArgOperand(i: 0)),
7214 N2: DAG.getTargetConstant(Val: (int)*RoundMode, DL: sdl, VT: MVT::i32));
7215 setValue(V: &I, NewN: Result);
7216
7217 return;
7218 }
7219 case Intrinsic::fmuladd: {
7220 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7221 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
7222 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
7223 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMA, DL: sdl,
7224 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7225 N1: getValue(V: I.getArgOperand(i: 0)),
7226 N2: getValue(V: I.getArgOperand(i: 1)),
7227 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7228 } else if (TLI.isOperationLegalOrCustom(Op: ISD::FMULADD, VT)) {
7229 // TODO: Support splitting the vector.
7230 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMULADD, DL: sdl,
7231 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7232 N1: getValue(V: I.getArgOperand(i: 0)),
7233 N2: getValue(V: I.getArgOperand(i: 1)),
7234 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
7235 } else {
7236 // TODO: Intrinsic calls should have fast-math-flags.
7237 SDValue Mul = DAG.getNode(
7238 Opcode: ISD::FMUL, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7239 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)), Flags);
7240 SDValue Add = DAG.getNode(Opcode: ISD::FADD, DL: sdl,
7241 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7242 N1: Mul, N2: getValue(V: I.getArgOperand(i: 2)), Flags);
7243 setValue(V: &I, NewN: Add);
7244 }
7245 return;
7246 }
7247 case Intrinsic::fptosi_sat: {
7248 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7249 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT_SAT, DL: sdl, VT,
7250 N1: getValue(V: I.getArgOperand(i: 0)),
7251 N2: DAG.getValueType(VT.getScalarType())));
7252 return;
7253 }
7254 case Intrinsic::fptoui_sat: {
7255 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7256 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT_SAT, DL: sdl, VT,
7257 N1: getValue(V: I.getArgOperand(i: 0)),
7258 N2: DAG.getValueType(VT.getScalarType())));
7259 return;
7260 }
7261 case Intrinsic::convert_from_arbitrary_fp: {
7262 // Extract format metadata and convert to semantics enum.
7263 EVT DstVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7264 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7265 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
7266 const fltSemantics *SrcSem =
7267 APFloatBase::getArbitraryFPSemantics(Format: FormatStr);
7268 if (!SrcSem) {
7269 DAG.getContext()->emitError(
7270 ErrorStr: "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7271 "'");
7272 setValue(V: &I, NewN: DAG.getPOISON(VT: DstVT));
7273 return;
7274 }
7275 APFloatBase::Semantics SemEnum = APFloatBase::SemanticsToEnum(Sem: *SrcSem);
7276
7277 SDValue IntVal = getValue(V: I.getArgOperand(i: 0));
7278
7279 // Emit ISD::CONVERT_FROM_ARBITRARY_FP node.
7280 SDValue SemConst =
7281 DAG.getTargetConstant(Val: static_cast<int>(SemEnum), DL: sdl, VT: MVT::i32);
7282 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: sdl, VT: DstVT, N1: IntVal,
7283 N2: SemConst));
7284 return;
7285 }
7286 case Intrinsic::convert_to_arbitrary_fp: {
7287 // Extract format metadata and convert to semantics enum.
7288 EVT DstVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7289 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
7290 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
7291 const fltSemantics *DstSem =
7292 APFloatBase::getArbitraryFPSemantics(Format: FormatStr);
7293 if (!DstSem) {
7294 DAG.getContext()->emitError(
7295 ErrorStr: "convert_to_arbitrary_fp: not implemented format '" + FormatStr +
7296 "'");
7297 setValue(V: &I, NewN: DAG.getPOISON(VT: DstVT));
7298 return;
7299 }
7300 APFloatBase::Semantics SemEnum = APFloatBase::SemanticsToEnum(Sem: *DstSem);
7301
7302 Metadata *RoundMD =
7303 cast<MetadataAsValue>(Val: I.getArgOperand(i: 2))->getMetadata();
7304 StringRef RoundStr = cast<MDString>(Val: RoundMD)->getString();
7305 std::optional<RoundingMode> RoundMode = convertStrToRoundingMode(RoundStr);
7306 assert(RoundMode && *RoundMode != RoundingMode::Dynamic &&
7307 "Dynamic rounding mode should have been rejected by the verifier");
7308
7309 uint64_t Saturate =
7310 cast<ConstantInt>(Val: I.getArgOperand(i: 3))->getZExtValue() ? 1 : 0;
7311
7312 SDValue FloatVal = getValue(V: I.getArgOperand(i: 0));
7313
7314 SDValue SemConst =
7315 DAG.getTargetConstant(Val: static_cast<int>(SemEnum), DL: sdl, VT: MVT::i32);
7316 SDValue RoundConst =
7317 DAG.getTargetConstant(Val: static_cast<int>(*RoundMode), DL: sdl, VT: MVT::i32);
7318 SDValue SatConst = DAG.getTargetConstant(Val: Saturate, DL: sdl, VT: MVT::i32);
7319 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL: sdl, VT: DstVT, N1: FloatVal,
7320 N2: SemConst, N3: RoundConst, N4: SatConst));
7321 return;
7322 }
7323 case Intrinsic::set_rounding:
7324 Res = DAG.getNode(Opcode: ISD::SET_ROUNDING, DL: sdl, VT: MVT::Other,
7325 Ops: {getRoot(), getValue(V: I.getArgOperand(i: 0))});
7326 setValue(V: &I, NewN: Res);
7327 DAG.setRoot(Res.getValue(R: 0));
7328 return;
7329 case Intrinsic::is_fpclass: {
7330 const DataLayout DLayout = DAG.getDataLayout();
7331 EVT DestVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7332 EVT ArgVT = TLI.getValueType(DL: DLayout, Ty: I.getArgOperand(i: 0)->getType());
7333 FPClassTest Test = static_cast<FPClassTest>(
7334 cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue());
7335 MachineFunction &MF = DAG.getMachineFunction();
7336 const Function &F = MF.getFunction();
7337 SDValue Op = getValue(V: I.getArgOperand(i: 0));
7338 SDNodeFlags Flags;
7339 Flags.setNoFPExcept(
7340 !F.getAttributes().hasFnAttr(Kind: llvm::Attribute::StrictFP));
7341 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7342 // expansion can use illegal types. Making expansion early allows
7343 // legalizing these types prior to selection.
7344 if (!TLI.isOperationLegal(Op: ISD::IS_FPCLASS, VT: ArgVT) &&
7345 !TLI.isOperationCustom(Op: ISD::IS_FPCLASS, VT: ArgVT)) {
7346 SDValue Result = TLI.expandIS_FPCLASS(ResultVT: DestVT, Op, Test, Flags, DL: sdl, DAG);
7347 setValue(V: &I, NewN: Result);
7348 return;
7349 }
7350
7351 SDValue Check = DAG.getTargetConstant(Val: Test, DL: sdl, VT: MVT::i32);
7352 SDValue V = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: sdl, VT: DestVT, Ops: {Op, Check}, Flags);
7353 setValue(V: &I, NewN: V);
7354 return;
7355 }
7356 case Intrinsic::get_fpenv: {
7357 const DataLayout DLayout = DAG.getDataLayout();
7358 EVT EnvVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7359 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7360 SDValue Chain = getRoot();
7361 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7362 // and temporary storage in stack.
7363 if (TLI.isOperationLegalOrCustom(Op: ISD::GET_FPENV, VT: EnvVT)) {
7364 Res = DAG.getNode(
7365 Opcode: ISD::GET_FPENV, DL: sdl,
7366 VTList: DAG.getVTList(VT1: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
7367 VT2: MVT::Other),
7368 N: Chain);
7369 } else {
7370 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7371 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7372 auto MPI =
7373 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7374 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7375 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
7376 BaseAlignment: TempAlign);
7377 Chain = DAG.getGetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7378 Res = DAG.getLoad(VT: EnvVT, dl: sdl, Chain, Ptr: Temp, PtrInfo: MPI);
7379 }
7380 setValue(V: &I, NewN: Res);
7381 DAG.setRoot(Res.getValue(R: 1));
7382 return;
7383 }
7384 case Intrinsic::set_fpenv: {
7385 const DataLayout DLayout = DAG.getDataLayout();
7386 SDValue Env = getValue(V: I.getArgOperand(i: 0));
7387 EVT EnvVT = Env.getValueType();
7388 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7389 SDValue Chain = getRoot();
7390 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7391 // environment from memory.
7392 if (TLI.isOperationLegalOrCustom(Op: ISD::SET_FPENV, VT: EnvVT)) {
7393 Chain = DAG.getNode(Opcode: ISD::SET_FPENV, DL: sdl, VT: MVT::Other, N1: Chain, N2: Env);
7394 } else {
7395 // Allocate space in stack, copy environment bits into it and use this
7396 // memory in SET_FPENV_MEM.
7397 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7398 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7399 auto MPI =
7400 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7401 Chain = DAG.getStore(Chain, dl: sdl, Val: Env, Ptr: Temp, PtrInfo: MPI, Alignment: TempAlign,
7402 MMOFlags: MachineMemOperand::MOStore);
7403 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7404 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
7405 BaseAlignment: TempAlign);
7406 Chain = DAG.getSetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7407 }
7408 DAG.setRoot(Chain);
7409 return;
7410 }
7411 case Intrinsic::reset_fpenv:
7412 DAG.setRoot(DAG.getNode(Opcode: ISD::RESET_FPENV, DL: sdl, VT: MVT::Other, Operand: getRoot()));
7413 return;
7414 case Intrinsic::get_fpmode:
7415 Res = DAG.getNode(
7416 Opcode: ISD::GET_FPMODE, DL: sdl,
7417 VTList: DAG.getVTList(VT1: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
7418 VT2: MVT::Other),
7419 N: DAG.getRoot());
7420 setValue(V: &I, NewN: Res);
7421 DAG.setRoot(Res.getValue(R: 1));
7422 return;
7423 case Intrinsic::set_fpmode:
7424 Res = DAG.getNode(Opcode: ISD::SET_FPMODE, DL: sdl, VT: MVT::Other, N1: {DAG.getRoot()},
7425 N2: getValue(V: I.getArgOperand(i: 0)));
7426 DAG.setRoot(Res);
7427 return;
7428 case Intrinsic::reset_fpmode: {
7429 Res = DAG.getNode(Opcode: ISD::RESET_FPMODE, DL: sdl, VT: MVT::Other, Operand: getRoot());
7430 DAG.setRoot(Res);
7431 return;
7432 }
7433 case Intrinsic::pcmarker: {
7434 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
7435 DAG.setRoot(DAG.getNode(Opcode: ISD::PCMARKER, DL: sdl, VT: MVT::Other, N1: getRoot(), N2: Tmp));
7436 return;
7437 }
7438 case Intrinsic::readcyclecounter: {
7439 SDValue Op = getRoot();
7440 Res = DAG.getNode(Opcode: ISD::READCYCLECOUNTER, DL: sdl,
7441 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N: Op);
7442 setValue(V: &I, NewN: Res);
7443 DAG.setRoot(Res.getValue(R: 1));
7444 return;
7445 }
7446 case Intrinsic::readsteadycounter: {
7447 SDValue Op = getRoot();
7448 Res = DAG.getNode(Opcode: ISD::READSTEADYCOUNTER, DL: sdl,
7449 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N: Op);
7450 setValue(V: &I, NewN: Res);
7451 DAG.setRoot(Res.getValue(R: 1));
7452 return;
7453 }
7454 case Intrinsic::bitreverse:
7455 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITREVERSE, DL: sdl,
7456 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7457 Operand: getValue(V: I.getArgOperand(i: 0))));
7458 return;
7459 case Intrinsic::bswap:
7460 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BSWAP, DL: sdl,
7461 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7462 Operand: getValue(V: I.getArgOperand(i: 0))));
7463 return;
7464 case Intrinsic::cttz: {
7465 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7466 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7467 EVT Ty = Arg.getValueType();
7468 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_POISON,
7469 DL: sdl, VT: Ty, Operand: Arg));
7470 return;
7471 }
7472 case Intrinsic::ctlz: {
7473 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7474 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7475 EVT Ty = Arg.getValueType();
7476 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_POISON,
7477 DL: sdl, VT: Ty, Operand: Arg));
7478 return;
7479 }
7480 case Intrinsic::ctpop: {
7481 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7482 EVT Ty = Arg.getValueType();
7483 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CTPOP, DL: sdl, VT: Ty, Operand: Arg));
7484 return;
7485 }
7486 case Intrinsic::fshl:
7487 case Intrinsic::fshr: {
7488 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7489 SDValue X = getValue(V: I.getArgOperand(i: 0));
7490 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7491 SDValue Z = getValue(V: I.getArgOperand(i: 2));
7492 EVT VT = X.getValueType();
7493
7494 if (X == Y) {
7495 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7496 setValue(V: &I, NewN: DAG.getNode(Opcode: RotateOpcode, DL: sdl, VT, N1: X, N2: Z));
7497 } else {
7498 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7499 setValue(V: &I, NewN: DAG.getNode(Opcode: FunnelOpcode, DL: sdl, VT, N1: X, N2: Y, N3: Z));
7500 }
7501 return;
7502 }
7503 case Intrinsic::clmul: {
7504 SDValue X = getValue(V: I.getArgOperand(i: 0));
7505 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7506 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CLMUL, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7507 return;
7508 }
7509 case Intrinsic::pext: {
7510 SDValue X = getValue(V: I.getArgOperand(i: 0));
7511 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7512 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::PEXT, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7513 return;
7514 }
7515 case Intrinsic::pdep: {
7516 SDValue X = getValue(V: I.getArgOperand(i: 0));
7517 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7518 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::PDEP, DL: sdl, VT: X.getValueType(), N1: X, N2: Y));
7519 return;
7520 }
7521 case Intrinsic::sadd_sat: {
7522 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7523 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7524 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7525 return;
7526 }
7527 case Intrinsic::uadd_sat: {
7528 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7529 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7530 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7531 return;
7532 }
7533 case Intrinsic::ssub_sat: {
7534 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7535 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7536 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SSUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7537 return;
7538 }
7539 case Intrinsic::usub_sat: {
7540 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7541 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7542 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::USUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7543 return;
7544 }
7545 case Intrinsic::sshl_sat:
7546 case Intrinsic::ushl_sat: {
7547 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7548 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7549
7550 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
7551 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
7552
7553 // Coerce the shift amount to the right type if we can. This exposes the
7554 // truncate or zext to optimization early.
7555 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
7556 assert(ShiftTy.getSizeInBits() >=
7557 Log2_32_Ceil(Op1.getValueSizeInBits()) &&
7558 "Unexpected shift type");
7559 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
7560 }
7561
7562 unsigned Opc =
7563 Intrinsic == Intrinsic::sshl_sat ? ISD::SSHLSAT : ISD::USHLSAT;
7564 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7565 return;
7566 }
7567 case Intrinsic::smul_fix:
7568 case Intrinsic::umul_fix:
7569 case Intrinsic::smul_fix_sat:
7570 case Intrinsic::umul_fix_sat: {
7571 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7572 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7573 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7574 setValue(V: &I, NewN: DAG.getNode(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7575 VT: Op1.getValueType(), N1: Op1, N2: Op2, N3: Op3));
7576 return;
7577 }
7578 case Intrinsic::sdiv_fix:
7579 case Intrinsic::udiv_fix:
7580 case Intrinsic::sdiv_fix_sat:
7581 case Intrinsic::udiv_fix_sat: {
7582 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7583 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7584 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7585 setValue(V: &I, NewN: expandDivFix(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7586 LHS: Op1, RHS: Op2, Scale: Op3, DAG, TLI));
7587 return;
7588 }
7589 case Intrinsic::smax: {
7590 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7591 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7592 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7593 return;
7594 }
7595 case Intrinsic::smin: {
7596 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7597 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7598 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7599 return;
7600 }
7601 case Intrinsic::umax: {
7602 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7603 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7604 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7605 return;
7606 }
7607 case Intrinsic::umin: {
7608 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7609 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7610 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7611 return;
7612 }
7613 case Intrinsic::abs: {
7614 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7615 bool IntMinIsPoison = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->isOne();
7616 unsigned Opc = IntMinIsPoison ? ISD::ABS_MIN_POISON : ISD::ABS;
7617 setValue(V: &I, NewN: DAG.getNode(Opcode: Opc, DL: sdl, VT: Op1.getValueType(), Operand: Op1));
7618 return;
7619 }
7620 case Intrinsic::scmp: {
7621 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7622 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7623 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7624 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7625 break;
7626 }
7627 case Intrinsic::ucmp: {
7628 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7629 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7630 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7631 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7632 break;
7633 }
7634 case Intrinsic::stackaddress:
7635 case Intrinsic::stacksave: {
7636 unsigned SDOpcode = Intrinsic == Intrinsic::stackaddress ? ISD::STACKADDRESS
7637 : ISD::STACKSAVE;
7638 SDValue Op = getRoot();
7639 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7640 Res = DAG.getNode(Opcode: SDOpcode, DL: sdl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), N: Op);
7641 setValue(V: &I, NewN: Res);
7642 DAG.setRoot(Res.getValue(R: 1));
7643 return;
7644 }
7645 case Intrinsic::stackrestore:
7646 Res = getValue(V: I.getArgOperand(i: 0));
7647 DAG.setRoot(DAG.getNode(Opcode: ISD::STACKRESTORE, DL: sdl, VT: MVT::Other, N1: getRoot(), N2: Res));
7648 return;
7649 case Intrinsic::get_dynamic_area_offset: {
7650 SDValue Op = getRoot();
7651 EVT ResTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7652 Res = DAG.getNode(Opcode: ISD::GET_DYNAMIC_AREA_OFFSET, DL: sdl, VTList: DAG.getVTList(VT: ResTy),
7653 N: Op);
7654 DAG.setRoot(Op);
7655 setValue(V: &I, NewN: Res);
7656 return;
7657 }
7658 case Intrinsic::stackguard: {
7659 MachineFunction &MF = DAG.getMachineFunction();
7660 const Module &M = *MF.getFunction().getParent();
7661 EVT PtrTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7662 SDValue Chain = getRoot();
7663 if (TLI.useLoadStackGuardNode(M)) {
7664 Res = getLoadStackGuard(DAG, DL: sdl, Chain);
7665 Res = DAG.getPtrExtOrTrunc(Op: Res, DL: sdl, VT: PtrTy);
7666 } else {
7667 const Value *Global = TLI.getSDagStackGuard(M, Libcalls: DAG.getLibcalls());
7668 if (!Global) {
7669 LLVMContext &Ctx = *DAG.getContext();
7670 Ctx.diagnose(DI: DiagnosticInfoGeneric("unable to lower stackguard"));
7671 setValue(V: &I, NewN: DAG.getPOISON(VT: PtrTy));
7672 return;
7673 }
7674
7675 Align Align = DAG.getDataLayout().getPrefTypeAlign(Ty: Global->getType());
7676 Res = DAG.getLoad(VT: PtrTy, dl: sdl, Chain, Ptr: getValue(V: Global),
7677 PtrInfo: MachinePointerInfo(Global, 0), Alignment: Align,
7678 MMOFlags: MachineMemOperand::MOVolatile);
7679 }
7680 // Mix the cookie with FP if enabled. Skip if using LOAD_STACK_GUARD
7681 // with post-RA mixing (AArch64 MSVCRT), as the mixing will be done during
7682 // post-RA expansion of LOAD_STACK_GUARD.
7683 if (TLI.useStackGuardMixFP() && !TLI.useLoadStackGuardNode(M))
7684 Res = TLI.emitStackGuardMixFP(DAG, Val: Res, DL: sdl);
7685 DAG.setRoot(Chain);
7686 setValue(V: &I, NewN: Res);
7687 return;
7688 }
7689 case Intrinsic::stackprotector: {
7690 // Emit code into the DAG to store the stack guard onto the stack.
7691 MachineFunction &MF = DAG.getMachineFunction();
7692 MachineFrameInfo &MFI = MF.getFrameInfo();
7693 const Module &M = *MF.getFunction().getParent();
7694 SDValue Src, Chain = getRoot();
7695
7696 if (TLI.useLoadStackGuardNode(M))
7697 Src = getLoadStackGuard(DAG, DL: sdl, Chain);
7698 else
7699 Src = getValue(V: I.getArgOperand(i: 0)); // The guard's value.
7700
7701 AllocaInst *Slot = cast<AllocaInst>(Val: I.getArgOperand(i: 1));
7702
7703 int FI = FuncInfo.StaticAllocaMap[Slot];
7704 MFI.setStackProtectorIndex(FI);
7705 EVT PtrTy = TLI.getFrameIndexTy(DL: DAG.getDataLayout());
7706
7707 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrTy);
7708
7709 // Store the stack protector onto the stack.
7710 Res = DAG.getStore(
7711 Chain, dl: sdl, Val: Src, Ptr: FIN,
7712 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI),
7713 Alignment: MaybeAlign(), MMOFlags: MachineMemOperand::MOVolatile);
7714 setValue(V: &I, NewN: Res);
7715 DAG.setRoot(Res);
7716 return;
7717 }
7718 case Intrinsic::objectsize:
7719 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7720
7721 case Intrinsic::is_constant:
7722 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7723
7724 case Intrinsic::annotation:
7725 case Intrinsic::ptr_annotation:
7726 case Intrinsic::launder_invariant_group:
7727 case Intrinsic::strip_invariant_group:
7728 // Drop the intrinsic, but forward the value
7729 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
7730 return;
7731
7732 case Intrinsic::type_test:
7733 case Intrinsic::public_type_test:
7734 case Intrinsic::type_checked_load:
7735 case Intrinsic::type_checked_load_relative: {
7736 // These intrinsics are expected to be lowered by the LowerTypeTests pass
7737 // before code generation. Surviving until here usually indicates a
7738 // misconfiguration, for instance when devirtualization is enabled but LTO
7739 // does not actually run.
7740 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7741 *I.getFunction(),
7742 Intrinsic::getBaseName(id: Intrinsic) +
7743 " intrinsic must be lowered by the LowerTypeTests pass "
7744 "before code generation",
7745 sdl.getDebugLoc()));
7746
7747 // Lower the result to poison so that compilation can continue and collect
7748 // any further diagnostics.
7749 setValueToPoison(V: &I, dl: sdl);
7750 return;
7751 }
7752
7753 case Intrinsic::assume:
7754 case Intrinsic::experimental_noalias_scope_decl:
7755 case Intrinsic::var_annotation:
7756 case Intrinsic::sideeffect:
7757 // Discard annotate attributes, noalias scope declarations, assumptions, and
7758 // artificial side-effects.
7759 return;
7760
7761 case Intrinsic::codeview_annotation: {
7762 // Emit a label associated with this metadata.
7763 MachineFunction &MF = DAG.getMachineFunction();
7764 MCSymbol *Label = MF.getContext().createTempSymbol(Name: "annotation", AlwaysAddSuffix: true);
7765 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
7766 MF.addCodeViewAnnotation(Label, MD: cast<MDNode>(Val: MD));
7767 Res = DAG.getLabelNode(Opcode: ISD::ANNOTATION_LABEL, dl: sdl, Root: getRoot(), Label);
7768 DAG.setRoot(Res);
7769 return;
7770 }
7771
7772 case Intrinsic::init_trampoline: {
7773 const Function *F = cast<Function>(Val: I.getArgOperand(i: 1)->stripPointerCasts());
7774
7775 SDValue Ops[6];
7776 Ops[0] = getRoot();
7777 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7778 Ops[2] = getValue(V: I.getArgOperand(i: 1));
7779 Ops[3] = getValue(V: I.getArgOperand(i: 2));
7780 Ops[4] = DAG.getSrcValue(v: I.getArgOperand(i: 0));
7781 Ops[5] = DAG.getSrcValue(v: F);
7782
7783 Res = DAG.getNode(Opcode: ISD::INIT_TRAMPOLINE, DL: sdl, VT: MVT::Other, Ops);
7784
7785 DAG.setRoot(Res);
7786 return;
7787 }
7788 case Intrinsic::adjust_trampoline:
7789 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ADJUST_TRAMPOLINE, DL: sdl,
7790 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
7791 Operand: getValue(V: I.getArgOperand(i: 0))));
7792 return;
7793 case Intrinsic::gcroot: {
7794 assert(DAG.getMachineFunction().getFunction().hasGC() &&
7795 "only valid in functions with gc specified, enforced by Verifier");
7796 assert(GFI && "implied by previous");
7797 const Value *Alloca = I.getArgOperand(i: 0)->stripPointerCasts();
7798 const Constant *TypeMap = cast<Constant>(Val: I.getArgOperand(i: 1));
7799
7800 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(Val: getValue(V: Alloca).getNode());
7801 GFI->addStackRoot(Num: FI->getIndex(), Metadata: TypeMap);
7802 return;
7803 }
7804 case Intrinsic::gcread:
7805 case Intrinsic::gcwrite:
7806 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7807 case Intrinsic::get_rounding:
7808 Res = DAG.getNode(Opcode: ISD::GET_ROUNDING, DL: sdl, ResultTys: {MVT::i32, MVT::Other}, Ops: getRoot());
7809 setValue(V: &I, NewN: Res);
7810 DAG.setRoot(Res.getValue(R: 1));
7811 return;
7812
7813 case Intrinsic::expect:
7814 case Intrinsic::expect_with_probability:
7815 // Just replace __builtin_expect(exp, c) and
7816 // __builtin_expect_with_probability(exp, c, p) with EXP.
7817 setValue(V: &I, NewN: getValue(V: I.getArgOperand(i: 0)));
7818 return;
7819
7820 case Intrinsic::ubsantrap:
7821 case Intrinsic::debugtrap:
7822 case Intrinsic::trap: {
7823 StringRef TrapFuncName =
7824 I.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
7825 if (TrapFuncName.empty()) {
7826 switch (Intrinsic) {
7827 case Intrinsic::trap:
7828 DAG.setRoot(DAG.getNode(Opcode: ISD::TRAP, DL: sdl, VT: MVT::Other, Operand: getRoot()));
7829 break;
7830 case Intrinsic::debugtrap:
7831 DAG.setRoot(DAG.getNode(Opcode: ISD::DEBUGTRAP, DL: sdl, VT: MVT::Other, Operand: getRoot()));
7832 break;
7833 case Intrinsic::ubsantrap:
7834 DAG.setRoot(DAG.getNode(
7835 Opcode: ISD::UBSANTRAP, DL: sdl, VT: MVT::Other, N1: getRoot(),
7836 N2: DAG.getTargetConstant(
7837 Val: cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue(), DL: sdl,
7838 VT: MVT::i32)));
7839 break;
7840 default: llvm_unreachable("unknown trap intrinsic");
7841 }
7842 DAG.addNoMergeSiteInfo(Node: DAG.getRoot().getNode(),
7843 NoMerge: I.hasFnAttr(Kind: Attribute::NoMerge));
7844 return;
7845 }
7846 TargetLowering::ArgListTy Args;
7847 if (Intrinsic == Intrinsic::ubsantrap) {
7848 Value *Arg = I.getArgOperand(i: 0);
7849 Args.emplace_back(args&: Arg, args: getValue(V: Arg));
7850 }
7851
7852 TargetLowering::CallLoweringInfo CLI(DAG);
7853 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7854 CC: CallingConv::C, ResultType: I.getType(),
7855 Target: DAG.getExternalSymbol(Sym: TrapFuncName.data(),
7856 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
7857 ArgsList: std::move(Args));
7858 CLI.NoMerge = I.hasFnAttr(Kind: Attribute::NoMerge);
7859 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7860 DAG.setRoot(Result.second);
7861 return;
7862 }
7863
7864 case Intrinsic::allow_runtime_check:
7865 case Intrinsic::allow_ubsan_check:
7866 setValue(V: &I, NewN: getValue(V: ConstantInt::getTrue(Ty: I.getType())));
7867 return;
7868
7869 case Intrinsic::uadd_with_overflow:
7870 case Intrinsic::sadd_with_overflow:
7871 case Intrinsic::usub_with_overflow:
7872 case Intrinsic::ssub_with_overflow:
7873 case Intrinsic::umul_with_overflow:
7874 case Intrinsic::smul_with_overflow: {
7875 ISD::NodeType Op;
7876 switch (Intrinsic) {
7877 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7878 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7879 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7880 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7881 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7882 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7883 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7884 }
7885 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7886 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7887
7888 EVT ResultVT = Op1.getValueType();
7889 EVT OverflowVT = ResultVT.changeElementType(Context&: *Context, EltVT: MVT::i1);
7890
7891 SDVTList VTs = DAG.getVTList(VT1: ResultVT, VT2: OverflowVT);
7892 setValue(V: &I, NewN: DAG.getNode(Opcode: Op, DL: sdl, VTList: VTs, N1: Op1, N2: Op2));
7893 return;
7894 }
7895 case Intrinsic::prefetch: {
7896 SDValue Ops[5];
7897 unsigned rw = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
7898 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
7899 Ops[0] = DAG.getRoot();
7900 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7901 Ops[2] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 1)), DL: sdl,
7902 VT: MVT::i32);
7903 Ops[3] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 2)), DL: sdl,
7904 VT: MVT::i32);
7905 Ops[4] = DAG.getTargetConstant(Val: *cast<ConstantInt>(Val: I.getArgOperand(i: 3)), DL: sdl,
7906 VT: MVT::i32);
7907 SDValue Result = DAG.getMemIntrinsicNode(
7908 Opcode: ISD::PREFETCH, dl: sdl, VTList: DAG.getVTList(VT: MVT::Other), Ops,
7909 MemVT: EVT::getIntegerVT(Context&: *Context, BitWidth: 8), PtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
7910 /* align */ Alignment: std::nullopt, Flags);
7911
7912 // Chain the prefetch in parallel with any pending loads, to stay out of
7913 // the way of later optimizations.
7914 PendingLoads.push_back(Elt: Result);
7915 Result = getRoot();
7916 DAG.setRoot(Result);
7917 return;
7918 }
7919 case Intrinsic::lifetime_start:
7920 case Intrinsic::lifetime_end: {
7921 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7922 // Stack coloring is not enabled in O0, discard region information.
7923 if (TM.getOptLevel() == CodeGenOptLevel::None)
7924 return;
7925
7926 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(Val: I.getArgOperand(i: 0));
7927 if (!LifetimeObject)
7928 return;
7929
7930 // First check that the Alloca is static, otherwise it won't have a
7931 // valid frame index.
7932 auto SI = FuncInfo.StaticAllocaMap.find(Val: LifetimeObject);
7933 if (SI == FuncInfo.StaticAllocaMap.end())
7934 return;
7935
7936 const int FrameIndex = SI->second;
7937 Res = DAG.getLifetimeNode(IsStart, dl: sdl, Chain: getRoot(), FrameIndex);
7938 DAG.setRoot(Res);
7939 return;
7940 }
7941 case Intrinsic::pseudoprobe: {
7942 auto Guid = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue();
7943 auto Index = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
7944 auto Attr = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
7945 Res = DAG.getPseudoProbeNode(Dl: sdl, Chain: getRoot(), Guid, Index, Attr);
7946 DAG.setRoot(Res);
7947 return;
7948 }
7949 case Intrinsic::invariant_start:
7950 // Discard region information.
7951 setValue(V: &I,
7952 NewN: DAG.getUNDEF(VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
7953 return;
7954 case Intrinsic::invariant_end:
7955 // Discard region information.
7956 return;
7957 case Intrinsic::clear_cache: {
7958 SDValue InputChain = DAG.getRoot();
7959 SDValue StartVal = getValue(V: I.getArgOperand(i: 0));
7960 SDValue EndVal = getValue(V: I.getArgOperand(i: 1));
7961 Res = DAG.getNode(Opcode: ISD::CLEAR_CACHE, DL: sdl, VTList: DAG.getVTList(VT: MVT::Other),
7962 Ops: {InputChain, StartVal, EndVal});
7963 setValue(V: &I, NewN: Res);
7964 DAG.setRoot(Res);
7965 return;
7966 }
7967 case Intrinsic::donothing:
7968 case Intrinsic::seh_try_begin:
7969 case Intrinsic::seh_scope_begin:
7970 case Intrinsic::seh_try_end:
7971 case Intrinsic::seh_scope_end:
7972 // ignore
7973 return;
7974 case Intrinsic::experimental_stackmap:
7975 visitStackmap(I);
7976 return;
7977 case Intrinsic::experimental_patchpoint_void:
7978 case Intrinsic::experimental_patchpoint:
7979 visitPatchpoint(CB: I);
7980 return;
7981 case Intrinsic::experimental_gc_statepoint:
7982 LowerStatepoint(I: cast<GCStatepointInst>(Val: I));
7983 return;
7984 case Intrinsic::experimental_gc_result:
7985 visitGCResult(I: cast<GCResultInst>(Val: I));
7986 return;
7987 case Intrinsic::experimental_gc_relocate:
7988 visitGCRelocate(Relocate: cast<GCRelocateInst>(Val: I));
7989 return;
7990 case Intrinsic::instrprof_cover:
7991 llvm_unreachable("instrprof failed to lower a cover");
7992 case Intrinsic::instrprof_increment:
7993 llvm_unreachable("instrprof failed to lower an increment");
7994 case Intrinsic::instrprof_timestamp:
7995 llvm_unreachable("instrprof failed to lower a timestamp");
7996 case Intrinsic::instrprof_value_profile:
7997 llvm_unreachable("instrprof failed to lower a value profiling call");
7998 case Intrinsic::instrprof_mcdc_parameters:
7999 llvm_unreachable("instrprof failed to lower mcdc parameters");
8000 case Intrinsic::instrprof_mcdc_tvbitmap_update:
8001 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
8002 case Intrinsic::localescape: {
8003 MachineFunction &MF = DAG.getMachineFunction();
8004 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
8005
8006 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
8007 // is the same on all targets.
8008 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
8009 Value *Arg = I.getArgOperand(i: Idx)->stripPointerCasts();
8010 if (isa<ConstantPointerNull>(Val: Arg))
8011 continue; // Skip null pointers. They represent a hole in index space.
8012 AllocaInst *Slot = cast<AllocaInst>(Val: Arg);
8013 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
8014 "can only escape static allocas");
8015 int FI = FuncInfo.StaticAllocaMap[Slot];
8016 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8017 FuncName: GlobalValue::dropLLVMManglingEscape(Name: MF.getName()), Idx);
8018 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: dl,
8019 MCID: TII->get(Opcode: TargetOpcode::LOCAL_ESCAPE))
8020 .addSym(Sym: FrameAllocSym)
8021 .addFrameIndex(Idx: FI);
8022 }
8023
8024 return;
8025 }
8026
8027 case Intrinsic::localrecover: {
8028 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
8029 MachineFunction &MF = DAG.getMachineFunction();
8030
8031 // Get the symbol that defines the frame offset.
8032 auto *Fn = cast<Function>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
8033 auto *Idx = cast<ConstantInt>(Val: I.getArgOperand(i: 2));
8034 unsigned IdxVal =
8035 unsigned(Idx->getLimitedValue(Limit: std::numeric_limits<int>::max()));
8036 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8037 FuncName: GlobalValue::dropLLVMManglingEscape(Name: Fn->getName()), Idx: IdxVal);
8038
8039 Value *FP = I.getArgOperand(i: 1);
8040 SDValue FPVal = getValue(V: FP);
8041 EVT PtrVT = FPVal.getValueType();
8042
8043 // Create a MCSymbol for the label to avoid any target lowering
8044 // that would make this PC relative.
8045 SDValue OffsetSym = DAG.getMCSymbol(Sym: FrameAllocSym, VT: PtrVT);
8046 SDValue OffsetVal =
8047 DAG.getNode(Opcode: ISD::LOCAL_RECOVER, DL: sdl, VT: PtrVT, Operand: OffsetSym);
8048
8049 // Add the offset to the FP.
8050 SDValue Add = DAG.getMemBasePlusOffset(Base: FPVal, Offset: OffsetVal, DL: sdl);
8051 setValue(V: &I, NewN: Add);
8052
8053 return;
8054 }
8055
8056 case Intrinsic::fake_use: {
8057 Value *V = I.getArgOperand(i: 0);
8058 SDValue Ops[2];
8059 // For Values not declared or previously used in this basic block, the
8060 // NodeMap will not have an entry, and `getValue` will assert if V has no
8061 // valid register value.
8062 auto FakeUseValue = [&]() -> SDValue {
8063 SDValue &N = NodeMap[V];
8064 if (N.getNode())
8065 return N;
8066
8067 // If there's a virtual register allocated and initialized for this
8068 // value, use it.
8069 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
8070 return copyFromReg;
8071 // FIXME: Do we want to preserve constants? It seems pointless.
8072 if (isa<Constant>(Val: V))
8073 return getValue(V);
8074 return SDValue();
8075 }();
8076 if (!FakeUseValue || FakeUseValue.isUndef())
8077 return;
8078 Ops[0] = getRoot();
8079 Ops[1] = FakeUseValue;
8080 // Also, do not translate a fake use with an undef operand, or any other
8081 // empty SDValues.
8082 if (!Ops[1] || Ops[1].isUndef())
8083 return;
8084 DAG.setRoot(DAG.getNode(Opcode: ISD::FAKE_USE, DL: sdl, VT: MVT::Other, Ops));
8085 return;
8086 }
8087
8088 case Intrinsic::reloc_none: {
8089 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
8090 StringRef SymbolName = cast<MDString>(Val: MD)->getString();
8091 SDValue Ops[2] = {
8092 getRoot(),
8093 DAG.getTargetExternalSymbol(
8094 Sym: SymbolName.data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout()))};
8095 DAG.setRoot(DAG.getNode(Opcode: ISD::RELOC_NONE, DL: sdl, VT: MVT::Other, Ops));
8096 return;
8097 }
8098
8099 case Intrinsic::cond_loop: {
8100 SDValue InputChain = DAG.getRoot();
8101 SDValue P = getValue(V: I.getArgOperand(i: 0));
8102 Res = DAG.getNode(Opcode: ISD::COND_LOOP, DL: sdl, VTList: DAG.getVTList(VT: MVT::Other),
8103 Ops: {InputChain, P});
8104 setValue(V: &I, NewN: Res);
8105 DAG.setRoot(Res);
8106 return;
8107 }
8108
8109 case Intrinsic::eh_exceptionpointer:
8110 case Intrinsic::eh_exceptioncode: {
8111 // Get the exception pointer vreg, copy from it, and resize it to fit.
8112 const auto *CPI = cast<CatchPadInst>(Val: I.getArgOperand(i: 0));
8113 MVT PtrVT = TLI.getPointerTy(DL: DAG.getDataLayout());
8114 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(VT: PtrVT);
8115 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, RC: PtrRC);
8116 SDValue N = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: sdl, Reg: VReg, VT: PtrVT);
8117 if (Intrinsic == Intrinsic::eh_exceptioncode)
8118 N = DAG.getZExtOrTrunc(Op: N, DL: sdl, VT: MVT::i32);
8119 setValue(V: &I, NewN: N);
8120 return;
8121 }
8122 case Intrinsic::xray_customevent: {
8123 // Here we want to make sure that the intrinsic behaves as if it has a
8124 // specific calling convention.
8125 const auto &Triple = DAG.getTarget().getTargetTriple();
8126 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64 &&
8127 Triple.getArch() != Triple::hexagon)
8128 return;
8129
8130 SmallVector<SDValue, 8> Ops;
8131
8132 // We want to say that we always want the arguments in registers.
8133 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 0));
8134 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 1));
8135 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
8136 SDValue Chain = getRoot();
8137 Ops.push_back(Elt: LogEntryVal);
8138 Ops.push_back(Elt: StrSizeVal);
8139 Ops.push_back(Elt: Chain);
8140
8141 // We need to enforce the calling convention for the callsite, so that
8142 // argument ordering is enforced correctly, and that register allocation can
8143 // see that some registers may be assumed clobbered and have to preserve
8144 // them across calls to the intrinsic.
8145 MachineSDNode *MN = DAG.getMachineNode(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL,
8146 dl: sdl, VTs: NodeTys, Ops);
8147 SDValue patchableNode = SDValue(MN, 0);
8148 DAG.setRoot(patchableNode);
8149 setValue(V: &I, NewN: patchableNode);
8150 return;
8151 }
8152 case Intrinsic::xray_typedevent: {
8153 // Here we want to make sure that the intrinsic behaves as if it has a
8154 // specific calling convention.
8155 const auto &Triple = DAG.getTarget().getTargetTriple();
8156 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64 &&
8157 Triple.getArch() != Triple::hexagon)
8158 return;
8159
8160 SmallVector<SDValue, 8> Ops;
8161
8162 // We want to say that we always want the arguments in registers.
8163 // It's unclear to me how manipulating the selection DAG here forces callers
8164 // to provide arguments in registers instead of on the stack.
8165 SDValue LogTypeId = getValue(V: I.getArgOperand(i: 0));
8166 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 1));
8167 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 2));
8168 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
8169 SDValue Chain = getRoot();
8170 Ops.push_back(Elt: LogTypeId);
8171 Ops.push_back(Elt: LogEntryVal);
8172 Ops.push_back(Elt: StrSizeVal);
8173 Ops.push_back(Elt: Chain);
8174
8175 // We need to enforce the calling convention for the callsite, so that
8176 // argument ordering is enforced correctly, and that register allocation can
8177 // see that some registers may be assumed clobbered and have to preserve
8178 // them across calls to the intrinsic.
8179 MachineSDNode *MN = DAG.getMachineNode(
8180 Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, dl: sdl, VTs: NodeTys, Ops);
8181 SDValue patchableNode = SDValue(MN, 0);
8182 DAG.setRoot(patchableNode);
8183 setValue(V: &I, NewN: patchableNode);
8184 return;
8185 }
8186 case Intrinsic::experimental_deoptimize:
8187 LowerDeoptimizeCall(CI: &I);
8188 return;
8189 case Intrinsic::stepvector:
8190 visitStepVector(I);
8191 return;
8192 case Intrinsic::vector_reduce_fadd:
8193 case Intrinsic::vector_reduce_fmul:
8194 case Intrinsic::vector_reduce_add:
8195 case Intrinsic::vector_reduce_mul:
8196 case Intrinsic::vector_reduce_and:
8197 case Intrinsic::vector_reduce_or:
8198 case Intrinsic::vector_reduce_xor:
8199 case Intrinsic::vector_reduce_smax:
8200 case Intrinsic::vector_reduce_smin:
8201 case Intrinsic::vector_reduce_umax:
8202 case Intrinsic::vector_reduce_umin:
8203 case Intrinsic::vector_reduce_fmax:
8204 case Intrinsic::vector_reduce_fmin:
8205 case Intrinsic::vector_reduce_fmaximum:
8206 case Intrinsic::vector_reduce_fminimum:
8207 visitVectorReduce(I, Intrinsic);
8208 return;
8209
8210 case Intrinsic::icall_branch_funnel: {
8211 SmallVector<SDValue, 16> Ops;
8212 Ops.push_back(Elt: getValue(V: I.getArgOperand(i: 0)));
8213
8214 int64_t Offset;
8215 auto *Base = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
8216 Ptr: I.getArgOperand(i: 1), Offset, DL: DAG.getDataLayout()));
8217 if (!Base)
8218 report_fatal_error(
8219 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
8220 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Base, DL: sdl, VT: MVT::i64, offset: 0));
8221
8222 struct BranchFunnelTarget {
8223 int64_t Offset;
8224 SDValue Target;
8225 };
8226 SmallVector<BranchFunnelTarget, 8> Targets;
8227
8228 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
8229 auto *ElemBase = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
8230 Ptr: I.getArgOperand(i: Op), Offset, DL: DAG.getDataLayout()));
8231 if (ElemBase != Base)
8232 report_fatal_error(reason: "all llvm.icall.branch.funnel operands must refer "
8233 "to the same GlobalValue");
8234
8235 SDValue Val = getValue(V: I.getArgOperand(i: Op + 1));
8236 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
8237 if (!GA)
8238 report_fatal_error(
8239 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
8240 Targets.push_back(Elt: {.Offset: Offset, .Target: DAG.getTargetGlobalAddress(
8241 GV: GA->getGlobal(), DL: sdl, VT: Val.getValueType(),
8242 offset: GA->getOffset())});
8243 }
8244 llvm::sort(C&: Targets,
8245 Comp: [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
8246 return T1.Offset < T2.Offset;
8247 });
8248
8249 for (auto &T : Targets) {
8250 Ops.push_back(Elt: DAG.getTargetConstant(Val: T.Offset, DL: sdl, VT: MVT::i32));
8251 Ops.push_back(Elt: T.Target);
8252 }
8253
8254 Ops.push_back(Elt: DAG.getRoot()); // Chain
8255 SDValue N(DAG.getMachineNode(Opcode: TargetOpcode::ICALL_BRANCH_FUNNEL, dl: sdl,
8256 VT: MVT::Other, Ops),
8257 0);
8258 DAG.setRoot(N);
8259 setValue(V: &I, NewN: N);
8260 HasTailCall = true;
8261 return;
8262 }
8263
8264 case Intrinsic::wasm_landingpad_index:
8265 // Information this intrinsic contained has been transferred to
8266 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
8267 // delete it now.
8268 return;
8269
8270 case Intrinsic::aarch64_settag:
8271 case Intrinsic::aarch64_settag_zero: {
8272 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8273 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
8274 SDValue Val = TSI.EmitTargetCodeForSetTag(
8275 DAG, dl: sdl, Chain: getRoot(), Addr: getValue(V: I.getArgOperand(i: 0)),
8276 Size: getValue(V: I.getArgOperand(i: 1)), DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
8277 ZeroData: ZeroMemory);
8278 DAG.setRoot(Val);
8279 setValue(V: &I, NewN: Val);
8280 return;
8281 }
8282 case Intrinsic::amdgcn_cs_chain: {
8283 // At this point we don't care if it's amdgpu_cs_chain or
8284 // amdgpu_cs_chain_preserve.
8285 CallingConv::ID CC = CallingConv::AMDGPU_CS_Chain;
8286
8287 Type *RetTy = I.getType();
8288 assert(RetTy->isVoidTy() && "Should not return");
8289
8290 SDValue Callee = getValue(V: I.getOperand(i_nocapture: 0));
8291
8292 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
8293 // We'll also tack the value of the EXEC mask at the end.
8294 TargetLowering::ArgListTy Args;
8295 Args.reserve(n: 3);
8296
8297 for (unsigned Idx : {2, 3, 1}) {
8298 TargetLowering::ArgListEntry Arg(getValue(V: I.getOperand(i_nocapture: Idx)),
8299 I.getOperand(i_nocapture: Idx)->getType());
8300 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8301 Args.push_back(x: Arg);
8302 }
8303
8304 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
8305 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
8306 Args[2].IsInReg = true; // EXEC should be inreg
8307
8308 // Forward the flags and any additional arguments.
8309 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
8310 TargetLowering::ArgListEntry Arg(getValue(V: I.getOperand(i_nocapture: Idx)),
8311 I.getOperand(i_nocapture: Idx)->getType());
8312 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8313 Args.push_back(x: Arg);
8314 }
8315
8316 TargetLowering::CallLoweringInfo CLI(DAG);
8317 CLI.setDebugLoc(getCurSDLoc())
8318 .setChain(getRoot())
8319 .setCallee(CC, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
8320 .setNoReturn(true)
8321 .setTailCall(true)
8322 .setConvergent(I.isConvergent());
8323 CLI.CB = &I;
8324 std::pair<SDValue, SDValue> Result =
8325 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
8326 (void)Result;
8327 assert(!Result.first.getNode() && !Result.second.getNode() &&
8328 "Should've lowered as tail call");
8329
8330 HasTailCall = true;
8331 return;
8332 }
8333 case Intrinsic::amdgcn_call_whole_wave: {
8334 TargetLowering::ArgListTy Args;
8335 bool isTailCall = I.isTailCall();
8336
8337 // The first argument is the callee. Skip it when assembling the call args.
8338 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
8339 TargetLowering::ArgListEntry Arg(getValue(V: I.getArgOperand(i: Idx)),
8340 I.getArgOperand(i: Idx)->getType());
8341 Arg.setAttributes(Call: &I, ArgIdx: Idx);
8342
8343 // If we have an explicit sret argument that is an Instruction, (i.e., it
8344 // might point to function-local memory), we can't meaningfully tail-call.
8345 if (Arg.IsSRet && isa<Instruction>(Val: I.getArgOperand(i: Idx)))
8346 isTailCall = false;
8347
8348 Args.push_back(x: Arg);
8349 }
8350
8351 SDValue ConvControlToken;
8352 if (auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
8353 auto *Token = Bundle->Inputs[0].get();
8354 ConvControlToken = getValue(V: Token);
8355 }
8356
8357 TargetLowering::CallLoweringInfo CLI(DAG);
8358 CLI.setDebugLoc(getCurSDLoc())
8359 .setChain(getRoot())
8360 .setCallee(CC: CallingConv::AMDGPU_Gfx_WholeWave, ResultType: I.getType(),
8361 Target: getValue(V: I.getArgOperand(i: 0)), ArgsList: std::move(Args))
8362 .setTailCall(isTailCall && canTailCall(CB: I))
8363 .setIsPreallocated(
8364 I.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
8365 .setConvergent(I.isConvergent())
8366 .setConvergenceControlToken(ConvControlToken);
8367 CLI.CB = &I;
8368
8369 std::pair<SDValue, SDValue> Result =
8370 lowerInvokable(CLI, /*EHPadBB=*/nullptr);
8371
8372 if (Result.first.getNode())
8373 setValue(V: &I, NewN: Result.first);
8374 return;
8375 }
8376 case Intrinsic::ptrmask: {
8377 SDValue Ptr = getValue(V: I.getOperand(i_nocapture: 0));
8378 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
8379
8380 // On arm64_32, pointers are 32 bits when stored in memory, but
8381 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
8382 // match the index type, but the pointer is 64 bits, so the mask must be
8383 // zero-extended up to 64 bits to match the pointer.
8384 EVT PtrVT =
8385 TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
8386 EVT MemVT =
8387 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
8388 assert(PtrVT == Ptr.getValueType());
8389 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
8390 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
8391 // 128-bit, so we have to pad the mask with ones for unused bits.
8392 auto HighOnes = DAG.getNode(
8393 Opcode: ISD::SHL, DL: sdl, VT: PtrVT, N1: DAG.getAllOnesConstant(DL: sdl, VT: PtrVT),
8394 N2: DAG.getShiftAmountConstant(Val: Mask.getValueType().getFixedSizeInBits(),
8395 VT: PtrVT, DL: sdl));
8396 Mask = DAG.getNode(Opcode: ISD::OR, DL: sdl, VT: PtrVT,
8397 N1: DAG.getZExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT), N2: HighOnes);
8398 } else if (Mask.getValueType() != PtrVT)
8399 Mask = DAG.getPtrExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT);
8400
8401 assert(Mask.getValueType() == PtrVT);
8402 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::AND, DL: sdl, VT: PtrVT, N1: Ptr, N2: Mask));
8403 return;
8404 }
8405 case Intrinsic::threadlocal_address: {
8406 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
8407 return;
8408 }
8409 case Intrinsic::get_active_lane_mask: {
8410 EVT CCVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8411 SDValue Index = getValue(V: I.getOperand(i_nocapture: 0));
8412 SDValue TripCount = getValue(V: I.getOperand(i_nocapture: 1));
8413 EVT ElementVT = Index.getValueType();
8414
8415 if (!TLI.shouldExpandGetActiveLaneMask(VT: CCVT, OpVT: ElementVT)) {
8416 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: sdl, VT: CCVT, N1: Index,
8417 N2: TripCount));
8418 return;
8419 }
8420
8421 EVT VecTy = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElementVT,
8422 EC: CCVT.getVectorElementCount());
8423
8424 SDValue VectorIndex = DAG.getSplat(VT: VecTy, DL: sdl, Op: Index);
8425 SDValue VectorTripCount = DAG.getSplat(VT: VecTy, DL: sdl, Op: TripCount);
8426 SDValue VectorStep = DAG.getStepVector(DL: sdl, ResVT: VecTy);
8427 SDValue VectorInduction = DAG.getNode(
8428 Opcode: ISD::UADDSAT, DL: sdl, VT: VecTy, N1: VectorIndex, N2: VectorStep);
8429 SDValue SetCC = DAG.getSetCC(DL: sdl, VT: CCVT, LHS: VectorInduction,
8430 RHS: VectorTripCount, Cond: ISD::CondCode::SETULT);
8431 setValue(V: &I, NewN: SetCC);
8432 return;
8433 }
8434 case Intrinsic::experimental_get_vector_length: {
8435 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8436 "Expected positive VF");
8437 unsigned VF = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 1))->getZExtValue();
8438 bool IsScalable = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->isOne();
8439
8440 SDValue Count = getValue(V: I.getOperand(i_nocapture: 0));
8441 EVT CountVT = Count.getValueType();
8442
8443 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8444 visitTargetIntrinsic(I, Intrinsic);
8445 return;
8446 }
8447
8448 // Expand to a umin between the trip count and the maximum elements the type
8449 // can hold.
8450 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8451
8452 // Extend the trip count to at least the result VT.
8453 if (CountVT.bitsLT(VT)) {
8454 Count = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: sdl, VT, Operand: Count);
8455 CountVT = VT;
8456 }
8457
8458 SDValue MaxEVL = DAG.getElementCount(DL: sdl, VT: CountVT,
8459 EC: ElementCount::get(MinVal: VF, Scalable: IsScalable));
8460
8461 SDValue UMin = DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: CountVT, N1: Count, N2: MaxEVL);
8462 // Clip to the result type if needed.
8463 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: sdl, VT, Operand: UMin);
8464
8465 setValue(V: &I, NewN: Trunc);
8466 return;
8467 }
8468 case Intrinsic::vector_partial_reduce_add: {
8469 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8470 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8471 setValue(V: &I,
8472 NewN: DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_UMLA, DL: sdl, VT: Acc.getValueType(), N1: Acc,
8473 N2: Input, N3: DAG.getConstant(Val: 1, DL: sdl, VT: Input.getValueType())));
8474 return;
8475 }
8476 case Intrinsic::vector_partial_reduce_fadd: {
8477 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8478 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8479 setValue(V: &I, NewN: DAG.getNode(
8480 Opcode: ISD::PARTIAL_REDUCE_FMLA, DL: sdl, VT: Acc.getValueType(), N1: Acc,
8481 N2: Input, N3: DAG.getConstantFP(Val: 1.0, DL: sdl, VT: Input.getValueType())));
8482 return;
8483 }
8484 case Intrinsic::experimental_cttz_elts: {
8485 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
8486 EVT OpVT = Op.getValueType();
8487 EVT RetTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8488 bool ZeroIsPoison =
8489 !cast<ConstantSDNode>(Val: getValue(V: I.getOperand(i_nocapture: 1)))->isZero();
8490 if (OpVT.getVectorElementType() != MVT::i1) {
8491 // Compare the input vector elements to zero & use to count trailing
8492 // zeros.
8493 SDValue AllZero = DAG.getConstant(Val: 0, DL: sdl, VT: OpVT);
8494 EVT I1OpVT = OpVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1);
8495 Op = DAG.getSetCC(DL: sdl, VT: I1OpVT, LHS: Op, RHS: AllZero, Cond: ISD::SETNE);
8496 }
8497 setValue(V: &I, NewN: DAG.getNode(Opcode: ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
8498 : ISD::CTTZ_ELTS,
8499 DL: sdl, VT: RetTy, Operand: Op));
8500 return;
8501 }
8502 case Intrinsic::vector_insert: {
8503 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8504 SDValue SubVec = getValue(V: I.getOperand(i_nocapture: 1));
8505 SDValue Index = getValue(V: I.getOperand(i_nocapture: 2));
8506
8507 // The intrinsic's index type is i64, but the SDNode requires an index type
8508 // suitable for the target. Convert the index as required.
8509 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8510 if (Index.getValueType() != VectorIdxTy)
8511 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8512
8513 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8514 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: SubVec,
8515 N3: Index));
8516 return;
8517 }
8518 case Intrinsic::vector_extract: {
8519 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8520 SDValue Index = getValue(V: I.getOperand(i_nocapture: 1));
8521 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8522
8523 // The intrinsic's index type is i64, but the SDNode requires an index type
8524 // suitable for the target. Convert the index as required.
8525 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8526 if (Index.getValueType() != VectorIdxTy)
8527 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8528
8529 setValue(V: &I,
8530 NewN: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: Index));
8531 return;
8532 }
8533 case Intrinsic::experimental_vector_match: {
8534 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
8535 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
8536 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
8537 EVT Op1VT = Op1.getValueType();
8538 EVT Op2VT = Op2.getValueType();
8539 EVT ResVT = Mask.getValueType();
8540 unsigned SearchSize = Op2VT.getVectorNumElements();
8541
8542 // If the target has native support for this vector match operation, lower
8543 // the intrinsic untouched; otherwise, expand it below.
8544 if (!TLI.shouldExpandVectorMatch(VT: Op1VT, SearchSize)) {
8545 visitTargetIntrinsic(I, Intrinsic);
8546 return;
8547 }
8548
8549 SDValue Ret = DAG.getConstant(Val: 0, DL: sdl, VT: ResVT);
8550
8551 for (unsigned i = 0; i < SearchSize; ++i) {
8552 SDValue Op2Elem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: sdl,
8553 VT: Op2VT.getVectorElementType(), N1: Op2,
8554 N2: DAG.getVectorIdxConstant(Val: i, DL: sdl));
8555 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: sdl, VT: Op1VT, Operand: Op2Elem);
8556 SDValue Cmp = DAG.getSetCC(DL: sdl, VT: ResVT, LHS: Op1, RHS: Splat, Cond: ISD::SETEQ);
8557 Ret = DAG.getNode(Opcode: ISD::OR, DL: sdl, VT: ResVT, N1: Ret, N2: Cmp);
8558 }
8559
8560 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::AND, DL: sdl, VT: ResVT, N1: Ret, N2: Mask));
8561 return;
8562 }
8563 case Intrinsic::vector_reverse:
8564 visitVectorReverse(I);
8565 return;
8566 case Intrinsic::vector_splice_left:
8567 case Intrinsic::vector_splice_right:
8568 visitVectorSplice(I);
8569 return;
8570 case Intrinsic::callbr_landingpad:
8571 visitCallBrLandingPad(I);
8572 return;
8573 case Intrinsic::vector_interleave2:
8574 visitVectorInterleave(I, Factor: 2);
8575 return;
8576 case Intrinsic::vector_interleave3:
8577 visitVectorInterleave(I, Factor: 3);
8578 return;
8579 case Intrinsic::vector_interleave4:
8580 visitVectorInterleave(I, Factor: 4);
8581 return;
8582 case Intrinsic::vector_interleave5:
8583 visitVectorInterleave(I, Factor: 5);
8584 return;
8585 case Intrinsic::vector_interleave6:
8586 visitVectorInterleave(I, Factor: 6);
8587 return;
8588 case Intrinsic::vector_interleave7:
8589 visitVectorInterleave(I, Factor: 7);
8590 return;
8591 case Intrinsic::vector_interleave8:
8592 visitVectorInterleave(I, Factor: 8);
8593 return;
8594 case Intrinsic::vector_deinterleave2:
8595 visitVectorDeinterleave(I, Factor: 2);
8596 return;
8597 case Intrinsic::vector_deinterleave3:
8598 visitVectorDeinterleave(I, Factor: 3);
8599 return;
8600 case Intrinsic::vector_deinterleave4:
8601 visitVectorDeinterleave(I, Factor: 4);
8602 return;
8603 case Intrinsic::vector_deinterleave5:
8604 visitVectorDeinterleave(I, Factor: 5);
8605 return;
8606 case Intrinsic::vector_deinterleave6:
8607 visitVectorDeinterleave(I, Factor: 6);
8608 return;
8609 case Intrinsic::vector_deinterleave7:
8610 visitVectorDeinterleave(I, Factor: 7);
8611 return;
8612 case Intrinsic::vector_deinterleave8:
8613 visitVectorDeinterleave(I, Factor: 8);
8614 return;
8615 case Intrinsic::experimental_vector_compress:
8616 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: sdl,
8617 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
8618 N1: getValue(V: I.getArgOperand(i: 0)),
8619 N2: getValue(V: I.getArgOperand(i: 1)),
8620 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
8621 return;
8622 case Intrinsic::experimental_convergence_anchor:
8623 case Intrinsic::experimental_convergence_entry:
8624 case Intrinsic::experimental_convergence_loop:
8625 visitConvergenceControl(I, Intrinsic);
8626 return;
8627 case Intrinsic::experimental_vector_histogram_add: {
8628 visitVectorHistogram(I, IntrinsicID: Intrinsic);
8629 return;
8630 }
8631 case Intrinsic::experimental_vector_extract_last_active: {
8632 visitVectorExtractLastActive(I, Intrinsic);
8633 return;
8634 }
8635 case Intrinsic::loop_dependence_war_mask:
8636 setValue(V: &I,
8637 NewN: DAG.getNode(Opcode: ISD::LOOP_DEPENDENCE_WAR_MASK, DL: sdl,
8638 VT: EVT::getEVT(Ty: I.getType()), N1: getValue(V: I.getOperand(i_nocapture: 0)),
8639 N2: getValue(V: I.getOperand(i_nocapture: 1)), N3: getValue(V: I.getOperand(i_nocapture: 2)),
8640 N4: DAG.getConstant(Val: 0, DL: sdl, VT: MVT::i64)));
8641 return;
8642 case Intrinsic::loop_dependence_raw_mask:
8643 setValue(V: &I,
8644 NewN: DAG.getNode(Opcode: ISD::LOOP_DEPENDENCE_RAW_MASK, DL: sdl,
8645 VT: EVT::getEVT(Ty: I.getType()), N1: getValue(V: I.getOperand(i_nocapture: 0)),
8646 N2: getValue(V: I.getOperand(i_nocapture: 1)), N3: getValue(V: I.getOperand(i_nocapture: 2)),
8647 N4: DAG.getConstant(Val: 0, DL: sdl, VT: MVT::i64)));
8648 return;
8649 case Intrinsic::masked_udiv:
8650 setValue(V: &I,
8651 NewN: DAG.getNode(Opcode: ISD::MASKED_UDIV, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8652 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8653 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8654 return;
8655 case Intrinsic::masked_sdiv:
8656 setValue(V: &I,
8657 NewN: DAG.getNode(Opcode: ISD::MASKED_SDIV, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8658 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8659 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8660 return;
8661 case Intrinsic::masked_urem:
8662 setValue(V: &I,
8663 NewN: DAG.getNode(Opcode: ISD::MASKED_UREM, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8664 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8665 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8666 return;
8667 case Intrinsic::masked_srem:
8668 setValue(V: &I,
8669 NewN: DAG.getNode(Opcode: ISD::MASKED_SREM, DL: sdl, VT: EVT::getEVT(Ty: I.getType()),
8670 N1: getValue(V: I.getOperand(i_nocapture: 0)), N2: getValue(V: I.getOperand(i_nocapture: 1)),
8671 N3: getValue(V: I.getOperand(i_nocapture: 2))));
8672 return;
8673 }
8674}
8675
8676void SelectionDAGBuilder::pushFPOpOutChain(SDValue Result,
8677 fp::ExceptionBehavior EB) {
8678 assert(Result.getNode()->getNumValues() == 2);
8679 SDValue OutChain = Result.getValue(R: 1);
8680 assert(OutChain.getValueType() == MVT::Other);
8681
8682 // Instead of updating the root immediately, push the produced chain to the
8683 // appropriate list, deferring the update until the root is requested. In this
8684 // case, the nodes from the lists are chained using TokenFactor, indicating
8685 // that the operations are independent.
8686 //
8687 // In particular, the root is updated before any call that might access the
8688 // floating-point environment, except for constrained intrinsics.
8689 switch (EB) {
8690 case fp::ExceptionBehavior::ebMayTrap:
8691 case fp::ExceptionBehavior::ebIgnore:
8692 PendingConstrainedFP.push_back(Elt: OutChain);
8693 break;
8694 case fp::ExceptionBehavior::ebStrict:
8695 PendingConstrainedFPStrict.push_back(Elt: OutChain);
8696 break;
8697 }
8698}
8699
8700void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8701 const ConstrainedFPIntrinsic &FPI) {
8702 SDLoc sdl = getCurSDLoc();
8703
8704 // We do not need to serialize constrained FP intrinsics against
8705 // each other or against (nonvolatile) loads, so they can be
8706 // chained like loads.
8707 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
8708 SDValue Chain = getFPOperationRoot(EB);
8709 SmallVector<SDValue, 4> Opers;
8710 Opers.push_back(Elt: Chain);
8711 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8712 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: I)));
8713
8714 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8715 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: FPI.getType());
8716 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
8717
8718 SDNodeFlags Flags;
8719 if (EB == fp::ExceptionBehavior::ebIgnore)
8720 Flags.setNoFPExcept(true);
8721
8722 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &FPI))
8723 Flags.copyFMF(FPMO: *FPOp);
8724
8725 unsigned Opcode;
8726 switch (FPI.getIntrinsicID()) {
8727 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8728#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8729 case Intrinsic::INTRINSIC: \
8730 Opcode = ISD::STRICT_##DAGN; \
8731 break;
8732#include "llvm/IR/ConstrainedOps.def"
8733 case Intrinsic::experimental_constrained_fmuladd: {
8734 Opcode = ISD::STRICT_FMA;
8735 // Break fmuladd into fmul and fadd.
8736 if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
8737 !TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
8738 Opers.pop_back();
8739 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8740 pushFPOpOutChain(Result: Mul, EB);
8741 Opcode = ISD::STRICT_FADD;
8742 Opers.clear();
8743 Opers.push_back(Elt: Mul.getValue(R: 1));
8744 Opers.push_back(Elt: Mul.getValue(R: 0));
8745 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: 2)));
8746 }
8747 break;
8748 }
8749 }
8750
8751 // A few strict DAG nodes carry additional operands that are not
8752 // set up by the default code above.
8753 switch (Opcode) {
8754 default: break;
8755 case ISD::STRICT_FP_ROUND:
8756 Opers.push_back(
8757 Elt: DAG.getTargetConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
8758 break;
8759 case ISD::STRICT_FSETCC:
8760 case ISD::STRICT_FSETCCS: {
8761 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
8762 ISD::CondCode Condition = getFCmpCondCode(Pred: FPCmp->getPredicate());
8763 if (DAG.isKnownNeverNaN(Op: Opers[1]) && DAG.isKnownNeverNaN(Op: Opers[2]))
8764 Condition = getFCmpCodeWithoutNaN(CC: Condition);
8765 Opers.push_back(Elt: DAG.getCondCode(Cond: Condition));
8766 break;
8767 }
8768 }
8769
8770 SDValue Result = DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8771 pushFPOpOutChain(Result, EB);
8772
8773 SDValue FPResult = Result.getValue(R: 0);
8774 setValue(V: &FPI, NewN: FPResult);
8775}
8776
8777static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8778 std::optional<unsigned> ResOPC;
8779 switch (VPIntrin.getIntrinsicID()) {
8780 case Intrinsic::vp_ctlz: {
8781 bool IsZeroUndef = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8782 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_POISON : ISD::VP_CTLZ;
8783 break;
8784 }
8785 case Intrinsic::vp_cttz: {
8786 bool IsZeroUndef = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8787 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_POISON : ISD::VP_CTTZ;
8788 break;
8789 }
8790 case Intrinsic::vp_cttz_elts: {
8791 bool IsZeroPoison = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8792 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_POISON : ISD::VP_CTTZ_ELTS;
8793 break;
8794 }
8795#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8796 case Intrinsic::VPID: \
8797 ResOPC = ISD::VPSD; \
8798 break;
8799#include "llvm/IR/VPIntrinsics.def"
8800 }
8801
8802 if (!ResOPC)
8803 llvm_unreachable(
8804 "Inconsistency: no SDNode available for this VPIntrinsic!");
8805
8806 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8807 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8808 if (VPIntrin.getFastMathFlags().allowReassoc())
8809 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8810 : ISD::VP_REDUCE_FMUL;
8811 }
8812
8813 return *ResOPC;
8814}
8815
8816void SelectionDAGBuilder::visitVPLoad(
8817 const VPIntrinsic &VPIntrin, EVT VT,
8818 const SmallVectorImpl<SDValue> &OpValues) {
8819 SDLoc DL = getCurSDLoc();
8820 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8821 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8822 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8823 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8824 SDValue LD;
8825 // Do not serialize variable-length loads of constant memory with
8826 // anything.
8827 if (!Alignment)
8828 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8829 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8830 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8831 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8832 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8833 MachineMemOperand::Flags MMOFlags =
8834 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8835 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8836 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
8837 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo, Ranges);
8838 LD = DAG.getLoadVP(VT, dl: DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
8839 MMO, IsExpanding: false /*IsExpanding */);
8840 if (AddToChain)
8841 PendingLoads.push_back(Elt: LD.getValue(R: 1));
8842 setValue(V: &VPIntrin, NewN: LD);
8843}
8844
8845void SelectionDAGBuilder::visitVPLoadFF(
8846 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,
8847 const SmallVectorImpl<SDValue> &OpValues) {
8848 assert(OpValues.size() == 3 && "Unexpected number of operands");
8849 SDLoc DL = getCurSDLoc();
8850 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8851 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8852 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8853 const MDNode *Ranges = VPIntrin.getMetadata(KindID: LLVMContext::MD_range);
8854 SDValue LD;
8855 // Do not serialize variable-length loads of constant memory with
8856 // anything.
8857 if (!Alignment)
8858 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8859 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8860 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8861 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8862 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8863 PtrInfo: MachinePointerInfo(PtrOperand), F: MachineMemOperand::MOLoad,
8864 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo, Ranges);
8865 LD = DAG.getLoadFFVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
8866 MMO);
8867 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EVLVT, Operand: LD.getValue(R: 1));
8868 if (AddToChain)
8869 PendingLoads.push_back(Elt: LD.getValue(R: 2));
8870 setValue(V: &VPIntrin, NewN: DAG.getMergeValues(Ops: {LD.getValue(R: 0), Trunc}, dl: DL));
8871}
8872
8873void SelectionDAGBuilder::visitVPGather(
8874 const VPIntrinsic &VPIntrin, EVT VT,
8875 const SmallVectorImpl<SDValue> &OpValues) {
8876 SDLoc DL = getCurSDLoc();
8877 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8878 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8879 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8880 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8881 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8882 SDValue LD;
8883 if (!Alignment)
8884 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8885 unsigned AS =
8886 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8887 MachineMemOperand::Flags MMOFlags =
8888 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8889 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8890 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
8891 BaseAlignment: *Alignment, AAInfo, Ranges);
8892 SDValue Base, Index, Scale;
8893 bool UniformBase =
8894 getUniformBase(Ptr: PtrOperand, Base, Index, Scale, SDB: this, CurBB: VPIntrin.getParent(),
8895 ElemSize: VT.getScalarStoreSize());
8896 if (!UniformBase) {
8897 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8898 Index = getValue(V: PtrOperand);
8899 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8900 }
8901 EVT IdxVT = Index.getValueType();
8902 EVT EltTy = IdxVT.getVectorElementType();
8903 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
8904 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
8905 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
8906 }
8907 LD = DAG.getGatherVP(
8908 VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), VT, dl: DL,
8909 Ops: {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8910 IndexType: ISD::SIGNED_SCALED);
8911 PendingLoads.push_back(Elt: LD.getValue(R: 1));
8912 setValue(V: &VPIntrin, NewN: LD);
8913}
8914
8915void SelectionDAGBuilder::visitVPStore(
8916 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8917 SDLoc DL = getCurSDLoc();
8918 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
8919 EVT VT = OpValues[0].getValueType();
8920 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8921 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8922 SDValue ST;
8923 if (!Alignment)
8924 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8925 SDValue Ptr = OpValues[1];
8926 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
8927 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8928 MachineMemOperand::Flags MMOFlags =
8929 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8930 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8931 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
8932 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo);
8933 ST = DAG.getStoreVP(Chain: getMemoryRoot(), dl: DL, Val: OpValues[0], Ptr, Offset,
8934 Mask: OpValues[2], EVL: OpValues[3], MemVT: VT, MMO, AM: ISD::UNINDEXED,
8935 /* IsTruncating */ false, /*IsCompressing*/ false);
8936 DAG.setRoot(ST);
8937 setValue(V: &VPIntrin, NewN: ST);
8938}
8939
8940void SelectionDAGBuilder::visitVPScatter(
8941 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8942 SDLoc DL = getCurSDLoc();
8943 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8944 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
8945 EVT VT = OpValues[0].getValueType();
8946 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8947 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8948 SDValue ST;
8949 if (!Alignment)
8950 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8951 unsigned AS =
8952 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8953 MachineMemOperand::Flags MMOFlags =
8954 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8955 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8956 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
8957 BaseAlignment: *Alignment, AAInfo);
8958 SDValue Base, Index, Scale;
8959 bool UniformBase =
8960 getUniformBase(Ptr: PtrOperand, Base, Index, Scale, SDB: this, CurBB: VPIntrin.getParent(),
8961 ElemSize: VT.getScalarStoreSize());
8962 if (!UniformBase) {
8963 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8964 Index = getValue(V: PtrOperand);
8965 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8966 }
8967 EVT IdxVT = Index.getValueType();
8968 EVT EltTy = IdxVT.getVectorElementType();
8969 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
8970 EVT NewIdxVT = IdxVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: EltTy);
8971 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
8972 }
8973 ST = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT, dl: DL,
8974 Ops: {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8975 OpValues[2], OpValues[3]},
8976 MMO, IndexType: ISD::SIGNED_SCALED);
8977 DAG.setRoot(ST);
8978 setValue(V: &VPIntrin, NewN: ST);
8979}
8980
8981void SelectionDAGBuilder::visitVPStridedLoad(
8982 const VPIntrinsic &VPIntrin, EVT VT,
8983 const SmallVectorImpl<SDValue> &OpValues) {
8984 SDLoc DL = getCurSDLoc();
8985 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8986 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8987 if (!Alignment)
8988 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8989 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8990 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8991 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8992 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8993 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8994 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8995 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8996 MachineMemOperand::Flags MMOFlags =
8997 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8998 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8999 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9000 BaseAlignment: *Alignment, AAInfo, Ranges);
9001
9002 SDValue LD = DAG.getStridedLoadVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Stride: OpValues[1],
9003 Mask: OpValues[2], EVL: OpValues[3], MMO,
9004 IsExpanding: false /*IsExpanding*/);
9005
9006 if (AddToChain)
9007 PendingLoads.push_back(Elt: LD.getValue(R: 1));
9008 setValue(V: &VPIntrin, NewN: LD);
9009}
9010
9011void SelectionDAGBuilder::visitVPStridedStore(
9012 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9013 SDLoc DL = getCurSDLoc();
9014 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
9015 EVT VT = OpValues[0].getValueType();
9016 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9017 if (!Alignment)
9018 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
9019 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9020 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9021 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9022 MachineMemOperand::Flags MMOFlags =
9023 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
9024 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9025 PtrInfo: MachinePointerInfo(AS), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
9026 BaseAlignment: *Alignment, AAInfo);
9027
9028 SDValue ST = DAG.getStridedStoreVP(
9029 Chain: getMemoryRoot(), DL, Val: OpValues[0], Ptr: OpValues[1],
9030 Offset: DAG.getUNDEF(VT: OpValues[1].getValueType()), Stride: OpValues[2], Mask: OpValues[3],
9031 EVL: OpValues[4], MemVT: VT, MMO, AM: ISD::UNINDEXED, /*IsTruncating*/ false,
9032 /*IsCompressing*/ false);
9033
9034 DAG.setRoot(ST);
9035 setValue(V: &VPIntrin, NewN: ST);
9036}
9037
9038void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
9039 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9040 SDLoc DL = getCurSDLoc();
9041
9042 ISD::CondCode Condition;
9043 CmpInst::Predicate CondCode = VPIntrin.getPredicate();
9044
9045 Value *Op1 = VPIntrin.getOperand(i_nocapture: 0);
9046 Value *Op2 = VPIntrin.getOperand(i_nocapture: 1);
9047 // #2 is the condition code
9048 SDValue MaskOp = getValue(V: VPIntrin.getOperand(i_nocapture: 3));
9049 SDValue EVL = getValue(V: VPIntrin.getOperand(i_nocapture: 4));
9050 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
9051 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
9052 "Unexpected target EVL type");
9053 EVL = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EVLParamVT, Operand: EVL);
9054
9055 if (VPIntrin.getOperand(i_nocapture: 0)->getType()->isFPOrFPVectorTy()) {
9056 Condition = getFCmpCondCode(Pred: CondCode);
9057 SimplifyQuery SQ(DAG.getDataLayout(), &VPIntrin);
9058 if (isKnownNeverNaN(V: Op2, SQ) && isKnownNeverNaN(V: Op1, SQ))
9059 Condition = getFCmpCodeWithoutNaN(CC: Condition);
9060 } else {
9061 Condition = getICmpCondCode(Pred: CondCode);
9062 }
9063
9064 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9065 Ty: VPIntrin.getType());
9066 setValue(V: &VPIntrin, NewN: DAG.getSetCCVP(DL, VT: DestVT, LHS: getValue(V: Op1), RHS: getValue(V: Op2),
9067 Cond: Condition, Mask: MaskOp, EVL));
9068}
9069
9070void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
9071 const VPIntrinsic &VPIntrin) {
9072 SDLoc DL = getCurSDLoc();
9073 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
9074
9075 auto IID = VPIntrin.getIntrinsicID();
9076
9077 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(Val: &VPIntrin))
9078 return visitVPCmp(VPIntrin: *CmpI);
9079
9080 SmallVector<EVT, 4> ValueVTs;
9081 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9082 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: VPIntrin.getType(), ValueVTs);
9083 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
9084
9085 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IntrinsicID: IID);
9086
9087 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
9088 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
9089 "Unexpected target EVL type");
9090
9091 // Request operands.
9092 SmallVector<SDValue, 7> OpValues;
9093 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
9094 auto Op = getValue(V: VPIntrin.getArgOperand(i: I));
9095 if (I == EVLParamPos)
9096 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EVLParamVT, Operand: Op);
9097 OpValues.push_back(Elt: Op);
9098 }
9099
9100 switch (Opcode) {
9101 default: {
9102 SDNodeFlags SDFlags;
9103 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &VPIntrin))
9104 SDFlags.copyFMF(FPMO: *FPMO);
9105 SDValue Result = DAG.getNode(Opcode, DL, VTList: VTs, Ops: OpValues, Flags: SDFlags);
9106 setValue(V: &VPIntrin, NewN: Result);
9107 break;
9108 }
9109 case ISD::VP_LOAD:
9110 visitVPLoad(VPIntrin, VT: ValueVTs[0], OpValues);
9111 break;
9112 case ISD::VP_LOAD_FF:
9113 visitVPLoadFF(VPIntrin, VT: ValueVTs[0], EVLVT: ValueVTs[1], OpValues);
9114 break;
9115 case ISD::VP_GATHER:
9116 visitVPGather(VPIntrin, VT: ValueVTs[0], OpValues);
9117 break;
9118 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9119 visitVPStridedLoad(VPIntrin, VT: ValueVTs[0], OpValues);
9120 break;
9121 case ISD::VP_STORE:
9122 visitVPStore(VPIntrin, OpValues);
9123 break;
9124 case ISD::VP_SCATTER:
9125 visitVPScatter(VPIntrin, OpValues);
9126 break;
9127 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9128 visitVPStridedStore(VPIntrin, OpValues);
9129 break;
9130 case ISD::VP_FMULADD: {
9131 assert(OpValues.size() == 5 && "Unexpected number of operands");
9132 SDNodeFlags SDFlags;
9133 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &VPIntrin))
9134 SDFlags.copyFMF(FPMO: *FPMO);
9135 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
9136 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), ValueVTs[0])) {
9137 setValue(V: &VPIntrin, NewN: DAG.getNode(Opcode: ISD::VP_FMA, DL, VTList: VTs, Ops: OpValues, Flags: SDFlags));
9138 } else {
9139 SDValue Mul = DAG.getNode(
9140 Opcode: ISD::VP_FMUL, DL, VTList: VTs,
9141 Ops: {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, Flags: SDFlags);
9142 SDValue Add =
9143 DAG.getNode(Opcode: ISD::VP_FADD, DL, VTList: VTs,
9144 Ops: {Mul, OpValues[2], OpValues[3], OpValues[4]}, Flags: SDFlags);
9145 setValue(V: &VPIntrin, NewN: Add);
9146 }
9147 break;
9148 }
9149 case ISD::VP_IS_FPCLASS: {
9150 const DataLayout DLayout = DAG.getDataLayout();
9151 EVT DestVT = TLI.getValueType(DL: DLayout, Ty: VPIntrin.getType());
9152 auto Constant = OpValues[1]->getAsZExtVal();
9153 SDValue Check = DAG.getTargetConstant(Val: Constant, DL, VT: MVT::i32);
9154 SDValue V = DAG.getNode(Opcode: ISD::VP_IS_FPCLASS, DL, VT: DestVT,
9155 Ops: {OpValues[0], Check, OpValues[2], OpValues[3]});
9156 setValue(V: &VPIntrin, NewN: V);
9157 return;
9158 }
9159 case ISD::VP_INTTOPTR: {
9160 SDValue N = OpValues[0];
9161 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: VPIntrin.getType());
9162 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: VPIntrin.getType());
9163 N = DAG.getVPPtrExtOrTrunc(DL: getCurSDLoc(), VT: DestVT, Op: N, Mask: OpValues[1],
9164 EVL: OpValues[2]);
9165 N = DAG.getVPZExtOrTrunc(DL: getCurSDLoc(), VT: PtrMemVT, Op: N, Mask: OpValues[1],
9166 EVL: OpValues[2]);
9167 setValue(V: &VPIntrin, NewN: N);
9168 break;
9169 }
9170 case ISD::VP_PTRTOINT: {
9171 SDValue N = OpValues[0];
9172 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9173 Ty: VPIntrin.getType());
9174 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(),
9175 Ty: VPIntrin.getOperand(i_nocapture: 0)->getType());
9176 N = DAG.getVPPtrExtOrTrunc(DL: getCurSDLoc(), VT: PtrMemVT, Op: N, Mask: OpValues[1],
9177 EVL: OpValues[2]);
9178 N = DAG.getVPZExtOrTrunc(DL: getCurSDLoc(), VT: DestVT, Op: N, Mask: OpValues[1],
9179 EVL: OpValues[2]);
9180 setValue(V: &VPIntrin, NewN: N);
9181 break;
9182 }
9183 case ISD::VP_ABS:
9184 case ISD::VP_CTLZ:
9185 case ISD::VP_CTLZ_ZERO_POISON:
9186 case ISD::VP_CTTZ:
9187 case ISD::VP_CTTZ_ZERO_POISON:
9188 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
9189 case ISD::VP_CTTZ_ELTS: {
9190 SDValue Result =
9191 DAG.getNode(Opcode, DL, VTList: VTs, Ops: {OpValues[0], OpValues[2], OpValues[3]});
9192 setValue(V: &VPIntrin, NewN: Result);
9193 break;
9194 }
9195 }
9196}
9197
9198SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
9199 const BasicBlock *EHPadBB,
9200 MCSymbol *&BeginLabel) {
9201 MachineFunction &MF = DAG.getMachineFunction();
9202
9203 // Insert a label before the invoke call to mark the try range. This can be
9204 // used to detect deletion of the invoke via the MachineModuleInfo.
9205 BeginLabel = MF.getContext().createTempSymbol();
9206
9207 // For SjLj, keep track of which landing pads go with which invokes
9208 // so as to maintain the ordering of pads in the LSDA.
9209 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
9210 if (CallSiteIndex) {
9211 MF.setCallSiteBeginLabel(BeginLabel, Site: CallSiteIndex);
9212 LPadToCallSiteMap[FuncInfo.getMBB(BB: EHPadBB)].push_back(Elt: CallSiteIndex);
9213
9214 // Now that the call site is handled, stop tracking it.
9215 FuncInfo.setCurrentCallSite(0);
9216 }
9217
9218 return DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: BeginLabel);
9219}
9220
9221SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
9222 const BasicBlock *EHPadBB,
9223 MCSymbol *BeginLabel) {
9224 assert(BeginLabel && "BeginLabel should've been set");
9225
9226 MachineFunction &MF = DAG.getMachineFunction();
9227
9228 // Insert a label at the end of the invoke call to mark the try range. This
9229 // can be used to detect deletion of the invoke via the MachineModuleInfo.
9230 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
9231 Chain = DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: EndLabel);
9232
9233 // Inform MachineModuleInfo of range.
9234 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
9235 // There is a platform (e.g. wasm) that uses funclet style IR but does not
9236 // actually use outlined funclets and their LSDA info style.
9237 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
9238 assert(II && "II should've been set");
9239 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
9240 EHInfo->addIPToStateRange(II, InvokeBegin: BeginLabel, InvokeEnd: EndLabel);
9241 } else if (!isScopedEHPersonality(Pers)) {
9242 assert(EHPadBB);
9243 MF.addInvoke(LandingPad: FuncInfo.getMBB(BB: EHPadBB), BeginLabel, EndLabel);
9244 }
9245
9246 return Chain;
9247}
9248
9249std::pair<SDValue, SDValue>
9250SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
9251 const BasicBlock *EHPadBB) {
9252 MCSymbol *BeginLabel = nullptr;
9253
9254 if (EHPadBB) {
9255 // Both PendingLoads and PendingExports must be flushed here;
9256 // this call might not return.
9257 (void)getRoot();
9258 DAG.setRoot(lowerStartEH(Chain: getControlRoot(), EHPadBB, BeginLabel));
9259 CLI.setChain(getRoot());
9260 }
9261
9262 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9263 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
9264
9265 assert((CLI.IsTailCall || Result.second.getNode()) &&
9266 "Non-null chain expected with non-tail call!");
9267 assert((Result.second.getNode() || !Result.first.getNode()) &&
9268 "Null value expected with tail call!");
9269
9270 if (!Result.second.getNode()) {
9271 // As a special case, a null chain means that a tail call has been emitted
9272 // and the DAG root is already updated.
9273 HasTailCall = true;
9274
9275 // Since there's no actual continuation from this block, nothing can be
9276 // relying on us setting vregs for them.
9277 PendingExports.clear();
9278 } else {
9279 DAG.setRoot(Result.second);
9280 }
9281
9282 if (EHPadBB) {
9283 DAG.setRoot(lowerEndEH(Chain: getRoot(), II: cast_or_null<InvokeInst>(Val: CLI.CB), EHPadBB,
9284 BeginLabel));
9285 Result.second = getRoot();
9286 }
9287
9288 return Result;
9289}
9290
9291bool SelectionDAGBuilder::canTailCall(const CallBase &CB) const {
9292 bool isMustTailCall = CB.isMustTailCall();
9293
9294 // Avoid emitting tail calls in functions with the disable-tail-calls
9295 // attribute.
9296 const Function *Caller = CB.getParent()->getParent();
9297 if (!isMustTailCall &&
9298 Caller->getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
9299 return false;
9300
9301 // We can't tail call inside a function with a swifterror argument. Lowering
9302 // does not support this yet. It would have to move into the swifterror
9303 // register before the call.
9304 if (DAG.getTargetLoweringInfo().supportSwiftError() &&
9305 Caller->getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError))
9306 return false;
9307
9308 // Check if target-independent constraints permit a tail call here.
9309 // Target-dependent constraints are checked within TLI->LowerCallTo.
9310 return isInTailCallPosition(Call: CB, TM: DAG.getTarget());
9311}
9312
9313void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
9314 bool isTailCall, bool isMustTailCall,
9315 const BasicBlock *EHPadBB,
9316 const TargetLowering::PtrAuthInfo *PAI) {
9317 auto &DL = DAG.getDataLayout();
9318 FunctionType *FTy = CB.getFunctionType();
9319 Type *RetTy = CB.getType();
9320
9321 TargetLowering::ArgListTy Args;
9322 Args.reserve(n: CB.arg_size());
9323
9324 const Value *SwiftErrorVal = nullptr;
9325 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9326
9327 if (isTailCall)
9328 isTailCall = canTailCall(CB);
9329
9330 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
9331 const Value *V = *I;
9332
9333 // Skip empty types
9334 if (V->getType()->isEmptyTy())
9335 continue;
9336
9337 SDValue ArgNode = getValue(V);
9338 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
9339 Entry.setAttributes(Call: &CB, ArgIdx: I - CB.arg_begin());
9340
9341 // Use swifterror virtual register as input to the call.
9342 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
9343 SwiftErrorVal = V;
9344 // We find the virtual register for the actual swifterror argument.
9345 // Instead of using the Value, we use the virtual register instead.
9346 Entry.Node =
9347 DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
9348 VT: EVT(TLI.getPointerTy(DL)));
9349 }
9350
9351 Args.push_back(x: Entry);
9352
9353 // If we have an explicit sret argument that is an Instruction, (i.e., it
9354 // might point to function-local memory), we can't meaningfully tail-call.
9355 if (Entry.IsSRet && isa<Instruction>(Val: V))
9356 isTailCall = false;
9357 }
9358
9359 // If call site has a cfguardtarget operand bundle, create and add an
9360 // additional ArgListEntry.
9361 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_cfguardtarget)) {
9362 Value *V = Bundle->Inputs[0];
9363 TargetLowering::ArgListEntry Entry(V, getValue(V));
9364 Entry.IsCFGuardTarget = true;
9365 Args.push_back(x: Entry);
9366 }
9367
9368 // Disable tail calls if there is an swifterror argument. Targets have not
9369 // been updated to support tail calls.
9370 if (TLI.supportSwiftError() && SwiftErrorVal)
9371 isTailCall = false;
9372
9373 ConstantInt *CFIType = nullptr;
9374 if (CB.isIndirectCall()) {
9375 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi)) {
9376 if (!TLI.supportKCFIBundles())
9377 report_fatal_error(
9378 reason: "Target doesn't support calls with kcfi operand bundles.");
9379 CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
9380 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
9381 }
9382 }
9383
9384 SDValue ConvControlToken;
9385 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
9386 auto *Token = Bundle->Inputs[0].get();
9387 ConvControlToken = getValue(V: Token);
9388 }
9389
9390 GlobalValue *DeactivationSymbol = nullptr;
9391 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
9392 DeactivationSymbol = cast<GlobalValue>(Val: Bundle->Inputs[0].get());
9393 }
9394
9395 TargetLowering::CallLoweringInfo CLI(DAG);
9396 CLI.setDebugLoc(getCurSDLoc())
9397 .setChain(getRoot())
9398 .setCallee(ResultType: RetTy, FTy, Target: Callee, ArgsList: std::move(Args), Call: CB)
9399 .setTailCall(isTailCall)
9400 .setConvergent(CB.isConvergent())
9401 .setIsPreallocated(
9402 CB.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
9403 .setCFIType(CFIType)
9404 .setConvergenceControlToken(ConvControlToken)
9405 .setDeactivationSymbol(DeactivationSymbol);
9406
9407 // Set the pointer authentication info if we have it.
9408 if (PAI) {
9409 if (!TLI.supportPtrAuthBundles())
9410 report_fatal_error(
9411 reason: "This target doesn't support calls with ptrauth operand bundles.");
9412 CLI.setPtrAuth(*PAI);
9413 }
9414
9415 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9416
9417 if (Result.first.getNode()) {
9418 Result.first = lowerRangeToAssertZExt(DAG, I: CB, Op: Result.first);
9419 Result.first = lowerNoFPClassToAssertNoFPClass(DAG, I: CB, Op: Result.first);
9420 setValue(V: &CB, NewN: Result.first);
9421 }
9422
9423 // The last element of CLI.InVals has the SDValue for swifterror return.
9424 // Here we copy it to a virtual register and update SwiftErrorMap for
9425 // book-keeping.
9426 if (SwiftErrorVal && TLI.supportSwiftError()) {
9427 // Get the last element of InVals.
9428 SDValue Src = CLI.InVals.back();
9429 Register VReg =
9430 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
9431 SDValue CopyNode = CLI.DAG.getCopyToReg(Chain: Result.second, dl: CLI.DL, Reg: VReg, N: Src);
9432 DAG.setRoot(CopyNode);
9433 }
9434}
9435
9436static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
9437 SelectionDAGBuilder &Builder) {
9438 // Check to see if this load can be trivially constant folded, e.g. if the
9439 // input is from a string literal.
9440 if (const Constant *LoadInput = dyn_cast<Constant>(Val: PtrVal)) {
9441 // Cast pointer to the type we really want to load.
9442 Type *LoadTy =
9443 Type::getIntNTy(C&: PtrVal->getContext(), N: LoadVT.getScalarSizeInBits());
9444 if (LoadVT.isVector())
9445 LoadTy = FixedVectorType::get(ElementType: LoadTy, NumElts: LoadVT.getVectorNumElements());
9446 if (const Constant *LoadCst =
9447 ConstantFoldLoadFromConstPtr(C: const_cast<Constant *>(LoadInput),
9448 Ty: LoadTy, DL: Builder.DAG.getDataLayout()))
9449 return Builder.getValue(V: LoadCst);
9450 }
9451
9452 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9453 // still constant memory, the input chain can be the entry node.
9454 SDValue Root;
9455 bool ConstantMemory = false;
9456
9457 // Do not serialize (non-volatile) loads of constant memory with anything.
9458 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(P: PtrVal)) {
9459 Root = Builder.DAG.getEntryNode();
9460 ConstantMemory = true;
9461 } else {
9462 // Do not serialize non-volatile loads against each other.
9463 Root = Builder.DAG.getRoot();
9464 }
9465
9466 SDValue Ptr = Builder.getValue(V: PtrVal);
9467 SDValue LoadVal =
9468 Builder.DAG.getLoad(VT: LoadVT, dl: Builder.getCurSDLoc(), Chain: Root, Ptr,
9469 PtrInfo: MachinePointerInfo(PtrVal), Alignment: Align(1));
9470
9471 if (!ConstantMemory)
9472 Builder.PendingLoads.push_back(Elt: LoadVal.getValue(R: 1));
9473 return LoadVal;
9474}
9475
9476/// Record the value for an instruction that produces an integer result,
9477/// converting the type where necessary.
9478void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9479 SDValue Value,
9480 bool IsSigned) {
9481 EVT VT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9482 Ty: I.getType(), AllowUnknown: true);
9483 Value = DAG.getExtOrTrunc(IsSigned, Op: Value, DL: getCurSDLoc(), VT);
9484 setValue(V: &I, NewN: Value);
9485}
9486
9487/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9488/// true and lower it. Otherwise return false, and it will be lowered like a
9489/// normal call.
9490/// The caller already checked that \p I calls the appropriate LibFunc with a
9491/// correct prototype.
9492bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9493 const Value *LHS = I.getArgOperand(i: 0), *RHS = I.getArgOperand(i: 1);
9494 const Value *Size = I.getArgOperand(i: 2);
9495 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(Val: getValue(V: Size));
9496 if (CSize && CSize->getZExtValue() == 0) {
9497 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9498 Ty: I.getType(), AllowUnknown: true);
9499 setValue(V: &I, NewN: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: CallVT));
9500 return true;
9501 }
9502
9503 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9504 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9505 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: LHS), Op2: getValue(V: RHS),
9506 Op3: getValue(V: Size), CI: &I);
9507 if (Res.first.getNode()) {
9508 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9509 PendingLoads.push_back(Elt: Res.second);
9510 return true;
9511 }
9512
9513 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9514 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9515 if (!CSize || !isOnlyUsedInZeroEqualityComparison(CxtI: &I))
9516 return false;
9517
9518 // If the target has a fast compare for the given size, it will return a
9519 // preferred load type for that size. Require that the load VT is legal and
9520 // that the target supports unaligned loads of that type. Otherwise, return
9521 // INVALID.
9522 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9523 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9524 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9525 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9526 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9527 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9528 // TODO: Check alignment of src and dest ptrs.
9529 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9530 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9531 if (!TLI.isTypeLegal(VT: LVT) ||
9532 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: SrcAS) ||
9533 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: DstAS))
9534 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
9535 }
9536
9537 return LVT;
9538 };
9539
9540 // This turns into unaligned loads. We only do this if the target natively
9541 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9542 // we'll only produce a small number of byte loads.
9543 MVT LoadVT;
9544 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9545 switch (NumBitsToCompare) {
9546 default:
9547 return false;
9548 case 16:
9549 LoadVT = MVT::i16;
9550 break;
9551 case 32:
9552 LoadVT = MVT::i32;
9553 break;
9554 case 64:
9555 case 128:
9556 case 256:
9557 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9558 break;
9559 }
9560
9561 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9562 return false;
9563
9564 SDValue LoadL = getMemCmpLoad(PtrVal: LHS, LoadVT, Builder&: *this);
9565 SDValue LoadR = getMemCmpLoad(PtrVal: RHS, LoadVT, Builder&: *this);
9566
9567 // Bitcast to a wide integer type if the loads are vectors.
9568 if (LoadVT.isVector()) {
9569 EVT CmpVT = EVT::getIntegerVT(Context&: LHS->getContext(), BitWidth: LoadVT.getSizeInBits());
9570 LoadL = DAG.getBitcast(VT: CmpVT, V: LoadL);
9571 LoadR = DAG.getBitcast(VT: CmpVT, V: LoadR);
9572 }
9573
9574 SDValue Cmp = DAG.getSetCC(DL: getCurSDLoc(), VT: MVT::i1, LHS: LoadL, RHS: LoadR, Cond: ISD::SETNE);
9575 processIntegerCallValue(I, Value: Cmp, IsSigned: false);
9576 return true;
9577}
9578
9579/// See if we can lower a memchr call into an optimized form. If so, return
9580/// true and lower it. Otherwise return false, and it will be lowered like a
9581/// normal call.
9582/// The caller already checked that \p I calls the appropriate LibFunc with a
9583/// correct prototype.
9584bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9585 const Value *Src = I.getArgOperand(i: 0);
9586 const Value *Char = I.getArgOperand(i: 1);
9587 const Value *Length = I.getArgOperand(i: 2);
9588
9589 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9590 std::pair<SDValue, SDValue> Res =
9591 TSI.EmitTargetCodeForMemchr(DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(),
9592 Src: getValue(V: Src), Char: getValue(V: Char), Length: getValue(V: Length),
9593 SrcPtrInfo: MachinePointerInfo(Src));
9594 if (Res.first.getNode()) {
9595 setValue(V: &I, NewN: Res.first);
9596 PendingLoads.push_back(Elt: Res.second);
9597 return true;
9598 }
9599
9600 return false;
9601}
9602
9603/// See if we can lower a memccpy call into an optimized form. If so, return
9604/// true and lower it, otherwise return false and it will be lowered like a
9605/// normal call.
9606/// The caller already checked that \p I calls the appropriate LibFunc with a
9607/// correct prototype.
9608bool SelectionDAGBuilder::visitMemCCpyCall(const CallInst &I) {
9609 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9610 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemccpy(
9611 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Dst: getValue(V: I.getArgOperand(i: 0)),
9612 Src: getValue(V: I.getArgOperand(i: 1)), C: getValue(V: I.getArgOperand(i: 2)),
9613 Size: getValue(V: I.getArgOperand(i: 3)), CI: &I);
9614
9615 if (Res.first) {
9616 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9617 PendingLoads.push_back(Elt: Res.second);
9618 return true;
9619 }
9620 return false;
9621}
9622
9623/// See if we can lower a mempcpy call into an optimized form. If so, return
9624/// true and lower it. Otherwise return false, and it will be lowered like a
9625/// normal call.
9626/// The caller already checked that \p I calls the appropriate LibFunc with a
9627/// correct prototype.
9628bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9629 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
9630 SDValue Src = getValue(V: I.getArgOperand(i: 1));
9631 SDValue Size = getValue(V: I.getArgOperand(i: 2));
9632
9633 Align DstAlign = DAG.InferPtrAlign(Ptr: Dst).valueOrOne();
9634 Align SrcAlign = DAG.InferPtrAlign(Ptr: Src).valueOrOne();
9635
9636 SDLoc sdl = getCurSDLoc();
9637
9638 // In the mempcpy context we need to pass in a false value for isTailCall
9639 // because the return pointer needs to be adjusted by the size of
9640 // the copied memory.
9641 SDValue Root = getMemoryRoot();
9642 SDValue MC = DAG.getMemcpy(
9643 Chain: Root, dl: sdl, Dst, Src, Size, DstAlign, SrcAlign, isVol: false, AlwaysInline: false,
9644 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
9645 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)), AAInfo: I.getAAMetadata());
9646 assert(MC.getNode() != nullptr &&
9647 "** memcpy should not be lowered as TailCall in mempcpy context **");
9648 DAG.setRoot(MC);
9649
9650 // Check if Size needs to be truncated or extended.
9651 Size = DAG.getSExtOrTrunc(Op: Size, DL: sdl, VT: Dst.getValueType());
9652
9653 // Adjust return pointer to point just past the last dst byte.
9654 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Base: Dst, Offset: Size, DL: sdl);
9655 setValue(V: &I, NewN: DstPlusSize);
9656 return true;
9657}
9658
9659/// See if we can lower a strcpy call into an optimized form. If so, return
9660/// true and lower it, otherwise return false and it will be lowered like a
9661/// normal call.
9662/// The caller already checked that \p I calls the appropriate LibFunc with a
9663/// correct prototype.
9664bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9665 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9666
9667 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9668 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcpy(
9669 DAG, DL: getCurSDLoc(), Chain: getRoot(), Dest: getValue(V: Arg0), Src: getValue(V: Arg1),
9670 DestPtrInfo: MachinePointerInfo(Arg0), SrcPtrInfo: MachinePointerInfo(Arg1), isStpcpy, CI: &I);
9671 if (Res.first.getNode()) {
9672 setValue(V: &I, NewN: Res.first);
9673 DAG.setRoot(Res.second);
9674 return true;
9675 }
9676
9677 return false;
9678}
9679
9680/// See if we can lower a strcmp call into an optimized form. If so, return
9681/// true and lower it, otherwise return false and it will be lowered like a
9682/// normal call.
9683/// The caller already checked that \p I calls the appropriate LibFunc with a
9684/// correct prototype.
9685bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9686 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9687
9688 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9689 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcmp(
9690 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: Arg0), Op2: getValue(V: Arg1),
9691 Op1PtrInfo: MachinePointerInfo(Arg0), Op2PtrInfo: MachinePointerInfo(Arg1), CI: &I);
9692 if (Res.first.getNode()) {
9693 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9694 PendingLoads.push_back(Elt: Res.second);
9695 return true;
9696 }
9697
9698 return false;
9699}
9700
9701/// See if we can lower a strlen call into an optimized form. If so, return
9702/// true and lower it, otherwise return false and it will be lowered like a
9703/// normal call.
9704/// The caller already checked that \p I calls the appropriate LibFunc with a
9705/// correct prototype.
9706bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9707 const Value *Arg0 = I.getArgOperand(i: 0);
9708
9709 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9710 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(
9711 DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(), Src: getValue(V: Arg0), CI: &I);
9712 if (Res.first.getNode()) {
9713 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9714 PendingLoads.push_back(Elt: Res.second);
9715 return true;
9716 }
9717
9718 return false;
9719}
9720
9721/// See if we can lower a strnlen call into an optimized form. If so, return
9722/// true and lower it, otherwise return false and it will be lowered like a
9723/// normal call.
9724/// The caller already checked that \p I calls the appropriate LibFunc with a
9725/// correct prototype.
9726bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9727 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9728
9729 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9730 std::pair<SDValue, SDValue> Res =
9731 TSI.EmitTargetCodeForStrnlen(DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(),
9732 Src: getValue(V: Arg0), MaxLength: getValue(V: Arg1),
9733 SrcPtrInfo: MachinePointerInfo(Arg0));
9734 if (Res.first.getNode()) {
9735 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9736 PendingLoads.push_back(Elt: Res.second);
9737 return true;
9738 }
9739
9740 return false;
9741}
9742
9743/// See if we can lower a Strstr call into an optimized form. If so, return
9744/// true and lower it, otherwise return false and it will be lowered like a
9745/// normal call.
9746/// The caller already checked that \p I calls the appropriate LibFunc with a
9747/// correct prototype.
9748bool SelectionDAGBuilder::visitStrstrCall(const CallInst &I) {
9749 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9750 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9751 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrstr(
9752 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: Arg0), Op2: getValue(V: Arg1), CI: &I);
9753 if (Res.first) {
9754 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9755 PendingLoads.push_back(Elt: Res.second);
9756 return true;
9757 }
9758 return false;
9759}
9760
9761/// See if we can lower a unary floating-point operation into an SDNode with
9762/// the specified Opcode. If so, return true and lower it, otherwise return
9763/// false and it will be lowered like a normal call.
9764/// The caller already checked that \p I calls the appropriate LibFunc with a
9765/// correct prototype.
9766bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9767 unsigned Opcode) {
9768 // We already checked this call's prototype; verify it doesn't modify errno.
9769 // Do not perform optimizations for call sites that require strict
9770 // floating-point semantics.
9771 if (!I.onlyReadsMemory() || I.isStrictFP())
9772 return false;
9773
9774 SDNodeFlags Flags;
9775 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9776
9777 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
9778 setValue(V: &I,
9779 NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Tmp.getValueType(), Operand: Tmp, Flags));
9780 return true;
9781}
9782
9783/// See if we can lower a binary floating-point operation into an SDNode with
9784/// the specified Opcode. If so, return true and lower it. Otherwise return
9785/// false, and it will be lowered like a normal call.
9786/// The caller already checked that \p I calls the appropriate LibFunc with a
9787/// correct prototype.
9788bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9789 unsigned Opcode) {
9790 // We already checked this call's prototype; verify it doesn't modify errno.
9791 // Do not perform optimizations for call sites that require strict
9792 // floating-point semantics.
9793 if (!I.onlyReadsMemory() || I.isStrictFP())
9794 return false;
9795
9796 SDNodeFlags Flags;
9797 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9798
9799 SDValue Tmp0 = getValue(V: I.getArgOperand(i: 0));
9800 SDValue Tmp1 = getValue(V: I.getArgOperand(i: 1));
9801 EVT VT = Tmp0.getValueType();
9802 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT, N1: Tmp0, N2: Tmp1, Flags));
9803 return true;
9804}
9805
9806void SelectionDAGBuilder::visitCall(const CallInst &I) {
9807 // Handle inline assembly differently.
9808 if (I.isInlineAsm()) {
9809 visitInlineAsm(Call: I);
9810 return;
9811 }
9812
9813 diagnoseDontCall(CI: I);
9814
9815 if (Function *F = I.getCalledFunction()) {
9816 if (F->isDeclaration()) {
9817 // Is this an LLVM intrinsic?
9818 if (unsigned IID = F->getIntrinsicID()) {
9819 visitIntrinsicCall(I, Intrinsic: IID);
9820 return;
9821 }
9822 }
9823
9824 // Check for well-known libc/libm calls. If the function is internal, it
9825 // can't be a library call. Don't do the check if marked as nobuiltin for
9826 // some reason.
9827 // This code should not handle libcalls that are already canonicalized to
9828 // intrinsics by the middle-end.
9829 LibFunc Func;
9830 if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&
9831 LibInfo->getLibFunc(FDecl: *F, F&: Func) && LibInfo->hasOptimizedCodeGen(F: Func)) {
9832 switch (Func) {
9833 default: break;
9834 case LibFunc_bcmp:
9835 if (visitMemCmpBCmpCall(I))
9836 return;
9837 break;
9838 case LibFunc_copysign:
9839 case LibFunc_copysignf:
9840 case LibFunc_copysignl:
9841 // We already checked this call's prototype; verify it doesn't modify
9842 // errno.
9843 if (I.onlyReadsMemory()) {
9844 SDValue LHS = getValue(V: I.getArgOperand(i: 0));
9845 SDValue RHS = getValue(V: I.getArgOperand(i: 1));
9846 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: getCurSDLoc(),
9847 VT: LHS.getValueType(), N1: LHS, N2: RHS));
9848 return;
9849 }
9850 break;
9851 case LibFunc_sin:
9852 case LibFunc_sinf:
9853 case LibFunc_sinl:
9854 if (visitUnaryFloatCall(I, Opcode: ISD::FSIN))
9855 return;
9856 break;
9857 case LibFunc_cos:
9858 case LibFunc_cosf:
9859 case LibFunc_cosl:
9860 if (visitUnaryFloatCall(I, Opcode: ISD::FCOS))
9861 return;
9862 break;
9863 case LibFunc_tan:
9864 case LibFunc_tanf:
9865 case LibFunc_tanl:
9866 if (visitUnaryFloatCall(I, Opcode: ISD::FTAN))
9867 return;
9868 break;
9869 case LibFunc_asin:
9870 case LibFunc_asinf:
9871 case LibFunc_asinl:
9872 if (visitUnaryFloatCall(I, Opcode: ISD::FASIN))
9873 return;
9874 break;
9875 case LibFunc_acos:
9876 case LibFunc_acosf:
9877 case LibFunc_acosl:
9878 if (visitUnaryFloatCall(I, Opcode: ISD::FACOS))
9879 return;
9880 break;
9881 case LibFunc_atan:
9882 case LibFunc_atanf:
9883 case LibFunc_atanl:
9884 if (visitUnaryFloatCall(I, Opcode: ISD::FATAN))
9885 return;
9886 break;
9887 case LibFunc_atan2:
9888 case LibFunc_atan2f:
9889 case LibFunc_atan2l:
9890 if (visitBinaryFloatCall(I, Opcode: ISD::FATAN2))
9891 return;
9892 break;
9893 case LibFunc_sinh:
9894 case LibFunc_sinhf:
9895 case LibFunc_sinhl:
9896 if (visitUnaryFloatCall(I, Opcode: ISD::FSINH))
9897 return;
9898 break;
9899 case LibFunc_cosh:
9900 case LibFunc_coshf:
9901 case LibFunc_coshl:
9902 if (visitUnaryFloatCall(I, Opcode: ISD::FCOSH))
9903 return;
9904 break;
9905 case LibFunc_tanh:
9906 case LibFunc_tanhf:
9907 case LibFunc_tanhl:
9908 if (visitUnaryFloatCall(I, Opcode: ISD::FTANH))
9909 return;
9910 break;
9911 case LibFunc_sqrt:
9912 case LibFunc_sqrtf:
9913 case LibFunc_sqrtl:
9914 case LibFunc_sqrt_finite:
9915 case LibFunc_sqrtf_finite:
9916 case LibFunc_sqrtl_finite:
9917 if (visitUnaryFloatCall(I, Opcode: ISD::FSQRT))
9918 return;
9919 break;
9920 case LibFunc_log2:
9921 case LibFunc_log2f:
9922 case LibFunc_log2l:
9923 if (visitUnaryFloatCall(I, Opcode: ISD::FLOG2))
9924 return;
9925 break;
9926 case LibFunc_exp2:
9927 case LibFunc_exp2f:
9928 case LibFunc_exp2l:
9929 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP2))
9930 return;
9931 break;
9932 case LibFunc_exp10:
9933 case LibFunc_exp10f:
9934 case LibFunc_exp10l:
9935 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP10))
9936 return;
9937 break;
9938 case LibFunc_ldexp:
9939 case LibFunc_ldexpf:
9940 case LibFunc_ldexpl:
9941 if (visitBinaryFloatCall(I, Opcode: ISD::FLDEXP))
9942 return;
9943 break;
9944 case LibFunc_strstr:
9945 if (visitStrstrCall(I))
9946 return;
9947 break;
9948 case LibFunc_memcmp:
9949 if (visitMemCmpBCmpCall(I))
9950 return;
9951 break;
9952 case LibFunc_memccpy:
9953 if (visitMemCCpyCall(I))
9954 return;
9955 break;
9956 case LibFunc_mempcpy:
9957 if (visitMemPCpyCall(I))
9958 return;
9959 break;
9960 case LibFunc_memchr:
9961 if (visitMemChrCall(I))
9962 return;
9963 break;
9964 case LibFunc_strcpy:
9965 if (visitStrCpyCall(I, isStpcpy: false))
9966 return;
9967 break;
9968 case LibFunc_stpcpy:
9969 if (visitStrCpyCall(I, isStpcpy: true))
9970 return;
9971 break;
9972 case LibFunc_strcmp:
9973 if (visitStrCmpCall(I))
9974 return;
9975 break;
9976 case LibFunc_strlen:
9977 if (visitStrLenCall(I))
9978 return;
9979 break;
9980 case LibFunc_strnlen:
9981 if (visitStrNLenCall(I))
9982 return;
9983 break;
9984 }
9985 }
9986 }
9987
9988 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
9989 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), /*EHPadBB=*/nullptr);
9990 return;
9991 }
9992
9993 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9994 // have to do anything here to lower funclet bundles.
9995 // CFGuardTarget bundles are lowered in LowerCallTo.
9996 failForInvalidBundles(
9997 I, Name: "calls",
9998 AllowedBundles: {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9999 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
10000 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
10001 LLVMContext::OB_convergencectrl, LLVMContext::OB_deactivation_symbol});
10002
10003 SDValue Callee = getValue(V: I.getCalledOperand());
10004
10005 if (I.hasDeoptState())
10006 LowerCallSiteWithDeoptBundle(Call: &I, Callee, EHPadBB: nullptr);
10007 else
10008 // Check if we can potentially perform a tail call. More detailed checking
10009 // is be done within LowerCallTo, after more information about the call is
10010 // known.
10011 LowerCallTo(CB: I, Callee, isTailCall: I.isTailCall(), isMustTailCall: I.isMustTailCall());
10012}
10013
10014void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(
10015 const CallBase &CB, const BasicBlock *EHPadBB) {
10016 auto PAB = CB.getOperandBundle(Name: "ptrauth");
10017 const Value *CalleeV = CB.getCalledOperand();
10018
10019 // Gather the call ptrauth data from the operand bundle:
10020 // [ i32 <key>, i64 <discriminator> ]
10021 const auto *Key = cast<ConstantInt>(Val: PAB->Inputs[0]);
10022 const Value *Discriminator = PAB->Inputs[1];
10023
10024 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
10025 assert(Discriminator->getType()->isIntegerTy(64) &&
10026 "Invalid ptrauth discriminator");
10027
10028 // Look through ptrauth constants to find the raw callee.
10029 // Do a direct unauthenticated call if we found it and everything matches.
10030 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CalleeV))
10031 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
10032 DL: DAG.getDataLayout()))
10033 return LowerCallTo(CB, Callee: getValue(V: CalleeCPA->getPointer()), isTailCall: CB.isTailCall(),
10034 isMustTailCall: CB.isMustTailCall(), EHPadBB);
10035
10036 // Functions should never be ptrauth-called directly.
10037 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
10038
10039 // Otherwise, do an authenticated indirect call.
10040 TargetLowering::PtrAuthInfo PAI = {.Key: Key->getZExtValue(),
10041 .Discriminator: getValue(V: Discriminator)};
10042
10043 LowerCallTo(CB, Callee: getValue(V: CalleeV), isTailCall: CB.isTailCall(), isMustTailCall: CB.isMustTailCall(),
10044 EHPadBB, PAI: &PAI);
10045}
10046
10047namespace {
10048
10049/// AsmOperandInfo - This contains information for each constraint that we are
10050/// lowering.
10051class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
10052public:
10053 /// CallOperand - If this is the result output operand or a clobber
10054 /// this is null, otherwise it is the incoming operand to the CallInst.
10055 /// This gets modified as the asm is processed.
10056 SDValue CallOperand;
10057
10058 /// AssignedRegs - If this is a register or register class operand, this
10059 /// contains the set of register corresponding to the operand.
10060 RegsForValue AssignedRegs;
10061
10062 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
10063 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
10064 }
10065
10066 /// Whether or not this operand accesses memory
10067 bool hasMemory(const TargetLowering &TLI) const {
10068 // Indirect operand accesses access memory.
10069 if (isIndirect)
10070 return true;
10071
10072 for (const auto &Code : Codes)
10073 if (TLI.getConstraintType(Constraint: Code) == TargetLowering::C_Memory)
10074 return true;
10075
10076 return false;
10077 }
10078};
10079
10080
10081} // end anonymous namespace
10082
10083/// Make sure that the output operand \p OpInfo and its corresponding input
10084/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
10085/// out).
10086static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
10087 SDISelAsmOperandInfo &MatchingOpInfo,
10088 SelectionDAG &DAG) {
10089 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
10090 return;
10091
10092 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
10093 const auto &TLI = DAG.getTargetLoweringInfo();
10094
10095 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
10096 TLI.getRegForInlineAsmConstraint(TRI, Constraint: OpInfo.ConstraintCode,
10097 VT: OpInfo.ConstraintVT);
10098 std::pair<unsigned, const TargetRegisterClass *> InputRC =
10099 TLI.getRegForInlineAsmConstraint(TRI, Constraint: MatchingOpInfo.ConstraintCode,
10100 VT: MatchingOpInfo.ConstraintVT);
10101 const bool OutOpIsIntOrFP =
10102 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
10103 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
10104 MatchingOpInfo.ConstraintVT.isFloatingPoint();
10105 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
10106 // FIXME: error out in a more elegant fashion
10107 report_fatal_error(reason: "Unsupported asm: input constraint"
10108 " with a matching output constraint of"
10109 " incompatible type!");
10110 }
10111 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
10112}
10113
10114/// Get a direct memory input to behave well as an indirect operand.
10115/// This may introduce stores, hence the need for a \p Chain.
10116/// \return The (possibly updated) chain.
10117static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
10118 SDISelAsmOperandInfo &OpInfo,
10119 SelectionDAG &DAG) {
10120 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10121
10122 // If we don't have an indirect input, put it in the constpool if we can,
10123 // otherwise spill it to a stack slot.
10124 // TODO: This isn't quite right. We need to handle these according to
10125 // the addressing mode that the constraint wants. Also, this may take
10126 // an additional register for the computation and we don't want that
10127 // either.
10128
10129 // If the operand is a float, integer, or vector constant, spill to a
10130 // constant pool entry to get its address.
10131 const Value *OpVal = OpInfo.CallOperandVal;
10132 if (isa<ConstantFP>(Val: OpVal) || isa<ConstantInt>(Val: OpVal) ||
10133 isa<ConstantVector>(Val: OpVal) || isa<ConstantDataVector>(Val: OpVal)) {
10134 OpInfo.CallOperand = DAG.getConstantPool(
10135 C: cast<Constant>(Val: OpVal), VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
10136 return Chain;
10137 }
10138
10139 // Otherwise, create a stack slot and emit a store to it before the asm.
10140 Type *Ty = OpVal->getType();
10141 auto &DL = DAG.getDataLayout();
10142 TypeSize TySize = DL.getTypeAllocSize(Ty);
10143 MachineFunction &MF = DAG.getMachineFunction();
10144 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
10145 int StackID = 0;
10146 if (TySize.isScalable())
10147 StackID = TFI->getStackIDForScalableVectors();
10148 int SSFI = MF.getFrameInfo().CreateStackObject(Size: TySize.getKnownMinValue(),
10149 Alignment: DL.getPrefTypeAlign(Ty), isSpillSlot: false,
10150 Alloca: nullptr, ID: StackID);
10151 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: TLI.getFrameIndexTy(DL));
10152 Chain = DAG.getTruncStore(Chain, dl: Location, Val: OpInfo.CallOperand, Ptr: StackSlot,
10153 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: SSFI),
10154 SVT: TLI.getMemValueType(DL, Ty));
10155 OpInfo.CallOperand = StackSlot;
10156
10157 return Chain;
10158}
10159
10160/// GetRegistersForValue - Assign registers (virtual or physical) for the
10161/// specified operand. We prefer to assign virtual registers, to allow the
10162/// register allocator to handle the assignment process. However, if the asm
10163/// uses features that we can't model on machineinstrs, we have SDISel do the
10164/// allocation. This produces generally horrible, but correct, code.
10165///
10166/// OpInfo describes the operand
10167/// RefOpInfo describes the matching operand if any, the operand otherwise
10168static std::optional<unsigned>
10169getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
10170 SDISelAsmOperandInfo &OpInfo,
10171 SDISelAsmOperandInfo &RefOpInfo) {
10172 LLVMContext &Context = *DAG.getContext();
10173 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10174
10175 MachineFunction &MF = DAG.getMachineFunction();
10176 SmallVector<Register, 4> Regs;
10177 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10178
10179 // No work to do for memory/address operands.
10180 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10181 OpInfo.ConstraintType == TargetLowering::C_Address)
10182 return std::nullopt;
10183
10184 // If this is a constraint for a single physreg, or a constraint for a
10185 // register class, find it.
10186 unsigned AssignedReg;
10187 const TargetRegisterClass *RC;
10188 std::tie(args&: AssignedReg, args&: RC) = TLI.getRegForInlineAsmConstraint(
10189 TRI: &TRI, Constraint: RefOpInfo.ConstraintCode, VT: RefOpInfo.ConstraintVT);
10190 // RC is unset only on failure. Return immediately.
10191 if (!RC)
10192 return std::nullopt;
10193
10194 // Get the actual register value type. This is important, because the user
10195 // may have asked for (e.g.) the AX register in i32 type. We need to
10196 // remember that AX is actually i16 to get the right extension.
10197 const MVT RegVT = *TRI.legalclasstypes_begin(RC: *RC);
10198
10199 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10200 // If this is an FP operand in an integer register (or visa versa), or more
10201 // generally if the operand value disagrees with the register class we plan
10202 // to stick it in, fix the operand type.
10203 //
10204 // If this is an input value, the bitcast to the new type is done now.
10205 // Bitcast for output value is done at the end of visitInlineAsm().
10206 if ((OpInfo.Type == InlineAsm::isOutput ||
10207 OpInfo.Type == InlineAsm::isInput) &&
10208 !TRI.isTypeLegalForClass(RC: *RC, T: OpInfo.ConstraintVT)) {
10209 // Try to convert to the first EVT that the reg class contains. If the
10210 // types are identical size, use a bitcast to convert (e.g. two differing
10211 // vector types). Note: output bitcast is done at the end of
10212 // visitInlineAsm().
10213 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10214 // Exclude indirect inputs while they are unsupported because the code
10215 // to perform the load is missing and thus OpInfo.CallOperand still
10216 // refers to the input address rather than the pointed-to value.
10217 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
10218 OpInfo.CallOperand =
10219 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: RegVT, Operand: OpInfo.CallOperand);
10220 OpInfo.ConstraintVT = RegVT;
10221 // If the operand is an FP value and we want it in integer registers,
10222 // use the corresponding integer type. This turns an f64 value into
10223 // i64, which can be passed with two i32 values on a 32-bit machine.
10224 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10225 MVT VT = MVT::getIntegerVT(BitWidth: OpInfo.ConstraintVT.getSizeInBits());
10226 if (OpInfo.Type == InlineAsm::isInput)
10227 OpInfo.CallOperand =
10228 DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: OpInfo.CallOperand);
10229 OpInfo.ConstraintVT = VT;
10230 }
10231 }
10232 }
10233
10234 // No need to allocate a matching input constraint since the constraint it's
10235 // matching to has already been allocated.
10236 if (OpInfo.isMatchingInputConstraint())
10237 return std::nullopt;
10238
10239 EVT ValueVT = OpInfo.ConstraintVT;
10240 if (OpInfo.ConstraintVT == MVT::Other)
10241 ValueVT = RegVT;
10242
10243 // Initialize NumRegs.
10244 unsigned NumRegs = 1;
10245 if (OpInfo.ConstraintVT != MVT::Other)
10246 NumRegs = TLI.getNumRegisters(Context, VT: OpInfo.ConstraintVT, RegisterVT: RegVT);
10247
10248 // If this is a constraint for a specific physical register, like {r17},
10249 // assign it now.
10250
10251 // If this associated to a specific register, initialize iterator to correct
10252 // place. If virtual, make sure we have enough registers
10253
10254 // Initialize iterator if necessary
10255 TargetRegisterClass::iterator I = RC->begin();
10256 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10257
10258 // Do not check for single registers.
10259 if (AssignedReg) {
10260 I = std::find(first: I, last: RC->end(), val: AssignedReg);
10261 if (I == RC->end()) {
10262 // RC does not contain the selected register, which indicates a
10263 // mismatch between the register and the required type/bitwidth.
10264 return {AssignedReg};
10265 }
10266 }
10267
10268 for (; NumRegs; --NumRegs, ++I) {
10269 assert(I != RC->end() && "Ran out of registers to allocate!");
10270 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RegClass: RC);
10271 Regs.push_back(Elt: R);
10272 }
10273
10274 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
10275 return std::nullopt;
10276}
10277
10278static unsigned
10279findMatchingInlineAsmOperand(unsigned OperandNo,
10280 const std::vector<SDValue> &AsmNodeOperands) {
10281 // Scan until we find the definition we already emitted of this operand.
10282 unsigned CurOp = InlineAsm::Op_FirstOperand;
10283 for (; OperandNo; --OperandNo) {
10284 // Advance to the next operand.
10285 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10286 const InlineAsm::Flag F(OpFlag);
10287 assert(
10288 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
10289 "Skipped past definitions?");
10290 CurOp += F.getNumOperandRegisters() + 1;
10291 }
10292 return CurOp;
10293}
10294
10295namespace {
10296
10297class ExtraFlags {
10298 unsigned Flags = 0;
10299
10300public:
10301 explicit ExtraFlags(const CallBase &Call) {
10302 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10303 if (IA->hasSideEffects())
10304 Flags |= InlineAsm::Extra_HasSideEffects;
10305 if (IA->isAlignStack())
10306 Flags |= InlineAsm::Extra_IsAlignStack;
10307 if (IA->canThrow())
10308 Flags |= InlineAsm::Extra_MayUnwind;
10309 if (Call.isConvergent())
10310 Flags |= InlineAsm::Extra_IsConvergent;
10311 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
10312 }
10313
10314 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
10315 // Ideally, we would only check against memory constraints. However, the
10316 // meaning of an Other constraint can be target-specific and we can't easily
10317 // reason about it. Therefore, be conservative and set MayLoad/MayStore
10318 // for Other constraints as well.
10319 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10320 OpInfo.ConstraintType == TargetLowering::C_Other) {
10321 if (OpInfo.Type == InlineAsm::isInput)
10322 Flags |= InlineAsm::Extra_MayLoad;
10323 else if (OpInfo.Type == InlineAsm::isOutput)
10324 Flags |= InlineAsm::Extra_MayStore;
10325 else if (OpInfo.Type == InlineAsm::isClobber)
10326 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
10327 }
10328 }
10329
10330 unsigned get() const { return Flags; }
10331};
10332
10333} // end anonymous namespace
10334
10335static bool isFunction(SDValue Op) {
10336 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
10337 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
10338 auto Fn = dyn_cast_or_null<Function>(Val: GA->getGlobal());
10339
10340 // In normal "call dllimport func" instruction (non-inlineasm) it force
10341 // indirect access by specifing call opcode. And usually specially print
10342 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
10343 // not do in this way now. (In fact, this is similar with "Data Access"
10344 // action). So here we ignore dllimport function.
10345 if (Fn && !Fn->hasDLLImportStorageClass())
10346 return true;
10347 }
10348 }
10349 return false;
10350}
10351
10352namespace {
10353
10354struct ConstraintDecisionInfo {
10355 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
10356 std::vector<SDValue> AsmNodeOperands;
10357 SDValue Glue, Chain;
10358 bool HasSideEffect = false;
10359 MCSymbol *BeginLabel = nullptr;
10360
10361 SmallVector<char> Buffer;
10362 raw_svector_ostream ErrorMsg;
10363
10364 ConstraintDecisionInfo() : ErrorMsg(Buffer) {}
10365};
10366
10367} // end anonymous namespace
10368
10369/// Construct operand info objects.
10370static bool
10371constructOperandInfo(ConstraintDecisionInfo &Info,
10372 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10373 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10374 ExtraFlags &ExtraInfo) {
10375 for (auto &T : TargetConstraints) {
10376 Info.ConstraintOperands.push_back(Elt: SDISelAsmOperandInfo(T));
10377 SDISelAsmOperandInfo &OpInfo = Info.ConstraintOperands.back();
10378
10379 if (OpInfo.CallOperandVal)
10380 OpInfo.CallOperand = Builder.getValue(V: OpInfo.CallOperandVal);
10381
10382 if (!Info.HasSideEffect)
10383 Info.HasSideEffect = OpInfo.hasMemory(TLI);
10384
10385 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
10386 // FIXME: Could we compute this on OpInfo rather than T?
10387
10388 // Compute the constraint code and ConstraintType to use.
10389 TLI.ComputeConstraintToUse(OpInfo&: T, Op: SDValue());
10390
10391 if (T.ConstraintType == TargetLowering::C_Immediate && OpInfo.CallOperand &&
10392 !isa<ConstantSDNode>(Val: OpInfo.CallOperand)) {
10393 // We've delayed emitting a diagnostic like the "n" constraint because
10394 // inlining could cause an integer showing up.
10395 Info.ErrorMsg << "constraint '" << T.ConstraintCode
10396 << "' expects an integer constant expression";
10397 return true;
10398 }
10399
10400 ExtraInfo.update(OpInfo: T);
10401 }
10402
10403 return false;
10404}
10405
10406/// Compute which constraint option to use for each operand.
10407static void
10408computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
10409 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10410 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10411 const TargetMachine &TM, SelectionDAG &DAG) {
10412 const auto *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10413 SmallVector<StringRef, 4> AsmStrs;
10414 IA->collectAsmStrs(AsmStrs);
10415
10416 int OpNo = -1;
10417 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10418 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10419 OpNo++;
10420
10421 // If this is an output operand with a matching input operand, look up the
10422 // matching input. If their types mismatch, e.g. one is an integer, the
10423 // other is floating point, or their sizes are different, flag it as an
10424 // error.
10425 if (OpInfo.hasMatchingInput()) {
10426 SDISelAsmOperandInfo &Input =
10427 Info.ConstraintOperands[OpInfo.MatchingInput];
10428 patchMatchingInput(OpInfo, MatchingOpInfo&: Input, DAG);
10429 }
10430
10431 // Compute the constraint code and ConstraintType to use.
10432 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
10433
10434 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10435 OpInfo.Type == InlineAsm::isClobber) ||
10436 OpInfo.ConstraintType == TargetLowering::C_Address)
10437 continue;
10438
10439 // In Linux PIC model, there are 4 cases about value/label addressing:
10440 //
10441 // 1: Function call or Label jmp inside the module.
10442 // 2: Data access (such as global variable, static variable) inside module.
10443 // 3: Function call or Label jmp outside the module.
10444 // 4: Data access (such as global variable) outside the module.
10445 //
10446 // Due to current llvm inline asm architecture designed to not "recognize"
10447 // the asm code, there are quite troubles for us to treat mem addressing
10448 // differently for same value/adress used in different instuctions.
10449 // For example, in pic model, call a func may in plt way or direclty
10450 // pc-related, but lea/mov a function adress may use got.
10451 //
10452 // Here we try to "recognize" function call for the case 1 and case 3 in
10453 // inline asm. And try to adjust the constraint for them.
10454 //
10455 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10456 // label, so here we don't handle jmp function label now, but we need to
10457 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10458 if (OpInfo.isIndirect && isFunction(Op: OpInfo.CallOperand) &&
10459 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10460 TM.getCodeModel() != CodeModel::Large) {
10461 OpInfo.isIndirect = false;
10462 OpInfo.ConstraintType = TargetLowering::C_Address;
10463 }
10464
10465 // If this is a memory input, and if the operand is not indirect, do what we
10466 // need to provide an address for the memory input.
10467 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10468 !OpInfo.isIndirect) {
10469 assert((OpInfo.isMultipleAlternative ||
10470 (OpInfo.Type == InlineAsm::isInput)) &&
10471 "Can only indirectify direct input operands!");
10472
10473 // Memory operands really want the address of the value.
10474 Info.Chain = getAddressForMemoryInput(Chain: Info.Chain, Location: Builder.getCurSDLoc(),
10475 OpInfo, DAG);
10476
10477 // There is no longer a Value* corresponding to this operand.
10478 OpInfo.CallOperandVal = nullptr;
10479
10480 // It is now an indirect operand.
10481 OpInfo.isIndirect = true;
10482 }
10483 }
10484}
10485
10486/// Prepare DAG-level operands. As part of this, assign virtual and physical
10487/// registers for inputs and output.
10488static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info,
10489 const CallBase &Call,
10490 SelectionDAGBuilder &Builder,
10491 const TargetLowering &TLI,
10492 SelectionDAG &DAG) {
10493 SDLoc DL = Builder.getCurSDLoc();
10494 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10495 // Assign Registers.
10496 SDISelAsmOperandInfo &RefOpInfo =
10497 OpInfo.isMatchingInputConstraint()
10498 ? Info.ConstraintOperands[OpInfo.getMatchedOperand()]
10499 : OpInfo;
10500 const auto RegError = getRegistersForValue(DAG, DL, OpInfo, RefOpInfo);
10501 if (RegError) {
10502 const MachineFunction &MF = DAG.getMachineFunction();
10503 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10504 const char *RegName = TRI.getName(RegNo: *RegError);
10505 Info.ErrorMsg << "register '" << RegName << "' allocated for constraint '"
10506 << OpInfo.ConstraintCode
10507 << "' does not match required type";
10508 return true;
10509 }
10510
10511 auto DetectWriteToReservedRegister = [&]() {
10512 const MachineFunction &MF = DAG.getMachineFunction();
10513 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10514
10515 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10516 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, PhysReg: Reg)) {
10517 Info.ErrorMsg << "write to reserved register '"
10518 << TRI.getRegAsmName(Reg) << "'";
10519 return true;
10520 }
10521 }
10522
10523 return false;
10524 };
10525 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10526 (OpInfo.Type == InlineAsm::isInput &&
10527 !OpInfo.isMatchingInputConstraint())) &&
10528 "Only address as input operand is allowed.");
10529
10530 switch (OpInfo.Type) {
10531 case InlineAsm::isOutput:
10532 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10533 const InlineAsm::ConstraintCode ConstraintID =
10534 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10535 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10536 "Failed to convert memory constraint code to constraint id.");
10537
10538 // Add information to the INLINEASM node to know about this output.
10539 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
10540 OpFlags.setMemConstraint(ConstraintID);
10541 Info.AsmNodeOperands.push_back(
10542 x: DAG.getTargetConstant(Val: OpFlags, DL, VT: MVT::i32));
10543 Info.AsmNodeOperands.push_back(x: OpInfo.CallOperand);
10544 } else {
10545 // Otherwise, this outputs to a register (directly for C_Register /
10546 // C_RegisterClass, and a target-defined fashion for
10547 // C_Immediate/C_Other). Find a register that we can use.
10548 if (OpInfo.AssignedRegs.Regs.empty()) {
10549 Info.ErrorMsg << "could not allocate output register for "
10550 << "constraint '" << OpInfo.ConstraintCode << "'";
10551 return true;
10552 }
10553
10554 if (DetectWriteToReservedRegister())
10555 return true;
10556
10557 // Add information to the INLINEASM node to know that this register is
10558 // set.
10559 OpInfo.AssignedRegs.AddInlineAsmOperands(
10560 Code: OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10561 : InlineAsm::Kind::RegDef,
10562 HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10563 }
10564 break;
10565
10566 case InlineAsm::isInput:
10567 case InlineAsm::isLabel: {
10568 SDValue InOperandVal = OpInfo.CallOperand;
10569
10570 if (OpInfo.isMatchingInputConstraint()) {
10571 // If this is required to match an output register we have already set,
10572 // just use its register.
10573 auto CurOp = findMatchingInlineAsmOperand(OperandNo: OpInfo.getMatchedOperand(),
10574 AsmNodeOperands: Info.AsmNodeOperands);
10575 InlineAsm::Flag Flag(Info.AsmNodeOperands[CurOp]->getAsZExtVal());
10576 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10577 if (OpInfo.isIndirect) {
10578 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10579 Info.ErrorMsg << "inline asm not supported yet: cannot handle "
10580 << "tied indirect register inputs";
10581 return true;
10582 }
10583
10584 SmallVector<Register, 4> Regs;
10585 MachineFunction &MF = DAG.getMachineFunction();
10586 MachineRegisterInfo &MRI = MF.getRegInfo();
10587 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10588 auto *R = cast<RegisterSDNode>(Val&: Info.AsmNodeOperands[CurOp + 1]);
10589 Register TiedReg = R->getReg();
10590 MVT RegVT = R->getSimpleValueType(ResNo: 0);
10591 const TargetRegisterClass *RC =
10592 TiedReg.isVirtual() ? MRI.getRegClass(Reg: TiedReg)
10593 : RegVT != MVT::Untyped ? TLI.getRegClassFor(VT: RegVT)
10594 : TRI.getMinimalPhysRegClass(Reg: TiedReg);
10595 for (unsigned I = 0, E = Flag.getNumOperandRegisters(); I != E; ++I)
10596 Regs.push_back(Elt: MRI.createVirtualRegister(RegClass: RC));
10597
10598 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10599
10600 // Use the produced MatchedRegs object to
10601 MatchedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl: DL, Chain&: Info.Chain,
10602 Glue: &Info.Glue, V: &Call);
10603 MatchedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::RegUse, HasMatching: true,
10604 MatchingIdx: OpInfo.getMatchedOperand(), dl: DL, DAG,
10605 Ops&: Info.AsmNodeOperands);
10606 break;
10607 }
10608
10609 assert(Flag.isMemKind() && "Unknown matching constraint!");
10610 assert(Flag.getNumOperandRegisters() == 1 &&
10611 "Unexpected number of operands");
10612
10613 // Add information to the INLINEASM node to know about this input.
10614 // See InlineAsm.h isUseOperandTiedToDef.
10615 Flag.clearMemConstraint();
10616 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10617 Info.AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10618 Val: Flag, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10619 Info.AsmNodeOperands.push_back(x: Info.AsmNodeOperands[CurOp + 1]);
10620 break;
10621 }
10622
10623 // Treat indirect 'X' constraint as memory.
10624 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10625 OpInfo.isIndirect)
10626 OpInfo.ConstraintType = TargetLowering::C_Memory;
10627
10628 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10629 OpInfo.ConstraintType == TargetLowering::C_Other) {
10630 std::vector<SDValue> Ops;
10631 TLI.LowerAsmOperandForConstraint(Op: InOperandVal, Constraint: OpInfo.ConstraintCode,
10632 Ops, DAG);
10633 if (Ops.empty()) {
10634 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10635 if (isa<ConstantSDNode>(Val: InOperandVal)) {
10636 Info.ErrorMsg << "value out of range for constraint '"
10637 << OpInfo.ConstraintCode << "'";
10638 return true;
10639 }
10640
10641 Info.ErrorMsg << "invalid operand for inline asm constraint '"
10642 << OpInfo.ConstraintCode << "'";
10643 return true;
10644 }
10645
10646 // Add information to the INLINEASM node to know about this input.
10647 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10648 Info.AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10649 Val: ResOpType, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10650 llvm::append_range(C&: Info.AsmNodeOperands, R&: Ops);
10651 break;
10652 }
10653
10654 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10655 assert((OpInfo.isIndirect ||
10656 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10657 "Operand must be indirect to be a mem!");
10658 assert(InOperandVal.getValueType() ==
10659 TLI.getPointerTy(DAG.getDataLayout()) &&
10660 "Memory operands expect pointer values");
10661
10662 const InlineAsm::ConstraintCode ConstraintID =
10663 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10664 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10665 "Failed to convert memory constraint code to constraint id.");
10666
10667 // Add information to the INLINEASM node to know about this input.
10668 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10669 ResOpType.setMemConstraint(ConstraintID);
10670 Info.AsmNodeOperands.push_back(
10671 x: DAG.getTargetConstant(Val: ResOpType, DL, VT: MVT::i32));
10672 Info.AsmNodeOperands.push_back(x: InOperandVal);
10673 break;
10674 }
10675
10676 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10677 const InlineAsm::ConstraintCode ConstraintID =
10678 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10679 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10680 "Failed to convert memory constraint code to constraint id.");
10681
10682 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10683
10684 SDValue AsmOp = InOperandVal;
10685 if (isFunction(Op: InOperandVal)) {
10686 auto *GA = cast<GlobalAddressSDNode>(Val&: InOperandVal);
10687 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10688 AsmOp = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL,
10689 VT: InOperandVal.getValueType(),
10690 offset: GA->getOffset());
10691 }
10692
10693 // Add information to the INLINEASM node to know about this input.
10694 ResOpType.setMemConstraint(ConstraintID);
10695
10696 Info.AsmNodeOperands.push_back(
10697 x: DAG.getTargetConstant(Val: ResOpType, DL, VT: MVT::i32));
10698 Info.AsmNodeOperands.push_back(x: AsmOp);
10699 break;
10700 }
10701
10702 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10703 OpInfo.ConstraintType != TargetLowering::C_Register) {
10704 Info.ErrorMsg << "unknown asm constraint '" << OpInfo.ConstraintCode
10705 << "'";
10706 return true;
10707 }
10708
10709 // TODO: Support this.
10710 if (OpInfo.isIndirect) {
10711 Info.ErrorMsg << "cannot handle indirect register inputs yet for "
10712 << "constraint '" << OpInfo.ConstraintCode << "'";
10713 return true;
10714 }
10715
10716 // Copy the input into the appropriate registers.
10717 if (OpInfo.AssignedRegs.Regs.empty()) {
10718 Info.ErrorMsg << "could not allocate input reg for constraint '"
10719 << OpInfo.ConstraintCode << "'";
10720 return true;
10721 }
10722
10723 if (DetectWriteToReservedRegister())
10724 return true;
10725
10726 OpInfo.AssignedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl: DL, Chain&: Info.Chain,
10727 Glue: &Info.Glue, V: &Call);
10728 OpInfo.AssignedRegs.AddInlineAsmOperands(
10729 Code: InlineAsm::Kind::RegUse, HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10730 break;
10731 }
10732
10733 case InlineAsm::isClobber:
10734 // Add the clobbered value to the operand list, so that the register
10735 // allocator is aware that the physreg got clobbered.
10736 if (!OpInfo.AssignedRegs.Regs.empty())
10737 OpInfo.AssignedRegs.AddInlineAsmOperands(
10738 Code: InlineAsm::Kind::Clobber, HasMatching: false, MatchingIdx: 0, dl: DL, DAG, Ops&: Info.AsmNodeOperands);
10739 break;
10740 }
10741 }
10742
10743 return false;
10744}
10745
10746/// DetermineConstraints - Find the constraints to use for inline asm operands.
10747static bool
10748determineConstraints(ConstraintDecisionInfo &Info,
10749 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10750 const CallBase &Call, SelectionDAGBuilder &Builder,
10751 const TargetLowering &TLI, const TargetMachine &TM,
10752 SelectionDAG &DAG, const BasicBlock *EHPadBB) {
10753 const auto *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
10754 ExtraFlags ExtraInfo(Call);
10755
10756 // First pass: Construct operand info objects.
10757 Info.HasSideEffect = IA->hasSideEffects();
10758 if (constructOperandInfo(Info, TargetConstraints, Builder, TLI, ExtraInfo))
10759 return true;
10760
10761 // We won't need to flush pending loads if this asm doesn't touch
10762 // memory and is nonvolatile.
10763 Info.Chain = Info.HasSideEffect ? Builder.getRoot() : DAG.getRoot();
10764
10765 bool IsCallBr = isa<CallBrInst>(Val: Call);
10766 bool EmitEHLabels = isa<InvokeInst>(Val: Call);
10767 if (IsCallBr || EmitEHLabels)
10768 // If this is a callbr or invoke we need to flush pending exports since
10769 // inlineasm_br and invoke are terminators.
10770 // We need to do this before nodes are glued to the inlineasm_br node.
10771 Info.Chain = Builder.getControlRoot();
10772
10773 if (EmitEHLabels)
10774 Info.Chain = Builder.lowerStartEH(Chain: Info.Chain, EHPadBB, BeginLabel&: Info.BeginLabel);
10775
10776 // Second pass: Compute which constraint option to use.
10777 computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM, DAG);
10778
10779 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10780 Info.AsmNodeOperands.push_back(x: SDValue()); // reserve space for input chain
10781 Info.AsmNodeOperands.push_back(x: DAG.getTargetExternalSymbol(
10782 Sym: IA->getAsmString().data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout())));
10783
10784 // If we have a !srcloc metadata node associated with it, we want to attach
10785 // this to the ultimately generated inline asm machineinstr. To do this, we
10786 // pass in the third operand as this (potentially null) inline asm MDNode.
10787 const MDNode *SrcLoc = Call.getMetadata(Kind: "srcloc");
10788 Info.AsmNodeOperands.push_back(x: DAG.getMDNode(MD: SrcLoc));
10789
10790 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10791 // bits as operand 3.
10792 Info.AsmNodeOperands.push_back(
10793 x: DAG.getTargetConstant(Val: ExtraInfo.get(), DL: Builder.getCurSDLoc(),
10794 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10795
10796 // Third pass: Prepare DAG-level operands
10797 return prepareDAGLevelOperands(Info, Call, Builder, TLI, DAG);
10798}
10799
10800/// visitInlineAsm - Handle a call to an InlineAsm object.
10801void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
10802 const BasicBlock *EHPadBB) {
10803 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10804 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
10805 DL: DAG.getDataLayout(), TRI: DAG.getSubtarget().getRegisterInfo(), Call);
10806
10807 assert((!isa<InvokeInst>(Call) || EHPadBB) &&
10808 "InvokeInst must have an EHPadBB");
10809
10810 ConstraintDecisionInfo Info;
10811 if (determineConstraints(Info, TargetConstraints, Call, Builder&: *this, TLI, TM, DAG,
10812 EHPadBB))
10813 return emitInlineAsmError(Call, Message: Info.ErrorMsg.str());
10814
10815 SDValue Glue = Info.Glue;
10816 SDValue Chain = Info.Chain;
10817
10818 // Finish up input operands. Set the input chain and add the flag last.
10819 Info.AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10820 if (Glue.getNode())
10821 Info.AsmNodeOperands.push_back(x: Glue);
10822
10823 bool IsCallBr = isa<CallBrInst>(Val: Call);
10824 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10825 Chain =
10826 DAG.getNode(Opcode: ISDOpc, DL: getCurSDLoc(), VTList: DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue),
10827 Ops: Info.AsmNodeOperands);
10828 Glue = Chain.getValue(R: 1);
10829
10830 // Do additional work to generate outputs.
10831
10832 SmallVector<EVT, 1> ResultVTs;
10833 SmallVector<SDValue, 1> ResultValues;
10834 SmallVector<SDValue, 8> OutChains;
10835
10836 llvm::Type *CallResultType = Call.getType();
10837 ArrayRef<Type *> ResultTypes;
10838 if (StructType *StructResult = dyn_cast<StructType>(Val: CallResultType))
10839 ResultTypes = StructResult->elements();
10840 else if (!CallResultType->isVoidTy())
10841 ResultTypes = ArrayRef(CallResultType);
10842
10843 auto CurResultType = ResultTypes.begin();
10844 auto handleRegAssign = [&](SDValue V) {
10845 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10846 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10847 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: *CurResultType);
10848 ++CurResultType;
10849 // If the type of the inline asm call site return value is different but has
10850 // same size as the type of the asm output bitcast it. One example of this
10851 // is for vectors with different width / number of elements. This can
10852 // happen for register classes that can contain multiple different value
10853 // types. The preg or vreg allocated may not have the same VT as was
10854 // expected.
10855 //
10856 // This can also happen for a return value that disagrees with the register
10857 // class it is put in, eg. a double in a general-purpose register on a
10858 // 32-bit machine.
10859 if (ResultVT != V.getValueType() &&
10860 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10861 V = DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10862 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10863 V.getValueType().isInteger()) {
10864 // If a result value was tied to an input value, the computed result
10865 // may have a wider width than the expected result. Extract the
10866 // relevant portion.
10867 V = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10868 }
10869 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10870 ResultVTs.push_back(Elt: ResultVT);
10871 ResultValues.push_back(Elt: V);
10872 };
10873
10874 // Deal with output operands.
10875 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10876 if (OpInfo.Type == InlineAsm::isOutput) {
10877 SDValue Val;
10878 // Skip trivial output operands.
10879 if (OpInfo.AssignedRegs.Regs.empty())
10880 continue;
10881
10882 switch (OpInfo.ConstraintType) {
10883 case TargetLowering::C_Register:
10884 case TargetLowering::C_RegisterClass:
10885 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(),
10886 Chain, Glue: &Glue, V: &Call);
10887 break;
10888 case TargetLowering::C_Immediate:
10889 case TargetLowering::C_Other:
10890 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, DL: getCurSDLoc(),
10891 OpInfo, DAG);
10892 break;
10893 case TargetLowering::C_Memory:
10894 break; // Already handled.
10895 case TargetLowering::C_Address:
10896 break; // Silence warning.
10897 case TargetLowering::C_Unknown:
10898 assert(false && "Unexpected unknown constraint");
10899 }
10900
10901 // Indirect output manifest as stores. Record output chains.
10902 if (OpInfo.isIndirect) {
10903 const Value *Ptr = OpInfo.CallOperandVal;
10904 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10905 SDValue Store = DAG.getStore(Chain, dl: getCurSDLoc(), Val, Ptr: getValue(V: Ptr),
10906 PtrInfo: MachinePointerInfo(Ptr));
10907 OutChains.push_back(Elt: Store);
10908 } else {
10909 // generate CopyFromRegs to associated registers.
10910 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10911 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10912 for (const SDValue &V : Val->op_values())
10913 handleRegAssign(V);
10914 } else
10915 handleRegAssign(Val);
10916 }
10917 }
10918 }
10919
10920 // Set results.
10921 if (!ResultValues.empty()) {
10922 assert(CurResultType == ResultTypes.end() &&
10923 "Mismatch in number of ResultTypes");
10924 assert(ResultValues.size() == ResultTypes.size() &&
10925 "Mismatch in number of output operands in asm result");
10926
10927 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
10928 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
10929 setValue(V: &Call, NewN: V);
10930 }
10931
10932 // Collect store chains.
10933 if (!OutChains.empty())
10934 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: getCurSDLoc(), VT: MVT::Other, Ops: OutChains);
10935
10936 if (const auto *II = dyn_cast<InvokeInst>(Val: &Call))
10937 Chain = lowerEndEH(Chain, II, EHPadBB, BeginLabel: Info.BeginLabel);
10938
10939 // Only Update Root if inline assembly has a memory effect.
10940 if (ResultValues.empty() || Info.HasSideEffect || !OutChains.empty() ||
10941 IsCallBr || isa<InvokeInst>(Val: Call))
10942 DAG.setRoot(Chain);
10943}
10944
10945void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10946 const Twine &Message) {
10947 LLVMContext &Ctx = *DAG.getContext();
10948 Ctx.diagnose(DI: DiagnosticInfoInlineAsm(Call, Message));
10949
10950 // Make sure we leave the DAG in a valid state
10951 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10952 SmallVector<EVT, 1> ValueVTs;
10953 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: Call.getType(), ValueVTs);
10954
10955 if (ValueVTs.empty())
10956 return;
10957
10958 SmallVector<SDValue, 1> Ops;
10959 for (const EVT &VT : ValueVTs)
10960 Ops.push_back(Elt: DAG.getUNDEF(VT));
10961
10962 setValue(V: &Call, NewN: DAG.getMergeValues(Ops, dl: getCurSDLoc()));
10963}
10964
10965void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10966 DAG.setRoot(DAG.getNode(Opcode: ISD::VASTART, DL: getCurSDLoc(),
10967 VT: MVT::Other, N1: getRoot(),
10968 N2: getValue(V: I.getArgOperand(i: 0)),
10969 N3: DAG.getSrcValue(v: I.getArgOperand(i: 0))));
10970}
10971
10972void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10973 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10974 const DataLayout &DL = DAG.getDataLayout();
10975 SDValue V = DAG.getVAArg(
10976 VT: TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType()), dl: getCurSDLoc(),
10977 Chain: getRoot(), Ptr: getValue(V: I.getOperand(i_nocapture: 0)), SV: DAG.getSrcValue(v: I.getOperand(i_nocapture: 0)),
10978 Align: DL.getABITypeAlign(Ty: I.getType()).value());
10979 DAG.setRoot(V.getValue(R: 1));
10980
10981 if (I.getType()->isPointerTy())
10982 V = DAG.getPtrExtOrTrunc(
10983 Op: V, DL: getCurSDLoc(), VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()));
10984 setValue(V: &I, NewN: V);
10985}
10986
10987void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10988 DAG.setRoot(DAG.getNode(Opcode: ISD::VAEND, DL: getCurSDLoc(),
10989 VT: MVT::Other, N1: getRoot(),
10990 N2: getValue(V: I.getArgOperand(i: 0)),
10991 N3: DAG.getSrcValue(v: I.getArgOperand(i: 0))));
10992}
10993
10994void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10995 DAG.setRoot(DAG.getNode(Opcode: ISD::VACOPY, DL: getCurSDLoc(),
10996 VT: MVT::Other, N1: getRoot(),
10997 N2: getValue(V: I.getArgOperand(i: 0)),
10998 N3: getValue(V: I.getArgOperand(i: 1)),
10999 N4: DAG.getSrcValue(v: I.getArgOperand(i: 0)),
11000 N5: DAG.getSrcValue(v: I.getArgOperand(i: 1))));
11001}
11002
11003SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
11004 const Instruction &I,
11005 SDValue Op) {
11006 std::optional<ConstantRange> CR = getRange(I);
11007
11008 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
11009 return Op;
11010
11011 APInt Hi = CR->getUnsignedMax();
11012 unsigned Bits = std::max(a: Hi.getActiveBits(),
11013 b: static_cast<unsigned>(IntegerType::MIN_INT_BITS));
11014
11015 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
11016
11017 SDLoc SL = getCurSDLoc();
11018
11019 SDValue ZExt = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: Op.getValueType(), N1: Op,
11020 N2: DAG.getValueType(SmallVT));
11021 unsigned NumVals = Op.getNode()->getNumValues();
11022 if (NumVals == 1)
11023 return ZExt;
11024
11025 SmallVector<SDValue, 4> Ops;
11026
11027 Ops.push_back(Elt: ZExt);
11028 for (unsigned I = 1; I != NumVals; ++I)
11029 Ops.push_back(Elt: Op.getValue(R: I));
11030
11031 return DAG.getMergeValues(Ops, dl: SL);
11032}
11033
11034SDValue SelectionDAGBuilder::lowerNoFPClassToAssertNoFPClass(
11035 SelectionDAG &DAG, const Instruction &I, SDValue Op) {
11036 FPClassTest Classes = getNoFPClass(I);
11037 if (Classes == fcNone)
11038 return Op;
11039
11040 SDLoc SL = getCurSDLoc();
11041 SDValue TestConst = DAG.getTargetConstant(Val: Classes, DL: SDLoc(), VT: MVT::i32);
11042
11043 if (Op.getOpcode() != ISD::MERGE_VALUES) {
11044 return DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: SL, VT: Op.getValueType(), N1: Op,
11045 N2: TestConst);
11046 }
11047
11048 SmallVector<SDValue, 8> Ops(Op.getNumOperands());
11049 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
11050 SDValue MergeOp = Op.getOperand(i: I);
11051 Ops[I] = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: SL, VT: MergeOp.getValueType(),
11052 N1: MergeOp, N2: TestConst);
11053 }
11054
11055 return DAG.getMergeValues(Ops, dl: SL);
11056}
11057
11058/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
11059/// the call being lowered.
11060///
11061/// This is a helper for lowering intrinsics that follow a target calling
11062/// convention or require stack pointer adjustment. Only a subset of the
11063/// intrinsic's operands need to participate in the calling convention.
11064void SelectionDAGBuilder::populateCallLoweringInfo(
11065 TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
11066 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
11067 AttributeSet RetAttrs, bool IsPatchPoint) {
11068 TargetLowering::ArgListTy Args;
11069 Args.reserve(n: NumArgs);
11070
11071 // Populate the argument list.
11072 // Attributes for args start at offset 1, after the return attribute.
11073 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
11074 ArgI != ArgE; ++ArgI) {
11075 const Value *V = Call->getOperand(i_nocapture: ArgI);
11076
11077 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
11078
11079 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
11080 Entry.setAttributes(Call, ArgIdx: ArgI);
11081 Args.push_back(x: Entry);
11082 }
11083
11084 CLI.setDebugLoc(getCurSDLoc())
11085 .setChain(getRoot())
11086 .setCallee(CC: Call->getCallingConv(), ResultType: ReturnTy, Target: Callee, ArgsList: std::move(Args),
11087 ResultAttrs: RetAttrs)
11088 .setDiscardResult(Call->use_empty())
11089 .setIsPatchPoint(IsPatchPoint)
11090 .setIsPreallocated(
11091 Call->countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0);
11092}
11093
11094/// Add a stack map intrinsic call's live variable operands to a stackmap
11095/// or patchpoint target node's operand list.
11096///
11097/// Constants are converted to TargetConstants purely as an optimization to
11098/// avoid constant materialization and register allocation.
11099///
11100/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
11101/// generate addess computation nodes, and so FinalizeISel can convert the
11102/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
11103/// address materialization and register allocation, but may also be required
11104/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
11105/// alloca in the entry block, then the runtime may assume that the alloca's
11106/// StackMap location can be read immediately after compilation and that the
11107/// location is valid at any point during execution (this is similar to the
11108/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
11109/// only available in a register, then the runtime would need to trap when
11110/// execution reaches the StackMap in order to read the alloca's location.
11111static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
11112 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
11113 SelectionDAGBuilder &Builder) {
11114 SelectionDAG &DAG = Builder.DAG;
11115 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
11116 SDValue Op = Builder.getValue(V: Call.getArgOperand(i: I));
11117
11118 // Things on the stack are pointer-typed, meaning that they are already
11119 // legal and can be emitted directly to target nodes.
11120 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op)) {
11121 Ops.push_back(Elt: DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: Op.getValueType()));
11122 } else {
11123 // Otherwise emit a target independent node to be legalised.
11124 Ops.push_back(Elt: Builder.getValue(V: Call.getArgOperand(i: I)));
11125 }
11126 }
11127}
11128
11129/// Lower llvm.experimental.stackmap.
11130void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
11131 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
11132 // [live variables...])
11133
11134 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
11135
11136 SDValue Chain, InGlue, Callee;
11137 SmallVector<SDValue, 32> Ops;
11138
11139 SDLoc DL = getCurSDLoc();
11140 Callee = getValue(V: CI.getCalledOperand());
11141
11142 // The stackmap intrinsic only records the live variables (the arguments
11143 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
11144 // intrinsic, this won't be lowered to a function call. This means we don't
11145 // have to worry about calling conventions and target specific lowering code.
11146 // Instead we perform the call lowering right here.
11147 //
11148 // chain, flag = CALLSEQ_START(chain, 0, 0)
11149 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
11150 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
11151 //
11152 Chain = DAG.getCALLSEQ_START(Chain: getRoot(), InSize: 0, OutSize: 0, DL);
11153 InGlue = Chain.getValue(R: 1);
11154
11155 // Add the STACKMAP operands, starting with DAG house-keeping.
11156 Ops.push_back(Elt: Chain);
11157 Ops.push_back(Elt: InGlue);
11158
11159 // Add the <id>, <numShadowBytes> operands.
11160 //
11161 // These do not require legalisation, and can be emitted directly to target
11162 // constant nodes.
11163 SDValue ID = getValue(V: CI.getArgOperand(i: 0));
11164 assert(ID.getValueType() == MVT::i64);
11165 SDValue IDConst =
11166 DAG.getTargetConstant(Val: ID->getAsZExtVal(), DL, VT: ID.getValueType());
11167 Ops.push_back(Elt: IDConst);
11168
11169 SDValue Shad = getValue(V: CI.getArgOperand(i: 1));
11170 assert(Shad.getValueType() == MVT::i32);
11171 SDValue ShadConst =
11172 DAG.getTargetConstant(Val: Shad->getAsZExtVal(), DL, VT: Shad.getValueType());
11173 Ops.push_back(Elt: ShadConst);
11174
11175 // Add the live variables.
11176 addStackMapLiveVars(Call: CI, StartIdx: 2, DL, Ops, Builder&: *this);
11177
11178 // Create the STACKMAP node.
11179 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
11180 Chain = DAG.getNode(Opcode: ISD::STACKMAP, DL, VTList: NodeTys, Ops);
11181 InGlue = Chain.getValue(R: 1);
11182
11183 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: InGlue, DL);
11184
11185 // Stackmaps don't generate values, so nothing goes into the NodeMap.
11186
11187 // Set the root to the target-lowered call chain.
11188 DAG.setRoot(Chain);
11189
11190 // Inform the Frame Information that we have a stackmap in this function.
11191 FuncInfo.MF->getFrameInfo().setHasStackMap();
11192}
11193
11194/// Lower llvm.experimental.patchpoint directly to its target opcode.
11195void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
11196 const BasicBlock *EHPadBB) {
11197 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
11198 // i32 <numBytes>,
11199 // i8* <target>,
11200 // i32 <numArgs>,
11201 // [Args...],
11202 // [live variables...])
11203
11204 CallingConv::ID CC = CB.getCallingConv();
11205 bool IsAnyRegCC = CC == CallingConv::AnyReg;
11206 bool HasDef = !CB.getType()->isVoidTy();
11207 SDLoc dl = getCurSDLoc();
11208 SDValue Callee = getValue(V: CB.getArgOperand(i: PatchPointOpers::TargetPos));
11209
11210 // Handle immediate and symbolic callees.
11211 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Val&: Callee))
11212 Callee = DAG.getIntPtrConstant(Val: ConstCallee->getZExtValue(), DL: dl,
11213 /*isTarget=*/true);
11214 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11215 Callee = DAG.getTargetGlobalAddress(GV: SymbolicCallee->getGlobal(),
11216 DL: SDLoc(SymbolicCallee),
11217 VT: SymbolicCallee->getValueType(ResNo: 0));
11218
11219 // Get the real number of arguments participating in the call <numArgs>
11220 SDValue NArgVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NArgPos));
11221 unsigned NumArgs = NArgVal->getAsZExtVal();
11222
11223 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
11224 // Intrinsics include all meta-operands up to but not including CC.
11225 unsigned NumMetaOpers = PatchPointOpers::CCPos;
11226 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
11227 "Not enough arguments provided to the patchpoint intrinsic");
11228
11229 // For AnyRegCC the arguments are lowered later on manually.
11230 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11231 Type *ReturnTy =
11232 IsAnyRegCC ? Type::getVoidTy(C&: *DAG.getContext()) : CB.getType();
11233
11234 TargetLowering::CallLoweringInfo CLI(DAG);
11235 populateCallLoweringInfo(CLI, Call: &CB, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee,
11236 ReturnTy, RetAttrs: CB.getAttributes().getRetAttrs(), IsPatchPoint: true);
11237 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
11238
11239 SDNode *CallEnd = Result.second.getNode();
11240 if (CallEnd->getOpcode() == ISD::EH_LABEL)
11241 CallEnd = CallEnd->getOperand(Num: 0).getNode();
11242 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
11243 CallEnd = CallEnd->getOperand(Num: 0).getNode();
11244
11245 /// Get a call instruction from the call sequence chain.
11246 /// Tail calls are not allowed.
11247 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
11248 "Expected a callseq node.");
11249 SDNode *Call = CallEnd->getOperand(Num: 0).getNode();
11250 bool HasGlue = Call->getGluedNode();
11251
11252 // Replace the target specific call node with the patchable intrinsic.
11253 SmallVector<SDValue, 8> Ops;
11254
11255 // Push the chain.
11256 Ops.push_back(Elt: *(Call->op_begin()));
11257
11258 // Optionally, push the glue (if any).
11259 if (HasGlue)
11260 Ops.push_back(Elt: *(Call->op_end() - 1));
11261
11262 // Push the register mask info.
11263 if (HasGlue)
11264 Ops.push_back(Elt: *(Call->op_end() - 2));
11265 else
11266 Ops.push_back(Elt: *(Call->op_end() - 1));
11267
11268 // Add the <id> and <numBytes> constants.
11269 SDValue IDVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::IDPos));
11270 Ops.push_back(Elt: DAG.getTargetConstant(Val: IDVal->getAsZExtVal(), DL: dl, VT: MVT::i64));
11271 SDValue NBytesVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NBytesPos));
11272 Ops.push_back(Elt: DAG.getTargetConstant(Val: NBytesVal->getAsZExtVal(), DL: dl, VT: MVT::i32));
11273
11274 // Add the callee.
11275 Ops.push_back(Elt: Callee);
11276
11277 // Adjust <numArgs> to account for any arguments that have been passed on the
11278 // stack instead.
11279 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
11280 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
11281 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11282 Ops.push_back(Elt: DAG.getTargetConstant(Val: NumCallRegArgs, DL: dl, VT: MVT::i32));
11283
11284 // Add the calling convention
11285 Ops.push_back(Elt: DAG.getTargetConstant(Val: (unsigned)CC, DL: dl, VT: MVT::i32));
11286
11287 // Add the arguments we omitted previously. The register allocator should
11288 // place these in any free register.
11289 if (IsAnyRegCC)
11290 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
11291 Ops.push_back(Elt: getValue(V: CB.getArgOperand(i)));
11292
11293 // Push the arguments from the call instruction.
11294 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
11295 Ops.append(in_start: Call->op_begin() + 2, in_end: e);
11296
11297 // Push live variables for the stack map.
11298 addStackMapLiveVars(Call: CB, StartIdx: NumMetaOpers + NumArgs, DL: dl, Ops, Builder&: *this);
11299
11300 SDVTList NodeTys;
11301 if (IsAnyRegCC && HasDef) {
11302 // Create the return types based on the intrinsic definition
11303 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11304 SmallVector<EVT, 3> ValueVTs;
11305 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: CB.getType(), ValueVTs);
11306 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
11307
11308 // There is always a chain and a glue type at the end
11309 ValueVTs.push_back(Elt: MVT::Other);
11310 ValueVTs.push_back(Elt: MVT::Glue);
11311 NodeTys = DAG.getVTList(VTs: ValueVTs);
11312 } else
11313 NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
11314
11315 // Replace the target specific call node with a PATCHPOINT node.
11316 SDValue PPV = DAG.getNode(Opcode: ISD::PATCHPOINT, DL: dl, VTList: NodeTys, Ops);
11317
11318 // Update the NodeMap.
11319 if (HasDef) {
11320 if (IsAnyRegCC)
11321 setValue(V: &CB, NewN: SDValue(PPV.getNode(), 0));
11322 else
11323 setValue(V: &CB, NewN: Result.first);
11324 }
11325
11326 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
11327 // call sequence. Furthermore the location of the chain and glue can change
11328 // when the AnyReg calling convention is used and the intrinsic returns a
11329 // value.
11330 if (IsAnyRegCC && HasDef) {
11331 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
11332 SDValue To[] = {PPV.getValue(R: 1), PPV.getValue(R: 2)};
11333 DAG.ReplaceAllUsesOfValuesWith(From, To, Num: 2);
11334 } else
11335 DAG.ReplaceAllUsesWith(From: Call, To: PPV.getNode());
11336 DAG.DeleteNode(N: Call);
11337
11338 // Inform the Frame Information that we have a patchpoint in this function.
11339 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11340}
11341
11342void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
11343 unsigned Intrinsic) {
11344 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11345 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
11346 SDValue Op2;
11347 if (I.arg_size() > 1)
11348 Op2 = getValue(V: I.getArgOperand(i: 1));
11349 SDLoc dl = getCurSDLoc();
11350 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
11351 SDValue Res;
11352 SDNodeFlags SDFlags;
11353 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
11354 SDFlags.copyFMF(FPMO: *FPMO);
11355
11356 switch (Intrinsic) {
11357 case Intrinsic::vector_reduce_fadd:
11358 if (SDFlags.hasAllowReassociation())
11359 Res = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Op1,
11360 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FADD, DL: dl, VT, Operand: Op2, Flags: SDFlags),
11361 Flags: SDFlags);
11362 else
11363 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FADD, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
11364 break;
11365 case Intrinsic::vector_reduce_fmul:
11366 if (SDFlags.hasAllowReassociation())
11367 Res = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Op1,
11368 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FMUL, DL: dl, VT, Operand: Op2, Flags: SDFlags),
11369 Flags: SDFlags);
11370 else
11371 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FMUL, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
11372 break;
11373 case Intrinsic::vector_reduce_add:
11374 Res = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT, Operand: Op1);
11375 break;
11376 case Intrinsic::vector_reduce_mul:
11377 Res = DAG.getNode(Opcode: ISD::VECREDUCE_MUL, DL: dl, VT, Operand: Op1);
11378 break;
11379 case Intrinsic::vector_reduce_and:
11380 Res = DAG.getNode(Opcode: ISD::VECREDUCE_AND, DL: dl, VT, Operand: Op1);
11381 break;
11382 case Intrinsic::vector_reduce_or:
11383 Res = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: dl, VT, Operand: Op1);
11384 break;
11385 case Intrinsic::vector_reduce_xor:
11386 Res = DAG.getNode(Opcode: ISD::VECREDUCE_XOR, DL: dl, VT, Operand: Op1);
11387 break;
11388 case Intrinsic::vector_reduce_smax:
11389 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMAX, DL: dl, VT, Operand: Op1);
11390 break;
11391 case Intrinsic::vector_reduce_smin:
11392 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMIN, DL: dl, VT, Operand: Op1);
11393 break;
11394 case Intrinsic::vector_reduce_umax:
11395 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL: dl, VT, Operand: Op1);
11396 break;
11397 case Intrinsic::vector_reduce_umin:
11398 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMIN, DL: dl, VT, Operand: Op1);
11399 break;
11400 case Intrinsic::vector_reduce_fmax:
11401 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAX, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11402 break;
11403 case Intrinsic::vector_reduce_fmin:
11404 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMIN, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11405 break;
11406 case Intrinsic::vector_reduce_fmaximum:
11407 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAXIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11408 break;
11409 case Intrinsic::vector_reduce_fminimum:
11410 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMINIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
11411 break;
11412 default:
11413 llvm_unreachable("Unhandled vector reduce intrinsic");
11414 }
11415 setValue(V: &I, NewN: Res);
11416}
11417
11418/// Returns an AttributeList representing the attributes applied to the return
11419/// value of the given call.
11420static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
11421 SmallVector<Attribute::AttrKind, 2> Attrs;
11422 if (CLI.RetSExt)
11423 Attrs.push_back(Elt: Attribute::SExt);
11424 if (CLI.RetZExt)
11425 Attrs.push_back(Elt: Attribute::ZExt);
11426 if (CLI.IsInReg)
11427 Attrs.push_back(Elt: Attribute::InReg);
11428
11429 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
11430 Kinds: Attrs);
11431}
11432
11433/// TargetLowering::LowerCallTo - This is the default LowerCallTo
11434/// implementation, which just calls LowerCall.
11435/// FIXME: When all targets are
11436/// migrated to using LowerCall, this hook should be integrated into SDISel.
11437std::pair<SDValue, SDValue>
11438TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
11439 LLVMContext &Context = CLI.RetTy->getContext();
11440
11441 // Handle the incoming return values from the call.
11442 CLI.Ins.clear();
11443 SmallVector<Type *, 4> RetOrigTys;
11444 SmallVector<TypeSize, 4> Offsets;
11445 auto &DL = CLI.DAG.getDataLayout();
11446 ComputeValueTypes(DL, Ty: CLI.OrigRetTy, Types&: RetOrigTys, Offsets: &Offsets);
11447
11448 SmallVector<EVT, 4> RetVTs;
11449 if (CLI.RetTy != CLI.OrigRetTy) {
11450 assert(RetOrigTys.size() == 1 &&
11451 "Only supported for non-aggregate returns");
11452 RetVTs.push_back(Elt: getValueType(DL, Ty: CLI.RetTy));
11453 } else {
11454 for (Type *Ty : RetOrigTys)
11455 RetVTs.push_back(Elt: getValueType(DL, Ty));
11456 }
11457
11458 if (CLI.IsPostTypeLegalization) {
11459 // If we are lowering a libcall after legalization, split the return type.
11460 SmallVector<Type *, 4> OldRetOrigTys;
11461 SmallVector<EVT, 4> OldRetVTs;
11462 SmallVector<TypeSize, 4> OldOffsets;
11463 RetOrigTys.swap(RHS&: OldRetOrigTys);
11464 RetVTs.swap(RHS&: OldRetVTs);
11465 Offsets.swap(RHS&: OldOffsets);
11466
11467 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
11468 EVT RetVT = OldRetVTs[i];
11469 uint64_t Offset = OldOffsets[i];
11470 MVT RegisterVT = getRegisterType(Context, VT: RetVT);
11471 unsigned NumRegs = getNumRegisters(Context, VT: RetVT);
11472 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
11473 RetOrigTys.append(NumInputs: NumRegs, Elt: OldRetOrigTys[i]);
11474 RetVTs.append(NumInputs: NumRegs, Elt: RegisterVT);
11475 for (unsigned j = 0; j != NumRegs; ++j)
11476 Offsets.push_back(Elt: TypeSize::getFixed(ExactSize: Offset + j * RegisterVTByteSZ));
11477 }
11478 }
11479
11480 SmallVector<ISD::OutputArg, 4> Outs;
11481 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI: *this, DL);
11482
11483 bool CanLowerReturn =
11484 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
11485 CLI.IsVarArg, Outs, Context, RetTy: CLI.RetTy);
11486
11487 SDValue DemoteStackSlot;
11488 int DemoteStackIdx = -100;
11489 if (!CanLowerReturn) {
11490 // FIXME: equivalent assert?
11491 // assert(!CS.hasInAllocaArgument() &&
11492 // "sret demotion is incompatible with inalloca");
11493 uint64_t TySize = DL.getTypeAllocSize(Ty: CLI.RetTy);
11494 Align Alignment = DL.getPrefTypeAlign(Ty: CLI.RetTy);
11495 MachineFunction &MF = CLI.DAG.getMachineFunction();
11496 DemoteStackIdx =
11497 MF.getFrameInfo().CreateStackObject(Size: TySize, Alignment, isSpillSlot: false);
11498 Type *StackSlotPtrType = PointerType::get(C&: Context, AddressSpace: DL.getAllocaAddrSpace());
11499
11500 DemoteStackSlot = CLI.DAG.getFrameIndex(FI: DemoteStackIdx, VT: getFrameIndexTy(DL));
11501 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11502 Entry.IsSRet = true;
11503 Entry.Alignment = Alignment;
11504 CLI.getArgs().insert(position: CLI.getArgs().begin(), x: Entry);
11505 CLI.NumFixedArgs += 1;
11506 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11507 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(C&: Context);
11508
11509 // sret demotion isn't compatible with tail-calls, since the sret argument
11510 // points into the callers stack frame.
11511 CLI.IsTailCall = false;
11512 } else {
11513 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11514 Ty: CLI.RetTy, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11515 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
11516 ISD::ArgFlagsTy Flags;
11517 if (NeedsRegBlock) {
11518 Flags.setInConsecutiveRegs();
11519 if (I == RetVTs.size() - 1)
11520 Flags.setInConsecutiveRegsLast();
11521 }
11522 EVT VT = RetVTs[I];
11523 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11524 unsigned NumRegs =
11525 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11526 for (unsigned i = 0; i != NumRegs; ++i) {
11527 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
11528 CLI.IsReturnValueUsed, ISD::InputArg::NoArgIndex, 0);
11529 if (CLI.RetTy->isPointerTy()) {
11530 Ret.Flags.setPointer();
11531 Ret.Flags.setPointerAddrSpace(
11532 cast<PointerType>(Val: CLI.RetTy)->getAddressSpace());
11533 }
11534 if (CLI.RetSExt)
11535 Ret.Flags.setSExt();
11536 if (CLI.RetZExt)
11537 Ret.Flags.setZExt();
11538 if (CLI.IsInReg)
11539 Ret.Flags.setInReg();
11540 CLI.Ins.push_back(Elt: Ret);
11541 }
11542 }
11543 }
11544
11545 // We push in swifterror return as the last element of CLI.Ins.
11546 ArgListTy &Args = CLI.getArgs();
11547 if (supportSwiftError()) {
11548 for (const ArgListEntry &Arg : Args) {
11549 if (Arg.IsSwiftError) {
11550 ISD::ArgFlagsTy Flags;
11551 Flags.setSwiftError();
11552 ISD::InputArg Ret(Flags, getPointerTy(DL), EVT(getPointerTy(DL)),
11553 PointerType::getUnqual(C&: Context),
11554 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
11555 CLI.Ins.push_back(Elt: Ret);
11556 }
11557 }
11558 }
11559
11560 // Handle all of the outgoing arguments.
11561 CLI.Outs.clear();
11562 CLI.OutVals.clear();
11563 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11564 SmallVector<Type *, 4> OrigArgTys;
11565 ComputeValueTypes(DL, Ty: Args[i].OrigTy, Types&: OrigArgTys);
11566 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11567 Type *FinalType = Args[i].Ty;
11568 if (Args[i].IsByVal)
11569 FinalType = Args[i].IndirectType;
11570 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11571 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11572 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
11573 ++Value) {
11574 Type *OrigArgTy = OrigArgTys[Value];
11575 Type *ArgTy = OrigArgTy;
11576 if (Args[i].Ty != Args[i].OrigTy) {
11577 assert(Value == 0 && "Only supported for non-aggregate arguments");
11578 ArgTy = Args[i].Ty;
11579 }
11580
11581 EVT VT = getValueType(DL, Ty: ArgTy);
11582 SDValue Op = SDValue(Args[i].Node.getNode(),
11583 Args[i].Node.getResNo() + Value);
11584 ISD::ArgFlagsTy Flags;
11585
11586 // Certain targets (such as MIPS), may have a different ABI alignment
11587 // for a type depending on the context. Give the target a chance to
11588 // specify the alignment it wants.
11589 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11590 Flags.setOrigAlign(OriginalAlignment);
11591
11592 if (i >= CLI.NumFixedArgs)
11593 Flags.setVarArg();
11594 if (ArgTy->isPointerTy()) {
11595 Flags.setPointer();
11596 Flags.setPointerAddrSpace(cast<PointerType>(Val: ArgTy)->getAddressSpace());
11597 }
11598 if (Args[i].IsZExt)
11599 Flags.setZExt();
11600 if (Args[i].IsSExt)
11601 Flags.setSExt();
11602 if (Args[i].IsNoExt)
11603 Flags.setNoExt();
11604 if (Args[i].IsInReg) {
11605 // If we are using vectorcall calling convention, a structure that is
11606 // passed InReg - is surely an HVA
11607 if (CLI.CallConv == CallingConv::X86_VectorCall &&
11608 isa<StructType>(Val: FinalType)) {
11609 // The first value of a structure is marked
11610 if (0 == Value)
11611 Flags.setHvaStart();
11612 Flags.setHva();
11613 }
11614 // Set InReg Flag
11615 Flags.setInReg();
11616 }
11617 if (Args[i].IsSRet)
11618 Flags.setSRet();
11619 if (Args[i].IsSwiftSelf)
11620 Flags.setSwiftSelf();
11621 if (Args[i].IsSwiftAsync)
11622 Flags.setSwiftAsync();
11623 if (Args[i].IsSwiftError)
11624 Flags.setSwiftError();
11625 if (Args[i].IsCFGuardTarget)
11626 Flags.setCFGuardTarget();
11627 if (Args[i].IsByVal)
11628 Flags.setByVal();
11629 if (Args[i].IsByRef)
11630 Flags.setByRef();
11631 if (Args[i].IsPreallocated) {
11632 Flags.setPreallocated();
11633 // Set the byval flag for CCAssignFn callbacks that don't know about
11634 // preallocated. This way we can know how many bytes we should've
11635 // allocated and how many bytes a callee cleanup function will pop. If
11636 // we port preallocated to more targets, we'll have to add custom
11637 // preallocated handling in the various CC lowering callbacks.
11638 Flags.setByVal();
11639 }
11640 if (Args[i].IsInAlloca) {
11641 Flags.setInAlloca();
11642 // Set the byval flag for CCAssignFn callbacks that don't know about
11643 // inalloca. This way we can know how many bytes we should've allocated
11644 // and how many bytes a callee cleanup function will pop. If we port
11645 // inalloca to more targets, we'll have to add custom inalloca handling
11646 // in the various CC lowering callbacks.
11647 Flags.setByVal();
11648 }
11649 Align MemAlign;
11650 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11651 unsigned FrameSize = DL.getTypeAllocSize(Ty: Args[i].IndirectType);
11652 Flags.setByValSize(FrameSize);
11653
11654 // info is not there but there are cases it cannot get right.
11655 if (auto MA = Args[i].Alignment)
11656 MemAlign = *MA;
11657 else
11658 MemAlign = getByValTypeAlignment(Ty: Args[i].IndirectType, DL);
11659 } else if (auto MA = Args[i].Alignment) {
11660 MemAlign = *MA;
11661 } else {
11662 MemAlign = OriginalAlignment;
11663 }
11664 Flags.setMemAlign(MemAlign);
11665 if (Args[i].IsNest)
11666 Flags.setNest();
11667 if (NeedsRegBlock)
11668 Flags.setInConsecutiveRegs();
11669
11670 MVT PartVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11671 unsigned NumParts =
11672 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11673 SmallVector<SDValue, 4> Parts(NumParts);
11674 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
11675
11676 if (Args[i].IsSExt)
11677 ExtendKind = ISD::SIGN_EXTEND;
11678 else if (Args[i].IsZExt)
11679 ExtendKind = ISD::ZERO_EXTEND;
11680
11681 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11682 // for now.
11683 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11684 CanLowerReturn) {
11685 assert((CLI.RetTy == Args[i].Ty ||
11686 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11687 CLI.RetTy->getPointerAddressSpace() ==
11688 Args[i].Ty->getPointerAddressSpace())) &&
11689 RetVTs.size() == NumValues && "unexpected use of 'returned'");
11690 // Before passing 'returned' to the target lowering code, ensure that
11691 // either the register MVT and the actual EVT are the same size or that
11692 // the return value and argument are extended in the same way; in these
11693 // cases it's safe to pass the argument register value unchanged as the
11694 // return register value (although it's at the target's option whether
11695 // to do so)
11696 // TODO: allow code generation to take advantage of partially preserved
11697 // registers rather than clobbering the entire register when the
11698 // parameter extension method is not compatible with the return
11699 // extension method
11700 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11701 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11702 CLI.RetZExt == Args[i].IsZExt))
11703 Flags.setReturned();
11704 }
11705
11706 getCopyToParts(DAG&: CLI.DAG, DL: CLI.DL, Val: Op, Parts: &Parts[0], NumParts, PartVT, V: CLI.CB,
11707 CallConv: CLI.CallConv, ExtendKind);
11708
11709 for (unsigned j = 0; j != NumParts; ++j) {
11710 // if it isn't first piece, alignment must be 1
11711 // For scalable vectors the scalable part is currently handled
11712 // by individual targets, so we just use the known minimum size here.
11713 ISD::OutputArg MyFlags(
11714 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
11715 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11716 if (NumParts > 1 && j == 0)
11717 MyFlags.Flags.setSplit();
11718 else if (j != 0) {
11719 MyFlags.Flags.setOrigAlign(Align(1));
11720 if (j == NumParts - 1)
11721 MyFlags.Flags.setSplitEnd();
11722 }
11723
11724 CLI.Outs.push_back(Elt: MyFlags);
11725 CLI.OutVals.push_back(Elt: Parts[j]);
11726 }
11727
11728 if (NeedsRegBlock && Value == NumValues - 1)
11729 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11730 }
11731 }
11732
11733 SmallVector<SDValue, 4> InVals;
11734 CLI.Chain = LowerCall(CLI, InVals);
11735
11736 // Update CLI.InVals to use outside of this function.
11737 CLI.InVals = InVals;
11738
11739 // Verify that the target's LowerCall behaved as expected.
11740 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11741 "LowerCall didn't return a valid chain!");
11742 assert((!CLI.IsTailCall || InVals.empty()) &&
11743 "LowerCall emitted a return value for a tail call!");
11744 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11745 "LowerCall didn't emit the correct number of values!");
11746
11747 // For a tail call, the return value is merely live-out and there aren't
11748 // any nodes in the DAG representing it. Return a special value to
11749 // indicate that a tail call has been emitted and no more Instructions
11750 // should be processed in the current block.
11751 if (CLI.IsTailCall) {
11752 CLI.DAG.setRoot(CLI.Chain);
11753 return std::make_pair(x: SDValue(), y: SDValue());
11754 }
11755
11756#ifndef NDEBUG
11757 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11758 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11759 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11760 "LowerCall emitted a value with the wrong type!");
11761 }
11762#endif
11763
11764 SmallVector<SDValue, 4> ReturnValues;
11765 if (!CanLowerReturn) {
11766 // The instruction result is the result of loading from the
11767 // hidden sret parameter.
11768 MVT PtrVT = getPointerTy(DL, AS: DL.getAllocaAddrSpace());
11769
11770 unsigned NumValues = RetVTs.size();
11771 ReturnValues.resize(N: NumValues);
11772 SmallVector<SDValue, 4> Chains(NumValues);
11773
11774 // An aggregate return value cannot wrap around the address space, so
11775 // offsets to its parts don't wrap either.
11776 MachineFunction &MF = CLI.DAG.getMachineFunction();
11777 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(ObjectIdx: DemoteStackIdx);
11778 for (unsigned i = 0; i < NumValues; ++i) {
11779 SDValue Add = CLI.DAG.getMemBasePlusOffset(
11780 Base: DemoteStackSlot, Offset: CLI.DAG.getConstant(Val: Offsets[i], DL: CLI.DL, VT: PtrVT),
11781 DL: CLI.DL, Flags: SDNodeFlags::NoUnsignedWrap);
11782 SDValue L = CLI.DAG.getLoad(
11783 VT: RetVTs[i], dl: CLI.DL, Chain: CLI.Chain, Ptr: Add,
11784 PtrInfo: MachinePointerInfo::getFixedStack(MF&: CLI.DAG.getMachineFunction(),
11785 FI: DemoteStackIdx, Offset: Offsets[i]),
11786 Alignment: HiddenSRetAlign);
11787 ReturnValues[i] = L;
11788 Chains[i] = L.getValue(R: 1);
11789 }
11790
11791 CLI.Chain = CLI.DAG.getNode(Opcode: ISD::TokenFactor, DL: CLI.DL, VT: MVT::Other, Ops: Chains);
11792 } else {
11793 // Collect the legal value parts into potentially illegal values
11794 // that correspond to the original function's return values.
11795 std::optional<ISD::NodeType> AssertOp;
11796 if (CLI.RetSExt)
11797 AssertOp = ISD::AssertSext;
11798 else if (CLI.RetZExt)
11799 AssertOp = ISD::AssertZext;
11800 unsigned CurReg = 0;
11801 for (EVT VT : RetVTs) {
11802 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CC: CLI.CallConv, VT);
11803 unsigned NumRegs =
11804 getNumRegistersForCallingConv(Context, CC: CLI.CallConv, VT);
11805
11806 ReturnValues.push_back(Elt: getCopyFromParts(
11807 DAG&: CLI.DAG, DL: CLI.DL, Parts: &InVals[CurReg], NumParts: NumRegs, PartVT: RegisterVT, ValueVT: VT, V: nullptr,
11808 InChain: CLI.Chain, CC: CLI.CallConv, AssertOp));
11809 CurReg += NumRegs;
11810 }
11811
11812 // For a function returning void, there is no return value. We can't create
11813 // such a node, so we just return a null return value in that case. In
11814 // that case, nothing will actually look at the value.
11815 if (ReturnValues.empty())
11816 return std::make_pair(x: SDValue(), y&: CLI.Chain);
11817 }
11818
11819 SDValue Res = CLI.DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: CLI.DL,
11820 VTList: CLI.DAG.getVTList(VTs: RetVTs), Ops: ReturnValues);
11821 return std::make_pair(x&: Res, y&: CLI.Chain);
11822}
11823
11824/// Places new result values for the node in Results (their number
11825/// and types must exactly match those of the original return values of
11826/// the node), or leaves Results empty, which indicates that the node is not
11827/// to be custom lowered after all.
11828void TargetLowering::LowerOperationWrapper(SDNode *N,
11829 SmallVectorImpl<SDValue> &Results,
11830 SelectionDAG &DAG) const {
11831 SDValue Res = LowerOperation(Op: SDValue(N, 0), DAG);
11832
11833 if (!Res.getNode())
11834 return;
11835
11836 // If the original node has one result, take the return value from
11837 // LowerOperation as is. It might not be result number 0.
11838 if (N->getNumValues() == 1) {
11839 Results.push_back(Elt: Res);
11840 return;
11841 }
11842
11843 // If the original node has multiple results, then the return node should
11844 // have the same number of results.
11845 assert((N->getNumValues() == Res->getNumValues()) &&
11846 "Lowering returned the wrong number of results!");
11847
11848 // Places new result values base on N result number.
11849 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11850 Results.push_back(Elt: Res.getValue(R: I));
11851}
11852
11853SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
11854 llvm_unreachable("LowerOperation not implemented for this target!");
11855}
11856
11857void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
11858 Register Reg,
11859 ISD::NodeType ExtendType) {
11860 SDValue Op = getNonRegisterValue(V);
11861 assert((Op.getOpcode() != ISD::CopyFromReg ||
11862 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11863 "Copy from a reg to the same reg!");
11864 assert(!Reg.isPhysical() && "Is a physreg");
11865
11866 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11867 // If this is an InlineAsm we have to match the registers required, not the
11868 // notional registers required by the type.
11869
11870 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11871 std::nullopt); // This is not an ABI copy.
11872 SDValue Chain = DAG.getEntryNode();
11873
11874 if (ExtendType == ISD::ANY_EXTEND) {
11875 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(Val: V);
11876 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11877 ExtendType = PreferredExtendIt->second;
11878 }
11879 RFV.getCopyToRegs(Val: Op, DAG, dl: getCurSDLoc(), Chain, Glue: nullptr, V, PreferredExtendType: ExtendType);
11880 PendingExports.push_back(Elt: Chain);
11881}
11882
11883#include "llvm/CodeGen/SelectionDAGISel.h"
11884
11885/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11886/// entry block, return true. This includes arguments used by switches, since
11887/// the switch may expand into multiple basic blocks.
11888static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11889 // With FastISel active, we may be splitting blocks, so force creation
11890 // of virtual registers for all non-dead arguments.
11891 if (FastISel)
11892 return A->use_empty();
11893
11894 const BasicBlock &Entry = A->getParent()->front();
11895 for (const User *U : A->users())
11896 if (cast<Instruction>(Val: U)->getParent() != &Entry || isa<SwitchInst>(Val: U))
11897 return false; // Use not in entry block.
11898
11899 return true;
11900}
11901
11902using ArgCopyElisionMapTy =
11903 DenseMap<const Argument *,
11904 std::pair<const AllocaInst *, const StoreInst *>>;
11905
11906/// Scan the entry block of the function in FuncInfo for arguments that look
11907/// like copies into a local alloca. Record any copied arguments in
11908/// ArgCopyElisionCandidates.
11909static void
11910findArgumentCopyElisionCandidates(const DataLayout &DL,
11911 FunctionLoweringInfo *FuncInfo,
11912 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
11913 // Record the state of every static alloca used in the entry block. Argument
11914 // allocas are all used in the entry block, so we need approximately as many
11915 // entries as we have arguments.
11916 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
11917 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
11918 unsigned NumArgs = FuncInfo->Fn->arg_size();
11919 StaticAllocas.reserve(NumEntries: NumArgs * 2);
11920
11921 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
11922 if (!V)
11923 return nullptr;
11924 V = V->stripPointerCasts();
11925 const auto *AI = dyn_cast<AllocaInst>(Val: V);
11926 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(Val: AI))
11927 return nullptr;
11928 auto Iter = StaticAllocas.insert(KV: {AI, Unknown});
11929 return &Iter.first->second;
11930 };
11931
11932 // Look for stores of arguments to static allocas. Look through bitcasts and
11933 // GEPs to handle type coercions, as long as the alloca is fully initialized
11934 // by the store. Any non-store use of an alloca escapes it and any subsequent
11935 // unanalyzed store might write it.
11936 // FIXME: Handle structs initialized with multiple stores.
11937 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11938 // Look for stores, and handle non-store uses conservatively.
11939 const auto *SI = dyn_cast<StoreInst>(Val: &I);
11940 if (!SI) {
11941 // We will look through cast uses, so ignore them completely.
11942 if (I.isCast())
11943 continue;
11944 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11945 // to allocas.
11946 if (I.isDebugOrPseudoInst())
11947 continue;
11948 // This is an unknown instruction. Assume it escapes or writes to all
11949 // static alloca operands.
11950 for (const Use &U : I.operands()) {
11951 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11952 *Info = StaticAllocaInfo::Clobbered;
11953 }
11954 continue;
11955 }
11956
11957 // If the stored value is a static alloca, mark it as escaped.
11958 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11959 *Info = StaticAllocaInfo::Clobbered;
11960
11961 // Check if the destination is a static alloca.
11962 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11963 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11964 if (!Info)
11965 continue;
11966 const AllocaInst *AI = cast<AllocaInst>(Val: Dst);
11967
11968 // Skip allocas that have been initialized or clobbered.
11969 if (*Info != StaticAllocaInfo::Unknown)
11970 continue;
11971
11972 // Check if the stored value is an argument, and that this store fully
11973 // initializes the alloca.
11974 // If the argument type has padding bits we can't directly forward a pointer
11975 // as the upper bits may contain garbage.
11976 // Don't elide copies from the same argument twice.
11977 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11978 const auto *Arg = dyn_cast<Argument>(Val);
11979 std::optional<TypeSize> AllocaSize = AI->getAllocationSize(DL);
11980 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11981 Arg->getType()->isEmptyTy() || !AllocaSize ||
11982 DL.getTypeStoreSize(Ty: Arg->getType()) != *AllocaSize ||
11983 !DL.typeSizeEqualsStoreSize(Ty: Arg->getType()) ||
11984 ArgCopyElisionCandidates.count(Val: Arg)) {
11985 *Info = StaticAllocaInfo::Clobbered;
11986 continue;
11987 }
11988
11989 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11990 << '\n');
11991
11992 // Mark this alloca and store for argument copy elision.
11993 *Info = StaticAllocaInfo::Elidable;
11994 ArgCopyElisionCandidates.insert(KV: {Arg, {AI, SI}});
11995
11996 // Stop scanning if we've seen all arguments. This will happen early in -O0
11997 // builds, which is useful, because -O0 builds have large entry blocks and
11998 // many allocas.
11999 if (ArgCopyElisionCandidates.size() == NumArgs)
12000 break;
12001 }
12002}
12003
12004/// Try to elide argument copies from memory into a local alloca. Succeeds if
12005/// ArgVal is a load from a suitable fixed stack object.
12006static void tryToElideArgumentCopy(
12007 FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
12008 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
12009 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
12010 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
12011 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
12012 // Check if this is a load from a fixed stack object.
12013 auto *LNode = dyn_cast<LoadSDNode>(Val: ArgVals[0]);
12014 if (!LNode)
12015 return;
12016 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode());
12017 if (!FINode)
12018 return;
12019
12020 // Check that the fixed stack object is the right size and alignment.
12021 // Look at the alignment that the user wrote on the alloca instead of looking
12022 // at the stack object.
12023 auto ArgCopyIter = ArgCopyElisionCandidates.find(Val: &Arg);
12024 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
12025 const AllocaInst *AI = ArgCopyIter->second.first;
12026 int FixedIndex = FINode->getIndex();
12027 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
12028 int OldIndex = AllocaIndex;
12029 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
12030 if (MFI.getObjectSize(ObjectIdx: FixedIndex) != MFI.getObjectSize(ObjectIdx: OldIndex)) {
12031 LLVM_DEBUG(
12032 dbgs() << " argument copy elision failed due to bad fixed stack "
12033 "object size\n");
12034 return;
12035 }
12036 Align RequiredAlignment = AI->getAlign();
12037 if (MFI.getObjectAlign(ObjectIdx: FixedIndex) < RequiredAlignment) {
12038 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
12039 "greater than stack argument alignment ("
12040 << DebugStr(RequiredAlignment) << " vs "
12041 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
12042 return;
12043 }
12044
12045 // Perform the elision. Delete the old stack object and replace its only use
12046 // in the variable info map. Mark the stack object as mutable and aliased.
12047 LLVM_DEBUG({
12048 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
12049 << " Replacing frame index " << OldIndex << " with " << FixedIndex
12050 << '\n';
12051 });
12052 MFI.RemoveStackObject(ObjectIdx: OldIndex);
12053 MFI.setIsImmutableObjectIndex(ObjectIdx: FixedIndex, IsImmutable: false);
12054 MFI.setIsAliasedObjectIndex(ObjectIdx: FixedIndex, IsAliased: true);
12055 AllocaIndex = FixedIndex;
12056 ArgCopyElisionFrameIndexMap.insert(KV: {OldIndex, FixedIndex});
12057 for (SDValue ArgVal : ArgVals)
12058 Chains.push_back(Elt: ArgVal.getValue(R: 1));
12059
12060 // Avoid emitting code for the store implementing the copy.
12061 const StoreInst *SI = ArgCopyIter->second.second;
12062 ElidedArgCopyInstrs.insert(Ptr: SI);
12063
12064 // Check for uses of the argument again so that we can avoid exporting ArgVal
12065 // if it is't used by anything other than the store.
12066 for (const Value *U : Arg.users()) {
12067 if (U != SI) {
12068 ArgHasUses = true;
12069 break;
12070 }
12071 }
12072}
12073
12074void SelectionDAGISel::LowerArguments(const Function &F) {
12075 SelectionDAG &DAG = SDB->DAG;
12076 SDLoc dl = SDB->getCurSDLoc();
12077 const DataLayout &DL = DAG.getDataLayout();
12078 SmallVector<ISD::InputArg, 16> Ins;
12079
12080 // In Naked functions we aren't going to save any registers.
12081 if (F.hasFnAttribute(Kind: Attribute::Naked))
12082 return;
12083
12084 if (!FuncInfo->CanLowerReturn) {
12085 // Put in an sret pointer parameter before all the other parameters.
12086 MVT ValueVT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
12087
12088 ISD::ArgFlagsTy Flags;
12089 Flags.setSRet();
12090 MVT RegisterVT = TLI->getRegisterType(Context&: *DAG.getContext(), VT: ValueVT);
12091 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
12092 ISD::InputArg::NoArgIndex, 0);
12093 Ins.push_back(Elt: RetArg);
12094 }
12095
12096 // Look for stores of arguments to static allocas. Mark such arguments with a
12097 // flag to ask the target to give us the memory location of that argument if
12098 // available.
12099 ArgCopyElisionMapTy ArgCopyElisionCandidates;
12100 findArgumentCopyElisionCandidates(DL, FuncInfo: FuncInfo.get(),
12101 ArgCopyElisionCandidates);
12102
12103 // Set up the incoming argument description vector.
12104 for (const Argument &Arg : F.args()) {
12105 unsigned ArgNo = Arg.getArgNo();
12106 SmallVector<Type *, 4> Types;
12107 ComputeValueTypes(DL: DAG.getDataLayout(), Ty: Arg.getType(), Types);
12108 bool isArgValueUsed = !Arg.use_empty();
12109 Type *FinalType = Arg.getType();
12110 if (Arg.hasAttribute(Kind: Attribute::ByVal))
12111 FinalType = Arg.getParamByValType();
12112 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
12113 Ty: FinalType, CallConv: F.getCallingConv(), isVarArg: F.isVarArg(), DL);
12114 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
12115 ++Value) {
12116 Type *ArgTy = Types[Value];
12117 EVT VT = TLI->getValueType(DL, Ty: ArgTy);
12118 ISD::ArgFlagsTy Flags;
12119
12120 if (ArgTy->isPointerTy()) {
12121 Flags.setPointer();
12122 Flags.setPointerAddrSpace(cast<PointerType>(Val: ArgTy)->getAddressSpace());
12123 }
12124 if (Arg.hasAttribute(Kind: Attribute::ZExt))
12125 Flags.setZExt();
12126 if (Arg.hasAttribute(Kind: Attribute::SExt))
12127 Flags.setSExt();
12128 if (Arg.hasAttribute(Kind: Attribute::InReg)) {
12129 // If we are using vectorcall calling convention, a structure that is
12130 // passed InReg - is surely an HVA
12131 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
12132 isa<StructType>(Val: Arg.getType())) {
12133 // The first value of a structure is marked
12134 if (0 == Value)
12135 Flags.setHvaStart();
12136 Flags.setHva();
12137 }
12138 // Set InReg Flag
12139 Flags.setInReg();
12140 }
12141 if (Arg.hasAttribute(Kind: Attribute::StructRet))
12142 Flags.setSRet();
12143 if (Arg.hasAttribute(Kind: Attribute::SwiftSelf))
12144 Flags.setSwiftSelf();
12145 if (Arg.hasAttribute(Kind: Attribute::SwiftAsync))
12146 Flags.setSwiftAsync();
12147 if (Arg.hasAttribute(Kind: Attribute::SwiftError))
12148 Flags.setSwiftError();
12149 if (Arg.hasAttribute(Kind: Attribute::ByVal))
12150 Flags.setByVal();
12151 if (Arg.hasAttribute(Kind: Attribute::ByRef))
12152 Flags.setByRef();
12153 if (Arg.hasAttribute(Kind: Attribute::InAlloca)) {
12154 Flags.setInAlloca();
12155 // Set the byval flag for CCAssignFn callbacks that don't know about
12156 // inalloca. This way we can know how many bytes we should've allocated
12157 // and how many bytes a callee cleanup function will pop. If we port
12158 // inalloca to more targets, we'll have to add custom inalloca handling
12159 // in the various CC lowering callbacks.
12160 Flags.setByVal();
12161 }
12162 if (Arg.hasAttribute(Kind: Attribute::Preallocated)) {
12163 Flags.setPreallocated();
12164 // Set the byval flag for CCAssignFn callbacks that don't know about
12165 // preallocated. This way we can know how many bytes we should've
12166 // allocated and how many bytes a callee cleanup function will pop. If
12167 // we port preallocated to more targets, we'll have to add custom
12168 // preallocated handling in the various CC lowering callbacks.
12169 Flags.setByVal();
12170 }
12171
12172 // Certain targets (such as MIPS), may have a different ABI alignment
12173 // for a type depending on the context. Give the target a chance to
12174 // specify the alignment it wants.
12175 const Align OriginalAlignment(
12176 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
12177 Flags.setOrigAlign(OriginalAlignment);
12178
12179 Align MemAlign;
12180 Type *ArgMemTy = nullptr;
12181 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
12182 Flags.isByRef()) {
12183 if (!ArgMemTy)
12184 ArgMemTy = Arg.getPointeeInMemoryValueType();
12185
12186 uint64_t MemSize = DL.getTypeAllocSize(Ty: ArgMemTy);
12187
12188 // For in-memory arguments, size and alignment should be passed from FE.
12189 // BE will guess if this info is not there but there are cases it cannot
12190 // get right.
12191 if (auto ParamAlign = Arg.getParamStackAlign())
12192 MemAlign = *ParamAlign;
12193 else if ((ParamAlign = Arg.getParamAlign()))
12194 MemAlign = *ParamAlign;
12195 else
12196 MemAlign = TLI->getByValTypeAlignment(Ty: ArgMemTy, DL);
12197 if (Flags.isByRef())
12198 Flags.setByRefSize(MemSize);
12199 else
12200 Flags.setByValSize(MemSize);
12201 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
12202 MemAlign = *ParamAlign;
12203 } else {
12204 MemAlign = OriginalAlignment;
12205 }
12206 Flags.setMemAlign(MemAlign);
12207
12208 if (Arg.hasAttribute(Kind: Attribute::Nest))
12209 Flags.setNest();
12210 if (NeedsRegBlock)
12211 Flags.setInConsecutiveRegs();
12212 if (ArgCopyElisionCandidates.count(Val: &Arg))
12213 Flags.setCopyElisionCandidate();
12214 if (Arg.hasAttribute(Kind: Attribute::Returned))
12215 Flags.setReturned();
12216
12217 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
12218 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12219 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
12220 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12221 for (unsigned i = 0; i != NumRegs; ++i) {
12222 // For scalable vectors, use the minimum size; individual targets
12223 // are responsible for handling scalable vector arguments and
12224 // return values.
12225 ISD::InputArg MyFlags(
12226 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12227 i * RegisterVT.getStoreSize().getKnownMinValue());
12228 if (NumRegs > 1 && i == 0)
12229 MyFlags.Flags.setSplit();
12230 // if it isn't first piece, alignment must be 1
12231 else if (i > 0) {
12232 MyFlags.Flags.setOrigAlign(Align(1));
12233 if (i == NumRegs - 1)
12234 MyFlags.Flags.setSplitEnd();
12235 }
12236 Ins.push_back(Elt: MyFlags);
12237 }
12238 if (NeedsRegBlock && Value == NumValues - 1)
12239 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
12240 }
12241 }
12242
12243 // Call the target to set up the argument values.
12244 SmallVector<SDValue, 8> InVals;
12245 SDValue NewRoot = TLI->LowerFormalArguments(
12246 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
12247
12248 // Verify that the target's LowerFormalArguments behaved as expected.
12249 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
12250 "LowerFormalArguments didn't return a valid chain!");
12251 assert(InVals.size() == Ins.size() &&
12252 "LowerFormalArguments didn't emit the correct number of values!");
12253 assert(all_of(InVals, [](SDValue InVal) { return InVal.getNode(); }) &&
12254 "LowerFormalArguments emitted a null value!");
12255
12256 // Update the DAG with the new chain value resulting from argument lowering.
12257 DAG.setRoot(NewRoot);
12258
12259 // Set up the argument values.
12260 unsigned i = 0;
12261 if (!FuncInfo->CanLowerReturn) {
12262 // Create a virtual register for the sret pointer, and put in a copy
12263 // from the sret argument into it.
12264 MVT VT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
12265 MVT RegVT = TLI->getRegisterType(Context&: *CurDAG->getContext(), VT);
12266 std::optional<ISD::NodeType> AssertOp;
12267 SDValue ArgValue =
12268 getCopyFromParts(DAG, DL: dl, Parts: &InVals[0], NumParts: 1, PartVT: RegVT, ValueVT: VT, V: nullptr, InChain: NewRoot,
12269 CC: F.getCallingConv(), AssertOp);
12270
12271 MachineFunction& MF = SDB->DAG.getMachineFunction();
12272 MachineRegisterInfo& RegInfo = MF.getRegInfo();
12273 Register SRetReg =
12274 RegInfo.createVirtualRegister(RegClass: TLI->getRegClassFor(VT: RegVT));
12275 FuncInfo->DemoteRegister = SRetReg;
12276 NewRoot =
12277 SDB->DAG.getCopyToReg(Chain: NewRoot, dl: SDB->getCurSDLoc(), Reg: SRetReg, N: ArgValue);
12278 DAG.setRoot(NewRoot);
12279
12280 // i indexes lowered arguments. Bump it past the hidden sret argument.
12281 ++i;
12282 }
12283
12284 SmallVector<SDValue, 4> Chains;
12285 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12286 for (const Argument &Arg : F.args()) {
12287 SmallVector<SDValue, 4> ArgValues;
12288 SmallVector<EVT, 4> ValueVTs;
12289 ComputeValueVTs(TLI: *TLI, DL: DAG.getDataLayout(), Ty: Arg.getType(), ValueVTs);
12290 unsigned NumValues = ValueVTs.size();
12291 if (NumValues == 0)
12292 continue;
12293
12294 bool ArgHasUses = !Arg.use_empty();
12295
12296 // Elide the copying store if the target loaded this argument from a
12297 // suitable fixed stack object.
12298 if (Ins[i].Flags.isCopyElisionCandidate()) {
12299 unsigned NumParts = 0;
12300 for (EVT VT : ValueVTs)
12301 NumParts += TLI->getNumRegistersForCallingConv(Context&: *CurDAG->getContext(),
12302 CC: F.getCallingConv(), VT);
12303
12304 tryToElideArgumentCopy(FuncInfo&: *FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
12305 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
12306 ArgVals: ArrayRef(&InVals[i], NumParts), ArgHasUses);
12307 }
12308
12309 // If this argument is unused then remember its value. It is used to generate
12310 // debugging information.
12311 bool isSwiftErrorArg =
12312 TLI->supportSwiftError() &&
12313 Arg.hasAttribute(Kind: Attribute::SwiftError);
12314 if (!ArgHasUses && !isSwiftErrorArg) {
12315 SDB->setUnusedArgValue(V: &Arg, NewN: InVals[i]);
12316
12317 // Also remember any frame index for use in FastISel.
12318 if (FrameIndexSDNode *FI =
12319 dyn_cast<FrameIndexSDNode>(Val: InVals[i].getNode()))
12320 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12321 }
12322
12323 for (unsigned Val = 0; Val != NumValues; ++Val) {
12324 EVT VT = ValueVTs[Val];
12325 MVT PartVT = TLI->getRegisterTypeForCallingConv(Context&: *CurDAG->getContext(),
12326 CC: F.getCallingConv(), VT);
12327 unsigned NumParts = TLI->getNumRegistersForCallingConv(
12328 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
12329
12330 // Even an apparent 'unused' swifterror argument needs to be returned. So
12331 // we do generate a copy for it that can be used on return from the
12332 // function.
12333 if (ArgHasUses || isSwiftErrorArg) {
12334 std::optional<ISD::NodeType> AssertOp;
12335 if (Arg.hasAttribute(Kind: Attribute::SExt))
12336 AssertOp = ISD::AssertSext;
12337 else if (Arg.hasAttribute(Kind: Attribute::ZExt))
12338 AssertOp = ISD::AssertZext;
12339
12340 SDValue OutVal =
12341 getCopyFromParts(DAG, DL: dl, Parts: &InVals[i], NumParts, PartVT, ValueVT: VT, V: nullptr,
12342 InChain: NewRoot, CC: F.getCallingConv(), AssertOp);
12343
12344 FPClassTest NoFPClass = Arg.getNoFPClass();
12345 if (NoFPClass != fcNone) {
12346 SDValue SDNoFPClass = DAG.getTargetConstant(
12347 Val: static_cast<uint64_t>(NoFPClass), DL: dl, VT: MVT::i32);
12348 OutVal = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: OutVal.getValueType(),
12349 N1: OutVal, N2: SDNoFPClass);
12350 }
12351 ArgValues.push_back(Elt: OutVal);
12352 }
12353
12354 i += NumParts;
12355 }
12356
12357 // We don't need to do anything else for unused arguments.
12358 if (ArgValues.empty())
12359 continue;
12360
12361 // Note down frame index.
12362 if (FrameIndexSDNode *FI =
12363 dyn_cast<FrameIndexSDNode>(Val: ArgValues[0].getNode()))
12364 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12365
12366 SDValue Res = DAG.getMergeValues(Ops: ArrayRef(ArgValues.data(), NumValues),
12367 dl: SDB->getCurSDLoc());
12368
12369 SDB->setValue(V: &Arg, NewN: Res);
12370 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
12371 // We want to associate the argument with the frame index, among
12372 // involved operands, that correspond to the lowest address. The
12373 // getCopyFromParts function, called earlier, is swapping the order of
12374 // the operands to BUILD_PAIR depending on endianness. The result of
12375 // that swapping is that the least significant bits of the argument will
12376 // be in the first operand of the BUILD_PAIR node, and the most
12377 // significant bits will be in the second operand.
12378 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
12379 if (LoadSDNode *LNode =
12380 dyn_cast<LoadSDNode>(Val: Res.getOperand(i: LowAddressOp).getNode()))
12381 if (FrameIndexSDNode *FI =
12382 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
12383 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
12384 }
12385
12386 // Analyses past this point are naive and don't expect an assertion.
12387 if (Res.getOpcode() == ISD::AssertZext)
12388 Res = Res.getOperand(i: 0);
12389
12390 // Update the SwiftErrorVRegDefMap.
12391 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
12392 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
12393 if (Reg.isVirtual())
12394 SwiftError->setCurrentVReg(MBB: FuncInfo->MBB, SwiftError->getFunctionArg(),
12395 Reg);
12396 }
12397
12398 // If this argument is live outside of the entry block, insert a copy from
12399 // wherever we got it to the vreg that other BB's will reference it as.
12400 if (Res.getOpcode() == ISD::CopyFromReg) {
12401 // If we can, though, try to skip creating an unnecessary vreg.
12402 // FIXME: This isn't very clean... it would be nice to make this more
12403 // general.
12404 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
12405 if (Reg.isVirtual()) {
12406 FuncInfo->ValueMap[&Arg] = Reg;
12407 continue;
12408 }
12409 }
12410 if (!isOnlyUsedInEntryBlock(A: &Arg, FastISel: TM.Options.EnableFastISel)) {
12411 FuncInfo->InitializeRegForValue(V: &Arg);
12412 SDB->CopyToExportRegsIfNeeded(V: &Arg);
12413 }
12414 }
12415
12416 if (!Chains.empty()) {
12417 Chains.push_back(Elt: NewRoot);
12418 NewRoot = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
12419 }
12420
12421 DAG.setRoot(NewRoot);
12422
12423 assert(i == InVals.size() && "Argument register count mismatch!");
12424
12425 // If any argument copy elisions occurred and we have debug info, update the
12426 // stale frame indices used in the dbg.declare variable info table.
12427 if (!ArgCopyElisionFrameIndexMap.empty()) {
12428 for (MachineFunction::VariableDbgInfo &VI :
12429 MF->getInStackSlotVariableDbgInfo()) {
12430 auto I = ArgCopyElisionFrameIndexMap.find(Val: VI.getStackSlot());
12431 if (I != ArgCopyElisionFrameIndexMap.end())
12432 VI.updateStackSlot(NewSlot: I->second);
12433 }
12434 }
12435
12436 // Finally, if the target has anything special to do, allow it to do so.
12437 emitFunctionEntryCode();
12438}
12439
12440/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
12441/// ensure constants are generated when needed. Remember the virtual registers
12442/// that need to be added to the Machine PHI nodes as input. We cannot just
12443/// directly add them, because expansion might result in multiple MBB's for one
12444/// BB. As such, the start of the BB might correspond to a different MBB than
12445/// the end.
12446void
12447SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
12448 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12449
12450 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12451
12452 // Check PHI nodes in successors that expect a value to be available from this
12453 // block.
12454 for (const BasicBlock *SuccBB : successors(I: LLVMBB->getTerminator())) {
12455 if (!isa<PHINode>(Val: SuccBB->begin())) continue;
12456 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(BB: SuccBB);
12457
12458 // If this terminator has multiple identical successors (common for
12459 // switches), only handle each succ once.
12460 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
12461 continue;
12462
12463 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
12464
12465 // At this point we know that there is a 1-1 correspondence between LLVM PHI
12466 // nodes and Machine PHI nodes, but the incoming operands have not been
12467 // emitted yet.
12468 for (const PHINode &PN : SuccBB->phis()) {
12469 // Ignore dead phi's.
12470 if (PN.use_empty())
12471 continue;
12472
12473 // Skip empty types
12474 if (PN.getType()->isEmptyTy())
12475 continue;
12476
12477 Register Reg;
12478 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
12479
12480 if (const auto *C = dyn_cast<Constant>(Val: PHIOp)) {
12481 Register &RegOut = ConstantsOut[C];
12482 if (!RegOut) {
12483 RegOut = FuncInfo.CreateRegs(V: &PN);
12484 // We need to zero/sign extend ConstantInt phi operands to match
12485 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12486 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12487 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
12488 ExtendType = TLI.signExtendConstant(C: CI) ? ISD::SIGN_EXTEND
12489 : ISD::ZERO_EXTEND;
12490 CopyValueToVirtualRegister(V: C, Reg: RegOut, ExtendType);
12491 }
12492 Reg = RegOut;
12493 } else {
12494 auto I = FuncInfo.ValueMap.find(Val: PHIOp);
12495 if (I != FuncInfo.ValueMap.end())
12496 Reg = I->second;
12497 else {
12498 assert(isa<AllocaInst>(PHIOp) &&
12499 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12500 "Didn't codegen value into a register!??");
12501 Reg = FuncInfo.CreateRegs(V: &PN);
12502 CopyValueToVirtualRegister(V: PHIOp, Reg);
12503 }
12504 }
12505
12506 // Remember that this register needs to added to the machine PHI node as
12507 // the input for this MBB.
12508 SmallVector<EVT, 4> ValueVTs;
12509 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: PN.getType(), ValueVTs);
12510 for (EVT VT : ValueVTs) {
12511 const unsigned NumRegisters = TLI.getNumRegisters(Context&: *DAG.getContext(), VT);
12512 for (unsigned i = 0; i != NumRegisters; ++i)
12513 FuncInfo.PHINodesToUpdate.emplace_back(args: &*MBBI++, args: Reg + i);
12514 Reg += NumRegisters;
12515 }
12516 }
12517 }
12518
12519 ConstantsOut.clear();
12520}
12521
12522MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12523 MachineFunction::iterator I(MBB);
12524 if (++I == FuncInfo.MF->end())
12525 return nullptr;
12526 return &*I;
12527}
12528
12529/// During lowering new call nodes can be created (such as memset, etc.).
12530/// Those will become new roots of the current DAG, but complications arise
12531/// when they are tail calls. In such cases, the call lowering will update
12532/// the root, but the builder still needs to know that a tail call has been
12533/// lowered in order to avoid generating an additional return.
12534void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12535 // If the node is null, we do have a tail call.
12536 if (MaybeTC.getNode() != nullptr)
12537 DAG.setRoot(MaybeTC);
12538 else
12539 HasTailCall = true;
12540}
12541
12542void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12543 MachineBasicBlock *SwitchMBB,
12544 MachineBasicBlock *DefaultMBB) {
12545 MachineFunction *CurMF = FuncInfo.MF;
12546 MachineBasicBlock *NextMBB = nullptr;
12547 MachineFunction::iterator BBI(W.MBB);
12548 if (++BBI != FuncInfo.MF->end())
12549 NextMBB = &*BBI;
12550
12551 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12552
12553 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12554
12555 if (Size == 2 && W.MBB == SwitchMBB) {
12556 // If any two of the cases has the same destination, and if one value
12557 // is the same as the other, but has one bit unset that the other has set,
12558 // use bit manipulation to do two compares at once. For example:
12559 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12560 // TODO: This could be extended to merge any 2 cases in switches with 3
12561 // cases.
12562 // TODO: Handle cases where W.CaseBB != SwitchBB.
12563 CaseCluster &Small = *W.FirstCluster;
12564 CaseCluster &Big = *W.LastCluster;
12565
12566 if (Small.Low == Small.High && Big.Low == Big.High &&
12567 Small.MBB == Big.MBB) {
12568 const APInt &SmallValue = Small.Low->getValue();
12569 const APInt &BigValue = Big.Low->getValue();
12570
12571 // Check that there is only one bit different.
12572 APInt CommonBit = BigValue ^ SmallValue;
12573 if (CommonBit.isPowerOf2()) {
12574 SDValue CondLHS = getValue(V: Cond);
12575 EVT VT = CondLHS.getValueType();
12576 SDLoc DL = getCurSDLoc();
12577
12578 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: CondLHS,
12579 N2: DAG.getConstant(Val: CommonBit, DL, VT));
12580 SDValue Cond = DAG.getSetCC(
12581 DL, VT: MVT::i1, LHS: Or, RHS: DAG.getConstant(Val: BigValue | SmallValue, DL, VT),
12582 Cond: ISD::SETEQ);
12583
12584 // Update successor info.
12585 // Both Small and Big will jump to Small.BB, so we sum up the
12586 // probabilities.
12587 addSuccessorWithProb(Src: SwitchMBB, Dst: Small.MBB, Prob: Small.Prob + Big.Prob);
12588 if (BPI)
12589 addSuccessorWithProb(
12590 Src: SwitchMBB, Dst: DefaultMBB,
12591 // The default destination is the first successor in IR.
12592 Prob: BPI->getEdgeProbability(Src: SwitchMBB->getBasicBlock(), IndexInSuccessors: (unsigned)0));
12593 else
12594 addSuccessorWithProb(Src: SwitchMBB, Dst: DefaultMBB);
12595
12596 // Insert the true branch.
12597 SDValue BrCond =
12598 DAG.getNode(Opcode: ISD::BRCOND, DL, VT: MVT::Other, N1: getControlRoot(), N2: Cond,
12599 N3: DAG.getBasicBlock(MBB: Small.MBB));
12600 // Insert the false branch.
12601 BrCond = DAG.getNode(Opcode: ISD::BR, DL, VT: MVT::Other, N1: BrCond,
12602 N2: DAG.getBasicBlock(MBB: DefaultMBB));
12603
12604 DAG.setRoot(BrCond);
12605 return;
12606 }
12607 }
12608 }
12609
12610 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12611 // Here, we order cases by probability so the most likely case will be
12612 // checked first. However, two clusters can have the same probability in
12613 // which case their relative ordering is non-deterministic. So we use Low
12614 // as a tie-breaker as clusters are guaranteed to never overlap.
12615 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
12616 Comp: [](const CaseCluster &a, const CaseCluster &b) {
12617 return a.Prob != b.Prob ?
12618 a.Prob > b.Prob :
12619 a.Low->getValue().slt(RHS: b.Low->getValue());
12620 });
12621
12622 // Rearrange the case blocks so that the last one falls through if possible
12623 // without changing the order of probabilities.
12624 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12625 --I;
12626 if (I->Prob > W.LastCluster->Prob)
12627 break;
12628 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12629 std::swap(a&: *I, b&: *W.LastCluster);
12630 break;
12631 }
12632 }
12633 }
12634
12635 // Compute total probability.
12636 BranchProbability DefaultProb = W.DefaultProb;
12637 BranchProbability UnhandledProbs = DefaultProb;
12638 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12639 UnhandledProbs += I->Prob;
12640
12641 MachineBasicBlock *CurMBB = W.MBB;
12642 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12643 bool FallthroughUnreachable = false;
12644 MachineBasicBlock *Fallthrough;
12645 if (I == W.LastCluster) {
12646 // For the last cluster, fall through to the default destination.
12647 Fallthrough = DefaultMBB;
12648 FallthroughUnreachable = isa<UnreachableInst>(
12649 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
12650 } else {
12651 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
12652 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
12653 // Put Cond in a virtual register to make it available from the new blocks.
12654 ExportFromCurrentBlock(V: Cond);
12655 }
12656 UnhandledProbs -= I->Prob;
12657
12658 switch (I->Kind) {
12659 case CC_JumpTable: {
12660 // FIXME: Optimize away range check based on pivot comparisons.
12661 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12662 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12663
12664 // The jump block hasn't been inserted yet; insert it here.
12665 MachineBasicBlock *JumpMBB = JT->MBB;
12666 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
12667
12668 auto JumpProb = I->Prob;
12669 auto FallthroughProb = UnhandledProbs;
12670
12671 // If the default statement is a target of the jump table, we evenly
12672 // distribute the default probability to successors of CurMBB. Also
12673 // update the probability on the edge from JumpMBB to Fallthrough.
12674 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12675 SE = JumpMBB->succ_end();
12676 SI != SE; ++SI) {
12677 if (*SI == DefaultMBB) {
12678 JumpProb += DefaultProb / 2;
12679 FallthroughProb -= DefaultProb / 2;
12680 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
12681 JumpMBB->normalizeSuccProbs();
12682 break;
12683 }
12684 }
12685
12686 // If the default clause is unreachable, propagate that knowledge into
12687 // JTH->FallthroughUnreachable which will use it to suppress the range
12688 // check.
12689 //
12690 // However, don't do this if we're doing branch target enforcement,
12691 // because a table branch _without_ a range check can be a tempting JOP
12692 // gadget - out-of-bounds inputs that are impossible in correct
12693 // execution become possible again if an attacker can influence the
12694 // control flow. So if an attacker doesn't already have a BTI bypass
12695 // available, we don't want them to be able to get one out of this
12696 // table branch.
12697 if (FallthroughUnreachable) {
12698 Function &CurFunc = CurMF->getFunction();
12699 if (!CurFunc.hasFnAttribute(Kind: "branch-target-enforcement"))
12700 JTH->FallthroughUnreachable = true;
12701 }
12702
12703 if (!JTH->FallthroughUnreachable)
12704 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
12705 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
12706 CurMBB->normalizeSuccProbs();
12707
12708 // The jump table header will be inserted in our current block, do the
12709 // range check, and fall through to our fallthrough block.
12710 JTH->HeaderBB = CurMBB;
12711 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12712
12713 // If we're in the right place, emit the jump table header right now.
12714 if (CurMBB == SwitchMBB) {
12715 visitJumpTableHeader(JT&: *JT, JTH&: *JTH, SwitchBB: SwitchMBB);
12716 JTH->Emitted = true;
12717 }
12718 break;
12719 }
12720 case CC_BitTests: {
12721 // FIXME: Optimize away range check based on pivot comparisons.
12722 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12723
12724 // The bit test blocks haven't been inserted yet; insert them here.
12725 for (BitTestCase &BTC : BTB->Cases)
12726 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
12727
12728 // Fill in fields of the BitTestBlock.
12729 BTB->Parent = CurMBB;
12730 BTB->Default = Fallthrough;
12731
12732 BTB->DefaultProb = UnhandledProbs;
12733 // If the cases in bit test don't form a contiguous range, we evenly
12734 // distribute the probability on the edge to Fallthrough to two
12735 // successors of CurMBB.
12736 if (!BTB->ContiguousRange) {
12737 BTB->Prob += DefaultProb / 2;
12738 BTB->DefaultProb -= DefaultProb / 2;
12739 }
12740
12741 if (FallthroughUnreachable)
12742 BTB->FallthroughUnreachable = true;
12743
12744 // If we're in the right place, emit the bit test header right now.
12745 if (CurMBB == SwitchMBB) {
12746 visitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
12747 BTB->Emitted = true;
12748 }
12749 break;
12750 }
12751 case CC_Range: {
12752 const Value *RHS, *LHS, *MHS;
12753 ISD::CondCode CC;
12754 if (I->Low == I->High) {
12755 // Check Cond == I->Low.
12756 CC = ISD::SETEQ;
12757 LHS = Cond;
12758 RHS=I->Low;
12759 MHS = nullptr;
12760 } else {
12761 // Check I->Low <= Cond <= I->High.
12762 CC = ISD::SETLE;
12763 LHS = I->Low;
12764 MHS = Cond;
12765 RHS = I->High;
12766 }
12767
12768 // If Fallthrough is unreachable, fold away the comparison.
12769 if (FallthroughUnreachable)
12770 CC = ISD::SETTRUE;
12771
12772 // The false probability is the sum of all unhandled cases.
12773 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12774 getCurSDLoc(), I->Prob, UnhandledProbs);
12775
12776 if (CurMBB == SwitchMBB)
12777 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12778 else
12779 SL->SwitchCases.push_back(x: CB);
12780
12781 break;
12782 }
12783 }
12784 CurMBB = Fallthrough;
12785 }
12786}
12787
12788void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12789 const SwitchWorkListItem &W,
12790 Value *Cond,
12791 MachineBasicBlock *SwitchMBB) {
12792 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12793 "Clusters not sorted?");
12794 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12795
12796 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12797 SL->computeSplitWorkItemInfo(W);
12798
12799 // Use the first element on the right as pivot since we will make less-than
12800 // comparisons against it.
12801 CaseClusterIt PivotCluster = FirstRight;
12802 assert(PivotCluster > W.FirstCluster);
12803 assert(PivotCluster <= W.LastCluster);
12804
12805 CaseClusterIt FirstLeft = W.FirstCluster;
12806 CaseClusterIt LastRight = W.LastCluster;
12807
12808 const ConstantInt *Pivot = PivotCluster->Low;
12809
12810 // New blocks will be inserted immediately after the current one.
12811 MachineFunction::iterator BBI(W.MBB);
12812 ++BBI;
12813
12814 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12815 // we can branch to its destination directly if it's squeezed exactly in
12816 // between the known lower bound and Pivot - 1.
12817 MachineBasicBlock *LeftMBB;
12818 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12819 FirstLeft->Low == W.GE &&
12820 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12821 LeftMBB = FirstLeft->MBB;
12822 } else {
12823 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12824 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
12825 WorkList.push_back(
12826 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
12827 // Put Cond in a virtual register to make it available from the new blocks.
12828 ExportFromCurrentBlock(V: Cond);
12829 }
12830
12831 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12832 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12833 // directly if RHS.High equals the current upper bound.
12834 MachineBasicBlock *RightMBB;
12835 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12836 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12837 RightMBB = FirstRight->MBB;
12838 } else {
12839 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12840 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
12841 WorkList.push_back(
12842 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
12843 // Put Cond in a virtual register to make it available from the new blocks.
12844 ExportFromCurrentBlock(V: Cond);
12845 }
12846
12847 // Create the CaseBlock record that will be used to lower the branch.
12848 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12849 getCurSDLoc(), LeftProb, RightProb);
12850
12851 if (W.MBB == SwitchMBB)
12852 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12853 else
12854 SL->SwitchCases.push_back(x: CB);
12855}
12856
12857// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12858// from the swith statement.
12859static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
12860 BranchProbability PeeledCaseProb) {
12861 if (PeeledCaseProb == BranchProbability::getOne())
12862 return BranchProbability::getZero();
12863 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12864
12865 uint32_t Numerator = CaseProb.getNumerator();
12866 uint32_t Denominator = SwitchProb.scale(Num: CaseProb.getDenominator());
12867 return BranchProbability(Numerator, std::max(a: Numerator, b: Denominator));
12868}
12869
12870// Try to peel the top probability case if it exceeds the threshold.
12871// Return current MachineBasicBlock for the switch statement if the peeling
12872// does not occur.
12873// If the peeling is performed, return the newly created MachineBasicBlock
12874// for the peeled switch statement. Also update Clusters to remove the peeled
12875// case. PeeledCaseProb is the BranchProbability for the peeled case.
12876MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12877 const SwitchInst &SI, CaseClusterVector &Clusters,
12878 BranchProbability &PeeledCaseProb) {
12879 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12880 // Don't perform if there is only one cluster or optimizing for size.
12881 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12882 TM.getOptLevel() == CodeGenOptLevel::None ||
12883 SwitchMBB->getParent()->getFunction().hasMinSize())
12884 return SwitchMBB;
12885
12886 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
12887 unsigned PeeledCaseIndex = 0;
12888 bool SwitchPeeled = false;
12889 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12890 CaseCluster &CC = Clusters[Index];
12891 if (CC.Prob < TopCaseProb)
12892 continue;
12893 TopCaseProb = CC.Prob;
12894 PeeledCaseIndex = Index;
12895 SwitchPeeled = true;
12896 }
12897 if (!SwitchPeeled)
12898 return SwitchMBB;
12899
12900 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12901 << TopCaseProb << "\n");
12902
12903 // Record the MBB for the peeled switch statement.
12904 MachineFunction::iterator BBI(SwitchMBB);
12905 ++BBI;
12906 MachineBasicBlock *PeeledSwitchMBB =
12907 FuncInfo.MF->CreateMachineBasicBlock(BB: SwitchMBB->getBasicBlock());
12908 FuncInfo.MF->insert(MBBI: BBI, MBB: PeeledSwitchMBB);
12909
12910 ExportFromCurrentBlock(V: SI.getCondition());
12911 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12912 SwitchWorkListItem W = {.MBB: SwitchMBB, .FirstCluster: PeeledCaseIt, .LastCluster: PeeledCaseIt,
12913 .GE: nullptr, .LT: nullptr, .DefaultProb: TopCaseProb.getCompl()};
12914 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB: PeeledSwitchMBB);
12915
12916 Clusters.erase(position: PeeledCaseIt);
12917 for (CaseCluster &CC : Clusters) {
12918 LLVM_DEBUG(
12919 dbgs() << "Scale the probablity for one cluster, before scaling: "
12920 << CC.Prob << "\n");
12921 CC.Prob = scaleCaseProbality(CaseProb: CC.Prob, PeeledCaseProb: TopCaseProb);
12922 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12923 }
12924 PeeledCaseProb = TopCaseProb;
12925 return PeeledSwitchMBB;
12926}
12927
12928void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12929 // Extract cases from the switch.
12930 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12931 CaseClusterVector Clusters;
12932 Clusters.reserve(n: SI.getNumCases());
12933 for (auto I : SI.cases()) {
12934 MachineBasicBlock *Succ = FuncInfo.getMBB(BB: I.getCaseSuccessor());
12935 const ConstantInt *CaseVal = I.getCaseValue();
12936 BranchProbability Prob =
12937 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
12938 : BranchProbability(1, SI.getNumCases() + 1);
12939 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
12940 }
12941
12942 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(BB: SI.getDefaultDest());
12943
12944 // Cluster adjacent cases with the same destination. We do this at all
12945 // optimization levels because it's cheap to do and will make codegen faster
12946 // if there are many clusters.
12947 sortAndRangeify(Clusters);
12948
12949 // The branch probablity of the peeled case.
12950 BranchProbability PeeledCaseProb = BranchProbability::getZero();
12951 MachineBasicBlock *PeeledSwitchMBB =
12952 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12953
12954 // If there is only the default destination, jump there directly.
12955 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12956 if (Clusters.empty()) {
12957 assert(PeeledSwitchMBB == SwitchMBB);
12958 SwitchMBB->addSuccessor(Succ: DefaultMBB);
12959 if (DefaultMBB != NextBlock(MBB: SwitchMBB)) {
12960 DAG.setRoot(DAG.getNode(Opcode: ISD::BR, DL: getCurSDLoc(), VT: MVT::Other,
12961 N1: getControlRoot(), N2: DAG.getBasicBlock(MBB: DefaultMBB)));
12962 }
12963 return;
12964 }
12965
12966 SL->findJumpTables(Clusters, SI: &SI, SL: getCurSDLoc(), DefaultMBB, PSI: DAG.getPSI(),
12967 BFI: DAG.getBFI());
12968 SL->findBitTestClusters(Clusters, SI: &SI);
12969
12970 LLVM_DEBUG({
12971 dbgs() << "Case clusters: ";
12972 for (const CaseCluster &C : Clusters) {
12973 if (C.Kind == CC_JumpTable)
12974 dbgs() << "JT:";
12975 if (C.Kind == CC_BitTests)
12976 dbgs() << "BT:";
12977
12978 C.Low->getValue().print(dbgs(), true);
12979 if (C.Low != C.High) {
12980 dbgs() << '-';
12981 C.High->getValue().print(dbgs(), true);
12982 }
12983 dbgs() << ' ';
12984 }
12985 dbgs() << '\n';
12986 });
12987
12988 assert(!Clusters.empty());
12989 SwitchWorkList WorkList;
12990 CaseClusterIt First = Clusters.begin();
12991 CaseClusterIt Last = Clusters.end() - 1;
12992 auto DefaultProb = getEdgeProbability(Src: PeeledSwitchMBB, Dst: DefaultMBB);
12993 // Scale the branchprobability for DefaultMBB if the peel occurs and
12994 // DefaultMBB is not replaced.
12995 if (PeeledCaseProb != BranchProbability::getZero() &&
12996 DefaultMBB == FuncInfo.getMBB(BB: SI.getDefaultDest()))
12997 DefaultProb = scaleCaseProbality(CaseProb: DefaultProb, PeeledCaseProb);
12998 WorkList.push_back(
12999 Elt: {.MBB: PeeledSwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
13000
13001 while (!WorkList.empty()) {
13002 SwitchWorkListItem W = WorkList.pop_back_val();
13003 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
13004
13005 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
13006 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
13007 // For optimized builds, lower large range as a balanced binary tree.
13008 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB);
13009 continue;
13010 }
13011
13012 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB);
13013 }
13014}
13015
13016void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
13017 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13018 auto DL = getCurSDLoc();
13019 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13020 setValue(V: &I, NewN: DAG.getStepVector(DL, ResVT: ResultVT));
13021}
13022
13023void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
13024 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13025 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13026
13027 SDLoc DL = getCurSDLoc();
13028 SDValue V = getValue(V: I.getOperand(i_nocapture: 0));
13029 assert(VT == V.getValueType() && "Malformed vector.reverse!");
13030
13031 if (VT.isScalableVector()) {
13032 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V));
13033 return;
13034 }
13035
13036 // Use VECTOR_SHUFFLE for the fixed-length vector
13037 // to maintain existing behavior.
13038 SmallVector<int, 8> Mask;
13039 unsigned NumElts = VT.getVectorMinNumElements();
13040 for (unsigned i = 0; i != NumElts; ++i)
13041 Mask.push_back(Elt: NumElts - 1 - i);
13042
13043 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V, N2: DAG.getUNDEF(VT), Mask));
13044}
13045
13046void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
13047 unsigned Factor) {
13048 auto DL = getCurSDLoc();
13049 SDValue InVec = getValue(V: I.getOperand(i_nocapture: 0));
13050
13051 SmallVector<EVT, 4> ValueVTs;
13052 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
13053 ValueVTs);
13054
13055 EVT OutVT = ValueVTs[0];
13056 unsigned OutNumElts = OutVT.getVectorMinNumElements();
13057
13058 SmallVector<SDValue, 4> SubVecs(Factor);
13059 for (unsigned i = 0; i != Factor; ++i) {
13060 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
13061 SubVecs[i] = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OutVT, N1: InVec,
13062 N2: DAG.getVectorIdxConstant(Val: OutNumElts * i, DL));
13063 }
13064
13065 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
13066 // from existing legalisation and combines.
13067 if (OutVT.isFixedLengthVector() && Factor == 2) {
13068 SDValue Even = DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: SubVecs[0], N2: SubVecs[1],
13069 Mask: createStrideMask(Start: 0, Stride: 2, VF: OutNumElts));
13070 SDValue Odd = DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: SubVecs[0], N2: SubVecs[1],
13071 Mask: createStrideMask(Start: 1, Stride: 2, VF: OutNumElts));
13072 SDValue Res = DAG.getMergeValues(Ops: {Even, Odd}, dl: getCurSDLoc());
13073 setValue(V: &I, NewN: Res);
13074 return;
13075 }
13076
13077 SDValue Res = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
13078 VTList: DAG.getVTList(VTs: ValueVTs), Ops: SubVecs);
13079 setValue(V: &I, NewN: Res);
13080}
13081
13082void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
13083 unsigned Factor) {
13084 auto DL = getCurSDLoc();
13085 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13086 EVT InVT = getValue(V: I.getOperand(i_nocapture: 0)).getValueType();
13087 EVT OutVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13088
13089 SmallVector<SDValue, 8> InVecs(Factor);
13090 for (unsigned i = 0; i < Factor; ++i) {
13091 InVecs[i] = getValue(V: I.getOperand(i_nocapture: i));
13092 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
13093 "Expected VTs to be the same");
13094 }
13095
13096 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
13097 // from existing legalisation and combines.
13098 if (OutVT.isFixedLengthVector() && Factor == 2) {
13099 unsigned NumElts = InVT.getVectorMinNumElements();
13100 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: InVecs);
13101 setValue(V: &I, NewN: DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: V, N2: DAG.getUNDEF(VT: OutVT),
13102 Mask: createInterleaveMask(VF: NumElts, NumVecs: 2)));
13103 return;
13104 }
13105
13106 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
13107 SDValue Res =
13108 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, VTList: DAG.getVTList(VTs: ValueVTs), Ops: InVecs);
13109
13110 SmallVector<SDValue, 8> Results(Factor);
13111 for (unsigned i = 0; i < Factor; ++i)
13112 Results[i] = Res.getValue(R: i);
13113
13114 Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: Results);
13115 setValue(V: &I, NewN: Res);
13116}
13117
13118void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
13119 SmallVector<EVT, 4> ValueVTs;
13120 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
13121 ValueVTs);
13122 unsigned NumValues = ValueVTs.size();
13123 if (NumValues == 0) return;
13124
13125 SmallVector<SDValue, 4> Values(NumValues);
13126 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
13127
13128 for (unsigned i = 0; i != NumValues; ++i)
13129 Values[i] = DAG.getNode(Opcode: ISD::FREEZE, DL: getCurSDLoc(), VT: ValueVTs[i],
13130 Operand: SDValue(Op.getNode(), Op.getResNo() + i));
13131
13132 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
13133 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
13134}
13135
13136void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
13137 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13138 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
13139
13140 SDLoc DL = getCurSDLoc();
13141 SDValue V1 = getValue(V: I.getOperand(i_nocapture: 0));
13142 SDValue V2 = getValue(V: I.getOperand(i_nocapture: 1));
13143 const bool IsLeft = I.getIntrinsicID() == Intrinsic::vector_splice_left;
13144
13145 // VECTOR_SHUFFLE doesn't support a scalable or non-constant mask.
13146 if (VT.isScalableVector() || !isa<ConstantInt>(Val: I.getOperand(i_nocapture: 2))) {
13147 SDValue Offset = DAG.getZExtOrTrunc(
13148 Op: getValue(V: I.getOperand(i_nocapture: 2)), DL, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
13149 setValue(V: &I, NewN: DAG.getNode(Opcode: IsLeft ? ISD::VECTOR_SPLICE_LEFT
13150 : ISD::VECTOR_SPLICE_RIGHT,
13151 DL, VT, N1: V1, N2: V2, N3: Offset));
13152 return;
13153 }
13154 uint64_t Imm = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->getZExtValue();
13155
13156 unsigned NumElts = VT.getVectorNumElements();
13157
13158 uint64_t Idx = IsLeft ? Imm : NumElts - Imm;
13159
13160 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
13161 SmallVector<int, 8> Mask;
13162 for (unsigned i = 0; i < NumElts; ++i)
13163 Mask.push_back(Elt: Idx + i);
13164 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask));
13165}
13166
13167// Consider the following MIR after SelectionDAG, which produces output in
13168// phyregs in the first case or virtregs in the second case.
13169//
13170// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
13171// %5:gr32 = COPY $ebx
13172// %6:gr32 = COPY $edx
13173// %1:gr32 = COPY %6:gr32
13174// %0:gr32 = COPY %5:gr32
13175//
13176// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
13177// %1:gr32 = COPY %6:gr32
13178// %0:gr32 = COPY %5:gr32
13179//
13180// Given %0, we'd like to return $ebx in the first case and %5 in the second.
13181// Given %1, we'd like to return $edx in the first case and %6 in the second.
13182//
13183// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
13184// to a single virtreg (such as %0). The remaining outputs monotonically
13185// increase in virtreg number from there. If a callbr has no outputs, then it
13186// should not have a corresponding callbr landingpad; in fact, the callbr
13187// landingpad would not even be able to refer to such a callbr.
13188static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
13189 MachineInstr *MI = MRI.def_begin(RegNo: Reg)->getParent();
13190 // There is definitely at least one copy.
13191 assert(MI->getOpcode() == TargetOpcode::COPY &&
13192 "start of copy chain MUST be COPY");
13193 Reg = MI->getOperand(i: 1).getReg();
13194
13195 // If the copied register in the first copy must be virtual.
13196 assert(Reg.isVirtual() && "expected COPY of virtual register");
13197 MI = MRI.def_begin(RegNo: Reg)->getParent();
13198
13199 // There may be an optional second copy.
13200 if (MI->getOpcode() == TargetOpcode::COPY) {
13201 assert(Reg.isVirtual() && "expected COPY of virtual register");
13202 Reg = MI->getOperand(i: 1).getReg();
13203 assert(Reg.isPhysical() && "expected COPY of physical register");
13204 } else {
13205 // The start of the chain must be an INLINEASM_BR.
13206 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13207 "end of copy chain MUST be INLINEASM_BR");
13208 }
13209
13210 return Reg;
13211}
13212
13213// We must do this walk rather than the simpler
13214// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
13215// otherwise we will end up with copies of virtregs only valid along direct
13216// edges.
13217void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
13218 SmallVector<EVT, 8> ResultVTs;
13219 SmallVector<SDValue, 8> ResultValues;
13220 const auto *CBR =
13221 cast<CallBrInst>(Val: I.getParent()->getUniquePredecessor()->getTerminator());
13222
13223 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13224 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
13225 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
13226
13227 Register InitialDef = FuncInfo.ValueMap[CBR];
13228 SDValue Chain = DAG.getRoot();
13229
13230 // Re-parse the asm constraints string.
13231 TargetLowering::AsmOperandInfoVector TargetConstraints =
13232 TLI.ParseConstraints(DL: DAG.getDataLayout(), TRI, Call: *CBR);
13233 for (auto &T : TargetConstraints) {
13234 SDISelAsmOperandInfo OpInfo(T);
13235 if (OpInfo.Type != InlineAsm::isOutput)
13236 continue;
13237
13238 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
13239 // individual constraint.
13240 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
13241
13242 switch (OpInfo.ConstraintType) {
13243 case TargetLowering::C_Register:
13244 case TargetLowering::C_RegisterClass: {
13245 // Fill in OpInfo.AssignedRegs.Regs.
13246 getRegistersForValue(DAG, DL: getCurSDLoc(), OpInfo, RefOpInfo&: OpInfo);
13247
13248 // getRegistersForValue may produce 1 to many registers based on whether
13249 // the OpInfo.ConstraintVT is legal on the target or not.
13250 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
13251 Register OriginalDef = FollowCopyChain(MRI, Reg: InitialDef++);
13252 if (OriginalDef.isPhysical())
13253 FuncInfo.MBB->addLiveIn(PhysReg: OriginalDef);
13254 // Update the assigned registers to use the original defs.
13255 Reg = OriginalDef;
13256 }
13257
13258 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
13259 DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V: CBR);
13260 ResultValues.push_back(Elt: V);
13261 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
13262 break;
13263 }
13264 case TargetLowering::C_Other: {
13265 SDValue Flag;
13266 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Glue&: Flag, DL: getCurSDLoc(),
13267 OpInfo, DAG);
13268 ++InitialDef;
13269 ResultValues.push_back(Elt: V);
13270 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
13271 break;
13272 }
13273 default:
13274 break;
13275 }
13276 }
13277 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
13278 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
13279 setValue(V: &I, NewN: V);
13280}
13281