1//===- Instructions.cpp - Implement the LLVM instructions -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements all of the non-inline methods for the LLVM instruction
10// classes.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/Instructions.h"
15#include "IROptions.h"
16#include "LLVMContextImpl.h"
17#include "llvm/ADT/SmallBitVector.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Twine.h"
20#include "llvm/IR/Attributes.h"
21#include "llvm/IR/BasicBlock.h"
22#include "llvm/IR/Constant.h"
23#include "llvm/IR/ConstantRange.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/InstrTypes.h"
29#include "llvm/IR/Instruction.h"
30#include "llvm/IR/IntrinsicInst.h"
31#include "llvm/IR/Intrinsics.h"
32#include "llvm/IR/LLVMContext.h"
33#include "llvm/IR/MDBuilder.h"
34#include "llvm/IR/Metadata.h"
35#include "llvm/IR/Module.h"
36#include "llvm/IR/Operator.h"
37#include "llvm/IR/PatternMatch.h"
38#include "llvm/IR/ProfDataUtils.h"
39#include "llvm/IR/Type.h"
40#include "llvm/IR/Value.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/CheckedArithmetic.h"
44#include "llvm/Support/Compiler.h"
45#include "llvm/Support/ErrorHandling.h"
46#include "llvm/Support/KnownBits.h"
47#include "llvm/Support/MathExtras.h"
48#include "llvm/Support/ModRef.h"
49#include "llvm/Support/TypeSize.h"
50#include <algorithm>
51#include <cassert>
52#include <cstdint>
53#include <optional>
54#include <vector>
55
56using namespace llvm;
57
58//===----------------------------------------------------------------------===//
59// AllocaInst Class
60//===----------------------------------------------------------------------===//
61
62TypeSize AllocaInst::getAllocationBaseSize(const DataLayout &DL) const {
63 return DL.getTypeAllocSize(Ty: getAllocatedType());
64}
65
66std::optional<TypeSize>
67AllocaInst::getAllocationSize(const DataLayout &DL) const {
68 TypeSize Size = getAllocationBaseSize(DL);
69 // Zero-sized types can return early since 0 * N = 0 for any array size N.
70 if (Size.isZero())
71 return Size;
72 if (isArrayAllocation()) {
73 auto *C = dyn_cast<ConstantInt>(Val: getArraySize());
74 if (!C)
75 return std::nullopt;
76 std::optional<uint64_t> NumElements = C->getValue().tryZExtValue();
77 if (!NumElements)
78 return std::nullopt;
79 assert(!Size.isScalable() && "Array elements cannot have a scalable size");
80 auto CheckedProd =
81 checkedMulUnsigned(LHS: Size.getKnownMinValue(), RHS: *NumElements);
82 if (!CheckedProd)
83 return std::nullopt;
84 return TypeSize::getFixed(ExactSize: *CheckedProd);
85 }
86 return Size;
87}
88
89std::optional<TypeSize>
90AllocaInst::getAllocationSizeInBits(const DataLayout &DL) const {
91 std::optional<TypeSize> Size = getAllocationSize(DL);
92 if (!Size)
93 return std::nullopt;
94 auto CheckedProd = checkedMulUnsigned(LHS: Size->getKnownMinValue(),
95 RHS: static_cast<TypeSize::ScalarTy>(8));
96 if (!CheckedProd)
97 return std::nullopt;
98 return TypeSize::get(Quantity: *CheckedProd, Scalable: Size->isScalable());
99}
100
101//===----------------------------------------------------------------------===//
102// SelectInst Class
103//===----------------------------------------------------------------------===//
104
105/// areInvalidOperands - Return a string if the specified operands are invalid
106/// for a select operation, otherwise return null.
107const char *SelectInst::areInvalidOperands(Value *Op0, Value *Op1, Value *Op2) {
108 if (Op1->getType() != Op2->getType())
109 return "both values to select must have same type";
110
111 if (Op1->getType()->isTokenTy())
112 return "select values cannot have token type";
113
114 if (VectorType *VT = dyn_cast<VectorType>(Val: Op0->getType())) {
115 // Vector select.
116 if (VT->getElementType() != Type::getInt1Ty(C&: Op0->getContext()))
117 return "vector select condition element type must be i1";
118 VectorType *ET = dyn_cast<VectorType>(Val: Op1->getType());
119 if (!ET)
120 return "selected values for vector select must be vectors";
121 if (ET->getElementCount() != VT->getElementCount())
122 return "vector select requires selected vectors to have "
123 "the same vector length as select condition";
124 } else if (Op0->getType() != Type::getInt1Ty(C&: Op0->getContext())) {
125 return "select condition must be i1 or <n x i1>";
126 }
127 return nullptr;
128}
129
130//===----------------------------------------------------------------------===//
131// PHINode Class
132//===----------------------------------------------------------------------===//
133
134PHINode::PHINode(const PHINode &PN)
135 : Instruction(PN.getType(), Instruction::PHI, AllocMarker),
136 ReservedSpace(PN.getNumOperands()) {
137 allocHungoffUses(N: PN.getNumOperands());
138 setNumHungOffUseOperands(PN.getNumOperands());
139 std::copy(first: PN.op_begin(), last: PN.op_end(), result: op_begin());
140 copyIncomingBlocks(BBRange: make_range(x: PN.block_begin(), y: PN.block_end()));
141 FMF = PN.FMF;
142}
143
144// removeIncomingValue - Remove an incoming value. This is useful if a
145// predecessor basic block is deleted.
146Value *PHINode::removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty) {
147 Value *Removed = getIncomingValue(i: Idx);
148 // Swap with the end of the list.
149 unsigned Last = getNumOperands() - 1;
150 if (Idx != Last) {
151 setIncomingValue(i: Idx, V: getIncomingValue(i: Last));
152 setIncomingBlock(i: Idx, BB: getIncomingBlock(i: Last));
153 }
154
155 // Nuke the last value.
156 Op<-1>().set(nullptr);
157 setNumHungOffUseOperands(getNumOperands() - 1);
158
159 // If the PHI node is dead, because it has zero entries, nuke it now.
160 if (getNumOperands() == 0 && DeletePHIIfEmpty) {
161 // If anyone is using this PHI, make them use a dummy value instead...
162 replaceAllUsesWith(V: PoisonValue::get(T: getType()));
163 eraseFromParent();
164 }
165 return Removed;
166}
167
168void PHINode::removeIncomingValueIf(function_ref<bool(unsigned)> Predicate,
169 bool DeletePHIIfEmpty) {
170 unsigned NumOps = getNumIncomingValues();
171
172 // Loop backwards in case the predicate is purely index based.
173 for (unsigned Idx = NumOps; Idx-- > 0;) {
174 if (Predicate(Idx)) {
175 unsigned LastIdx = NumOps - 1;
176 if (Idx != LastIdx) {
177 setIncomingValue(i: Idx, V: getIncomingValue(i: LastIdx));
178 setIncomingBlock(i: Idx, BB: getIncomingBlock(i: LastIdx));
179 }
180 getOperandUse(i: LastIdx).set(nullptr);
181 NumOps--;
182 }
183 }
184
185 setNumHungOffUseOperands(NumOps);
186
187 // If the PHI node is dead, because it has zero entries, nuke it now.
188 if (getNumOperands() == 0 && DeletePHIIfEmpty) {
189 // If anyone is using this PHI, make them use a dummy value instead...
190 replaceAllUsesWith(V: PoisonValue::get(T: getType()));
191 eraseFromParent();
192 }
193}
194
195/// growOperands - grow operands - This grows the operand list in response
196/// to a push_back style of operation. This grows the number of ops by 1.5
197/// times.
198///
199void PHINode::growOperands() {
200 unsigned e = getNumOperands();
201 unsigned NumOps = e + e / 2;
202 if (NumOps < 2) NumOps = 2; // 2 op PHI nodes are VERY common.
203
204 ReservedSpace = NumOps;
205 growHungoffUses(N: ReservedSpace, /*WithExtraValues=*/true);
206}
207
208/// hasConstantValue - If the specified PHI node always merges together the same
209/// value, return the value, otherwise return null.
210Value *PHINode::hasConstantValue() const {
211 // Exploit the fact that phi nodes always have at least one entry.
212 Value *ConstantValue = getIncomingValue(i: 0);
213 for (unsigned i = 1, e = getNumIncomingValues(); i != e; ++i)
214 if (getIncomingValue(i) != ConstantValue && getIncomingValue(i) != this) {
215 if (ConstantValue != this)
216 return nullptr; // Incoming values not all the same.
217 // The case where the first value is this PHI.
218 ConstantValue = getIncomingValue(i);
219 }
220 if (ConstantValue == this)
221 return PoisonValue::get(T: getType());
222 return ConstantValue;
223}
224
225/// hasConstantOrUndefValue - Whether the specified PHI node always merges
226/// together the same value, assuming that undefs result in the same value as
227/// non-undefs.
228/// Unlike \ref hasConstantValue, this does not return a value because the
229/// unique non-undef incoming value need not dominate the PHI node.
230bool PHINode::hasConstantOrUndefValue() const {
231 Value *ConstantValue = nullptr;
232 for (unsigned i = 0, e = getNumIncomingValues(); i != e; ++i) {
233 Value *Incoming = getIncomingValue(i);
234 if (Incoming != this && !isa<UndefValue>(Val: Incoming)) {
235 if (ConstantValue && ConstantValue != Incoming)
236 return false;
237 ConstantValue = Incoming;
238 }
239 }
240 return true;
241}
242
243//===----------------------------------------------------------------------===//
244// LandingPadInst Implementation
245//===----------------------------------------------------------------------===//
246
247LandingPadInst::LandingPadInst(Type *RetTy, unsigned NumReservedValues,
248 const Twine &NameStr,
249 InsertPosition InsertBefore)
250 : Instruction(RetTy, Instruction::LandingPad, AllocMarker, InsertBefore) {
251 init(NumReservedValues, NameStr);
252}
253
254LandingPadInst::LandingPadInst(const LandingPadInst &LP)
255 : Instruction(LP.getType(), Instruction::LandingPad, AllocMarker),
256 ReservedSpace(LP.getNumOperands()) {
257 allocHungoffUses(N: LP.getNumOperands());
258 setNumHungOffUseOperands(LP.getNumOperands());
259 Use *OL = getOperandList();
260 const Use *InOL = LP.getOperandList();
261 for (unsigned I = 0, E = ReservedSpace; I != E; ++I)
262 OL[I] = InOL[I];
263
264 setCleanup(LP.isCleanup());
265}
266
267LandingPadInst *LandingPadInst::Create(Type *RetTy, unsigned NumReservedClauses,
268 const Twine &NameStr,
269 InsertPosition InsertBefore) {
270 return new LandingPadInst(RetTy, NumReservedClauses, NameStr, InsertBefore);
271}
272
273void LandingPadInst::init(unsigned NumReservedValues, const Twine &NameStr) {
274 ReservedSpace = NumReservedValues;
275 allocHungoffUses(N: ReservedSpace);
276 setNumHungOffUseOperands(0);
277 setName(NameStr);
278 setCleanup(false);
279}
280
281/// growOperands - grow operands - This grows the operand list in response to a
282/// push_back style of operation. This grows the number of ops by 2 times.
283void LandingPadInst::growOperands(unsigned Size) {
284 unsigned e = getNumOperands();
285 if (ReservedSpace >= e + Size) return;
286 ReservedSpace = (std::max(a: e, b: 1U) + Size / 2) * 2;
287 growHungoffUses(N: ReservedSpace);
288}
289
290void LandingPadInst::addClause(Constant *Val) {
291 unsigned OpNo = getNumOperands();
292 growOperands(Size: 1);
293 assert(OpNo < ReservedSpace && "Growing didn't work!");
294 setNumHungOffUseOperands(getNumOperands() + 1);
295 getOperandList()[OpNo] = Val;
296}
297
298//===----------------------------------------------------------------------===//
299// CallBase Implementation
300//===----------------------------------------------------------------------===//
301
302CallBase *CallBase::Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles,
303 InsertPosition InsertPt) {
304 switch (CB->getOpcode()) {
305 case Instruction::Call:
306 return CallInst::Create(CI: cast<CallInst>(Val: CB), Bundles, InsertPt);
307 case Instruction::Invoke:
308 return InvokeInst::Create(II: cast<InvokeInst>(Val: CB), Bundles, InsertPt);
309 case Instruction::CallBr:
310 return CallBrInst::Create(CBI: cast<CallBrInst>(Val: CB), Bundles, InsertBefore: InsertPt);
311 default:
312 llvm_unreachable("Unknown CallBase sub-class!");
313 }
314}
315
316CallBase *CallBase::Create(CallBase *CI, OperandBundleDef OpB,
317 InsertPosition InsertPt) {
318 SmallVector<OperandBundleDef, 2> OpDefs;
319 for (unsigned i = 0, e = CI->getNumOperandBundles(); i < e; ++i) {
320 auto ChildOB = CI->getOperandBundleAt(Index: i);
321 if (ChildOB.getTagName() != OpB.getTag())
322 OpDefs.emplace_back(Args&: ChildOB);
323 }
324 OpDefs.emplace_back(Args&: OpB);
325 return CallBase::Create(CB: CI, Bundles: OpDefs, InsertPt);
326}
327
328Function *CallBase::getCaller() { return getParent()->getParent(); }
329
330unsigned CallBase::getNumSubclassExtraOperandsDynamic() const {
331 assert(getOpcode() == Instruction::CallBr && "Unexpected opcode!");
332 return cast<CallBrInst>(Val: this)->getNumIndirectDests() + 1;
333}
334
335bool CallBase::isIndirectCall() const {
336 const Value *V = getCalledOperand();
337 if (isa<Function>(Val: V) || isa<Constant>(Val: V))
338 return false;
339 return !isInlineAsm();
340}
341
342/// Tests if this call site must be tail call optimized. Only a CallInst can
343/// be tail call optimized.
344bool CallBase::isMustTailCall() const {
345 if (auto *CI = dyn_cast<CallInst>(Val: this))
346 return CI->isMustTailCall();
347 return false;
348}
349
350/// Tests if this call site is marked as a tail call.
351bool CallBase::isTailCall() const {
352 if (auto *CI = dyn_cast<CallInst>(Val: this))
353 return CI->isTailCall();
354 return false;
355}
356
357Intrinsic::ID CallBase::getIntrinsicID() const {
358 if (auto *F = dyn_cast_or_null<Function>(Val: getCalledOperand()))
359 return F->getIntrinsicID();
360 return Intrinsic::not_intrinsic;
361}
362
363FPClassTest CallBase::getRetNoFPClass() const {
364 FPClassTest Mask = Attrs.getRetNoFPClass();
365
366 if (const Function *F = getCalledFunction())
367 Mask |= F->getAttributes().getRetNoFPClass();
368 return Mask;
369}
370
371FPClassTest CallBase::getParamNoFPClass(unsigned i) const {
372 FPClassTest Mask = Attrs.getParamNoFPClass(ArgNo: i);
373
374 if (const Function *F = getCalledFunction())
375 Mask |= F->getAttributes().getParamNoFPClass(ArgNo: i);
376 return Mask;
377}
378
379std::optional<ConstantRange> CallBase::getRange() const {
380 Attribute CallAttr = Attrs.getRetAttr(Kind: Attribute::Range);
381 Attribute FnAttr;
382 if (const Function *F = getCalledFunction())
383 FnAttr = F->getRetAttribute(Kind: Attribute::Range);
384
385 if (CallAttr.isValid() && FnAttr.isValid())
386 return CallAttr.getRange().intersectWith(CR: FnAttr.getRange());
387 if (CallAttr.isValid())
388 return CallAttr.getRange();
389 if (FnAttr.isValid())
390 return FnAttr.getRange();
391 return std::nullopt;
392}
393
394bool CallBase::isReturnNonNull() const {
395 if (hasRetAttr(Kind: Attribute::NonNull))
396 return true;
397
398 if (getRetDereferenceableBytes() > 0 &&
399 !NullPointerIsDefined(F: getCaller(), AS: getType()->getPointerAddressSpace()))
400 return true;
401
402 return false;
403}
404
405Value *CallBase::getArgOperandWithAttribute(Attribute::AttrKind Kind) const {
406 unsigned Index;
407
408 if (Attrs.hasAttrSomewhere(Kind, Index: &Index))
409 return getArgOperand(i: Index - AttributeList::FirstArgIndex);
410 if (const Function *F = getCalledFunction())
411 if (F->getAttributes().hasAttrSomewhere(Kind, Index: &Index))
412 return getArgOperand(i: Index - AttributeList::FirstArgIndex);
413
414 return nullptr;
415}
416
417/// Determine whether the argument or parameter has the given attribute.
418bool CallBase::paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
419 assert(ArgNo < arg_size() && "Param index out of bounds!");
420
421 if (Attrs.hasParamAttr(ArgNo, Kind))
422 return true;
423
424 const Function *F = getCalledFunction();
425 if (!F)
426 return false;
427
428 if (!F->getAttributes().hasParamAttr(ArgNo, Kind))
429 return false;
430
431 // Take into account mod/ref by operand bundles.
432 switch (Kind) {
433 case Attribute::ReadNone:
434 return !hasReadingOperandBundles() && !hasClobberingOperandBundles();
435 case Attribute::ReadOnly:
436 return !hasClobberingOperandBundles();
437 case Attribute::WriteOnly:
438 return !hasReadingOperandBundles();
439 default:
440 return true;
441 }
442}
443
444bool CallBase::paramHasNonNullAttr(unsigned ArgNo,
445 bool AllowUndefOrPoison) const {
446 assert(getArgOperand(ArgNo)->getType()->isPointerTy() &&
447 "Argument must be a pointer");
448 if (paramHasAttr(ArgNo, Kind: Attribute::NonNull) &&
449 (AllowUndefOrPoison || paramHasAttr(ArgNo, Kind: Attribute::NoUndef)))
450 return true;
451
452 if (paramHasAttr(ArgNo, Kind: Attribute::Dereferenceable) &&
453 !NullPointerIsDefined(
454 F: getCaller(),
455 AS: getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace()))
456 return true;
457
458 return false;
459}
460
461bool CallBase::isNoInline() const {
462 if (Attrs.hasFnAttr(Kind: Attribute::NoInline))
463 return true;
464 if (const auto *F = dyn_cast<Function>(Val: getCalledOperand()))
465 return F->isNoInline();
466 return false;
467}
468
469bool CallBase::hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const {
470 if (auto *F = dyn_cast<Function>(Val: getCalledOperand()))
471 return F->getAttributes().hasFnAttr(Kind);
472
473 return false;
474}
475
476bool CallBase::hasFnAttrOnCalledFunction(StringRef Kind) const {
477 if (auto *F = dyn_cast<Function>(Val: getCalledOperand()))
478 return F->getAttributes().hasFnAttr(Kind);
479
480 return false;
481}
482
483template <typename AK>
484Attribute CallBase::getFnAttrOnCalledFunction(AK Kind) const {
485 if constexpr (std::is_same_v<AK, Attribute::AttrKind>) {
486 // getMemoryEffects() correctly combines memory effects from the call-site,
487 // operand bundles and function.
488 assert(Kind != Attribute::Memory && "Use getMemoryEffects() instead");
489 }
490
491 if (auto *F = dyn_cast<Function>(Val: getCalledOperand()))
492 return F->getAttributes().getFnAttr(Kind);
493
494 return Attribute();
495}
496
497template LLVM_ABI Attribute
498CallBase::getFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;
499template LLVM_ABI Attribute
500CallBase::getFnAttrOnCalledFunction(StringRef Kind) const;
501
502template <typename AK>
503Attribute CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,
504 AK Kind) const {
505 Value *V = getCalledOperand();
506
507 if (auto *F = dyn_cast<Function>(Val: V))
508 return F->getAttributes().getParamAttr(ArgNo, Kind);
509
510 return Attribute();
511}
512template LLVM_ABI Attribute CallBase::getParamAttrOnCalledFunction(
513 unsigned ArgNo, Attribute::AttrKind Kind) const;
514template LLVM_ABI Attribute
515CallBase::getParamAttrOnCalledFunction(unsigned ArgNo, StringRef Kind) const;
516
517void CallBase::getOperandBundlesAsDefs(
518 SmallVectorImpl<OperandBundleDef> &Defs) const {
519 for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)
520 Defs.emplace_back(Args: getOperandBundleAt(Index: i));
521}
522
523CallBase::op_iterator
524CallBase::populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles,
525 const unsigned BeginIndex) {
526 auto It = op_begin() + BeginIndex;
527 for (auto &B : Bundles)
528 It = std::copy(first: B.input_begin(), last: B.input_end(), result: It);
529
530 auto *ContextImpl = getContext().pImpl;
531 auto BI = Bundles.begin();
532 unsigned CurrentIndex = BeginIndex;
533
534 for (auto &BOI : bundle_op_infos()) {
535 assert(BI != Bundles.end() && "Incorrect allocation?");
536
537 BOI.Tag = ContextImpl->getOrInsertBundleTag(Tag: BI->getTag());
538 BOI.Begin = CurrentIndex;
539 BOI.End = CurrentIndex + BI->input_size();
540 CurrentIndex = BOI.End;
541 BI++;
542 }
543
544 assert(BI == Bundles.end() && "Incorrect allocation?");
545
546 return It;
547}
548
549CallBase::BundleOpInfo &CallBase::getBundleOpInfoForOperand(unsigned OpIdx) {
550 /// When there isn't many bundles, we do a simple linear search.
551 /// Else fallback to a binary-search that use the fact that bundles usually
552 /// have similar number of argument to get faster convergence.
553 if (bundle_op_info_end() - bundle_op_info_begin() < 8) {
554 for (auto &BOI : bundle_op_infos())
555 if (BOI.Begin <= OpIdx && OpIdx < BOI.End)
556 return BOI;
557
558 llvm_unreachable("Did not find operand bundle for operand!");
559 }
560
561 assert(OpIdx >= arg_size() && "the Idx is not in the operand bundles");
562 assert(bundle_op_info_end() - bundle_op_info_begin() > 0 &&
563 OpIdx < std::prev(bundle_op_info_end())->End &&
564 "The Idx isn't in the operand bundle");
565
566 /// We need a decimal number below and to prevent using floating point numbers
567 /// we use an intergal value multiplied by this constant.
568 constexpr unsigned NumberScaling = 1024;
569
570 bundle_op_iterator Begin = bundle_op_info_begin();
571 bundle_op_iterator End = bundle_op_info_end();
572 bundle_op_iterator Current = Begin;
573
574 while (Begin != End) {
575 unsigned ScaledOperandPerBundle =
576 NumberScaling * (std::prev(x: End)->End - Begin->Begin) / (End - Begin);
577 Current = Begin + (((OpIdx - Begin->Begin) * NumberScaling) /
578 ScaledOperandPerBundle);
579 if (Current >= End)
580 Current = std::prev(x: End);
581 assert(Current < End && Current >= Begin &&
582 "the operand bundle doesn't cover every value in the range");
583 if (OpIdx >= Current->Begin && OpIdx < Current->End)
584 break;
585 if (OpIdx >= Current->End)
586 Begin = Current + 1;
587 else
588 End = Current;
589 }
590
591 assert(OpIdx >= Current->Begin && OpIdx < Current->End &&
592 "the operand bundle doesn't cover every value in the range");
593 return *Current;
594}
595
596CallBase *CallBase::addOperandBundle(CallBase *CB, uint32_t ID,
597 OperandBundleDef OB,
598 InsertPosition InsertPt) {
599 if (CB->getOperandBundle(ID))
600 return CB;
601
602 SmallVector<OperandBundleDef, 1> Bundles;
603 CB->getOperandBundlesAsDefs(Defs&: Bundles);
604 Bundles.push_back(Elt: OB);
605 return Create(CB, Bundles, InsertPt);
606}
607
608CallBase *CallBase::removeOperandBundle(CallBase *CB, uint32_t ID,
609 InsertPosition InsertPt) {
610 SmallVector<OperandBundleDef, 1> Bundles;
611 bool CreateNew = false;
612
613 for (unsigned I = 0, E = CB->getNumOperandBundles(); I != E; ++I) {
614 auto Bundle = CB->getOperandBundleAt(Index: I);
615 if (Bundle.getTagID() == ID) {
616 CreateNew = true;
617 continue;
618 }
619 Bundles.emplace_back(Args&: Bundle);
620 }
621
622 return CreateNew ? Create(CB, Bundles, InsertPt) : CB;
623}
624
625CallBase *CallBase::removeOperandBundleAt(CallBase *CB, size_t Offset,
626 InsertPosition InsertPt) {
627 auto OpBundleCount = CB->getNumOperandBundles();
628 assert(Offset < OpBundleCount &&
629 "Trying to remove non-existant operand bundle");
630 SmallVector<OperandBundleDef> Bundles;
631 Bundles.reserve(N: OpBundleCount - 1);
632 size_t I = 0;
633 for (; I != Offset; ++I)
634 Bundles.emplace_back(Args: CB->getOperandBundleAt(Index: I));
635 ++I;
636 for (; I != OpBundleCount; ++I)
637 Bundles.emplace_back(Args: CB->getOperandBundleAt(Index: I));
638 return Create(CB, Bundles, InsertPt);
639}
640
641bool CallBase::hasReadingOperandBundles() const {
642 // Implementation note: this is a conservative implementation of operand
643 // bundle semantics, where *any* non-assume operand bundle (other than
644 // ptrauth) forces a callsite to be at least readonly.
645 return hasOperandBundlesOtherThan(IDs: {LLVMContext::OB_ptrauth,
646 LLVMContext::OB_kcfi,
647 LLVMContext::OB_convergencectrl,
648 LLVMContext::OB_deactivation_symbol}) &&
649 getIntrinsicID() != Intrinsic::assume;
650}
651
652bool CallBase::hasClobberingOperandBundles() const {
653 return hasOperandBundlesOtherThan(
654 IDs: {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
655 LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi,
656 LLVMContext::OB_convergencectrl,
657 LLVMContext::OB_deactivation_symbol}) &&
658 getIntrinsicID() != Intrinsic::assume;
659}
660
661bool CallBase::isVolatile() const {
662 // There are a very limited number of intrinsics with volatile flags.
663 if (const auto *MI = dyn_cast<MemIntrinsic>(Val: this))
664 return MI->isVolatile();
665
666 switch (getIntrinsicID()) {
667 case Intrinsic::matrix_column_major_load:
668 return cast<ConstantInt>(Val: getArgOperand(i: 2))->isOne();
669 case Intrinsic::matrix_column_major_store:
670 return cast<ConstantInt>(Val: getArgOperand(i: 3))->isOne();
671 default:
672 return false;
673 }
674}
675
676MemoryEffects CallBase::getMemoryEffects() const {
677 MemoryEffects ME = getAttributes().getMemoryEffects();
678 if (auto *Fn = dyn_cast<Function>(Val: getCalledOperand())) {
679 MemoryEffects FnME = Fn->getMemoryEffects();
680 if (hasOperandBundles()) {
681 // TODO: Add a method to get memory effects for operand bundles instead.
682 if (hasReadingOperandBundles())
683 FnME |= MemoryEffects::readOnly();
684 if (hasClobberingOperandBundles())
685 FnME |= MemoryEffects::writeOnly();
686 }
687 if (isVolatile()) {
688 // Volatile operations also access inaccessible memory.
689 FnME |= MemoryEffects::inaccessibleMemOnly();
690 }
691 ME &= FnME;
692 }
693 return ME;
694}
695void CallBase::setMemoryEffects(MemoryEffects ME) {
696 addFnAttr(Attr: Attribute::getWithMemoryEffects(Context&: getContext(), ME));
697}
698
699/// Determine if the function does not access memory.
700bool CallBase::doesNotAccessMemory() const {
701 return getMemoryEffects().doesNotAccessMemory();
702}
703void CallBase::setDoesNotAccessMemory() {
704 setMemoryEffects(MemoryEffects::none());
705}
706
707/// Determine if the function does not access or only reads memory.
708bool CallBase::onlyReadsMemory() const {
709 return getMemoryEffects().onlyReadsMemory();
710}
711void CallBase::setOnlyReadsMemory() {
712 setMemoryEffects(getMemoryEffects() & MemoryEffects::readOnly());
713}
714
715/// Determine if the function does not access or only writes memory.
716bool CallBase::onlyWritesMemory() const {
717 return getMemoryEffects().onlyWritesMemory();
718}
719void CallBase::setOnlyWritesMemory() {
720 setMemoryEffects(getMemoryEffects() & MemoryEffects::writeOnly());
721}
722
723/// Determine if the call can access memmory only using pointers based
724/// on its arguments.
725bool CallBase::onlyAccessesArgMemory() const {
726 return getMemoryEffects().onlyAccessesArgPointees();
727}
728void CallBase::setOnlyAccessesArgMemory() {
729 setMemoryEffects(getMemoryEffects() & MemoryEffects::argMemOnly());
730}
731
732/// Determine if the function may only access memory that is
733/// inaccessible from the IR.
734bool CallBase::onlyAccessesInaccessibleMemory() const {
735 return getMemoryEffects().onlyAccessesInaccessibleMem();
736}
737void CallBase::setOnlyAccessesInaccessibleMemory() {
738 setMemoryEffects(getMemoryEffects() & MemoryEffects::inaccessibleMemOnly());
739}
740
741/// Determine if the function may only access memory that is
742/// either inaccessible from the IR or pointed to by its arguments.
743bool CallBase::onlyAccessesInaccessibleMemOrArgMem() const {
744 return getMemoryEffects().onlyAccessesInaccessibleOrArgMem();
745}
746void CallBase::setOnlyAccessesInaccessibleMemOrArgMem() {
747 setMemoryEffects(getMemoryEffects() &
748 MemoryEffects::inaccessibleOrArgMemOnly());
749}
750
751CaptureInfo CallBase::getCaptureInfo(unsigned OpNo) const {
752 if (OpNo < arg_size()) {
753 // If the argument is passed byval, the callee does not have access to the
754 // original pointer and thus cannot capture it.
755 if (isByValArgument(ArgNo: OpNo))
756 return CaptureInfo::none();
757
758 CaptureInfo CI = getParamAttributes(ArgNo: OpNo).getCaptureInfo();
759 if (auto *Fn = dyn_cast<Function>(Val: getCalledOperand()))
760 CI &= Fn->getAttributes().getParamAttrs(ArgNo: OpNo).getCaptureInfo();
761 return CI;
762 }
763
764 // Bundles on assumes are captures(none).
765 if (getIntrinsicID() == Intrinsic::assume)
766 return CaptureInfo::none();
767
768 // deopt operand bundles are captures(none)
769 auto &BOI = getBundleOpInfoForOperand(OpIdx: OpNo);
770 auto OBU = operandBundleFromBundleOpInfo(BOI);
771 return OBU.isDeoptOperandBundle() ? CaptureInfo::none() : CaptureInfo::all();
772}
773
774bool CallBase::hasArgumentWithAdditionalReturnCaptureComponents() const {
775 for (unsigned I = 0, E = arg_size(); I < E; ++I) {
776 if (!getArgOperand(i: I)->getType()->isPointerTy())
777 continue;
778
779 CaptureInfo CI = getParamAttributes(ArgNo: I).getCaptureInfo();
780 if (auto *Fn = dyn_cast<Function>(Val: getCalledOperand()))
781 CI &= Fn->getAttributes().getParamAttrs(ArgNo: I).getCaptureInfo();
782 if (capturesAnything(CC: CI.getRetComponents() & ~CI.getOtherComponents()))
783 return true;
784 }
785 return false;
786}
787
788//===----------------------------------------------------------------------===//
789// CallInst Implementation
790//===----------------------------------------------------------------------===//
791
792void CallInst::init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
793 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr) {
794 this->FTy = FTy;
795 assert(getNumOperands() == Args.size() + CountBundleInputs(Bundles) + 1 &&
796 "NumOperands not set up?");
797
798#ifndef NDEBUG
799 assert((Args.size() == FTy->getNumParams() ||
800 (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&
801 "Calling a function with bad signature!");
802
803 for (unsigned i = 0; i != Args.size(); ++i)
804 assert((i >= FTy->getNumParams() ||
805 FTy->getParamType(i) == Args[i]->getType()) &&
806 "Calling a function with a bad signature!");
807#endif
808
809 // Set operands in order of their index to match use-list-order
810 // prediction.
811 llvm::copy(Range&: Args, Out: op_begin());
812 setCalledOperand(Func);
813
814 auto It = populateBundleOperandInfos(Bundles, BeginIndex: Args.size());
815 (void)It;
816 assert(It + 1 == op_end() && "Should add up!");
817
818 setName(NameStr);
819}
820
821void CallInst::init(FunctionType *FTy, Value *Func, const Twine &NameStr) {
822 this->FTy = FTy;
823 assert(getNumOperands() == 1 && "NumOperands not set up?");
824 setCalledOperand(Func);
825
826 assert(FTy->getNumParams() == 0 && "Calling a function with bad signature");
827
828 setName(NameStr);
829}
830
831CallInst::CallInst(FunctionType *Ty, Value *Func, const Twine &Name,
832 AllocInfo AllocInfo, InsertPosition InsertBefore)
833 : CallBase(Ty->getReturnType(), Instruction::Call, AllocInfo,
834 InsertBefore) {
835 init(FTy: Ty, Func, NameStr: Name);
836}
837
838CallInst::CallInst(const CallInst &CI, AllocInfo AllocInfo)
839 : CallBase(CI.Attrs, CI.FTy, CI.getType(), Instruction::Call, AllocInfo) {
840 assert(getNumOperands() == CI.getNumOperands() &&
841 "Wrong number of operands allocated");
842 setTailCallKind(CI.getTailCallKind());
843 setCallingConv(CI.getCallingConv());
844
845 std::copy(first: CI.op_begin(), last: CI.op_end(), result: op_begin());
846 std::copy(first: CI.bundle_op_info_begin(), last: CI.bundle_op_info_end(),
847 result: bundle_op_info_begin());
848 FMF = CI.FMF;
849}
850
851CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB,
852 InsertPosition InsertPt) {
853 std::vector<Value *> Args(CI->arg_begin(), CI->arg_end());
854
855 auto *NewCI = CallInst::Create(Ty: CI->getFunctionType(), Func: CI->getCalledOperand(),
856 Args, Bundles: OpB, NameStr: CI->getName(), InsertBefore: InsertPt);
857 NewCI->setTailCallKind(CI->getTailCallKind());
858 NewCI->setCallingConv(CI->getCallingConv());
859 NewCI->FMF = CI->FMF;
860 NewCI->setAttributes(CI->getAttributes());
861 NewCI->setDebugLoc(CI->getDebugLoc());
862 return NewCI;
863}
864
865// Update profile weight for call instruction by scaling it using the ratio
866// of S/T. The meaning of "branch_weights" meta data for call instruction is
867// transfered to represent call count.
868void CallInst::updateProfWeight(uint64_t S, uint64_t T) {
869 if (T == 0) {
870 LLVM_DEBUG(dbgs() << "Attempting to update profile weights will result in "
871 "div by 0. Ignoring. Likely the function "
872 << getParent()->getParent()->getName()
873 << " has 0 entry count, and contains call instructions "
874 "with non-zero prof info.");
875 return;
876 }
877 scaleProfData(I&: *this, S, T);
878}
879
880//===----------------------------------------------------------------------===//
881// InvokeInst Implementation
882//===----------------------------------------------------------------------===//
883
884void InvokeInst::init(FunctionType *FTy, Value *Fn, BasicBlock *IfNormal,
885 BasicBlock *IfException, ArrayRef<Value *> Args,
886 ArrayRef<OperandBundleDef> Bundles,
887 const Twine &NameStr) {
888 this->FTy = FTy;
889
890 assert(getNumOperands() ==
891 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles)) &&
892 "NumOperands not set up?");
893
894#ifndef NDEBUG
895 assert(((Args.size() == FTy->getNumParams()) ||
896 (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&
897 "Invoking a function with bad signature");
898
899 for (unsigned i = 0, e = Args.size(); i != e; i++)
900 assert((i >= FTy->getNumParams() ||
901 FTy->getParamType(i) == Args[i]->getType()) &&
902 "Invoking a function with a bad signature!");
903#endif
904
905 // Set operands in order of their index to match use-list-order
906 // prediction.
907 llvm::copy(Range&: Args, Out: op_begin());
908 setNormalDest(IfNormal);
909 setUnwindDest(IfException);
910 setCalledOperand(Fn);
911
912 auto It = populateBundleOperandInfos(Bundles, BeginIndex: Args.size());
913 (void)It;
914 assert(It + 3 == op_end() && "Should add up!");
915
916 setName(NameStr);
917}
918
919InvokeInst::InvokeInst(const InvokeInst &II, AllocInfo AllocInfo)
920 : CallBase(II.Attrs, II.FTy, II.getType(), Instruction::Invoke, AllocInfo) {
921 assert(getNumOperands() == II.getNumOperands() &&
922 "Wrong number of operands allocated");
923 setCallingConv(II.getCallingConv());
924 std::copy(first: II.op_begin(), last: II.op_end(), result: op_begin());
925 std::copy(first: II.bundle_op_info_begin(), last: II.bundle_op_info_end(),
926 result: bundle_op_info_begin());
927 SubclassOptionalData = II.SubclassOptionalData;
928}
929
930InvokeInst *InvokeInst::Create(InvokeInst *II, ArrayRef<OperandBundleDef> OpB,
931 InsertPosition InsertPt) {
932 std::vector<Value *> Args(II->arg_begin(), II->arg_end());
933
934 auto *NewII = InvokeInst::Create(
935 Ty: II->getFunctionType(), Func: II->getCalledOperand(), IfNormal: II->getNormalDest(),
936 IfException: II->getUnwindDest(), Args, Bundles: OpB, NameStr: II->getName(), InsertBefore: InsertPt);
937 NewII->setCallingConv(II->getCallingConv());
938 NewII->SubclassOptionalData = II->SubclassOptionalData;
939 NewII->setAttributes(II->getAttributes());
940 NewII->setDebugLoc(II->getDebugLoc());
941 return NewII;
942}
943
944LandingPadInst *InvokeInst::getLandingPadInst() const {
945 return cast<LandingPadInst>(Val: getUnwindDest()->getFirstNonPHIIt());
946}
947
948void InvokeInst::updateProfWeight(uint64_t S, uint64_t T) {
949 if (T == 0) {
950 LLVM_DEBUG(dbgs() << "Attempting to update profile weights will result in "
951 "div by 0. Ignoring. Likely the function "
952 << getParent()->getParent()->getName()
953 << " has 0 entry count, and contains call instructions "
954 "with non-zero prof info.");
955 return;
956 }
957 scaleProfData(I&: *this, S, T);
958}
959
960//===----------------------------------------------------------------------===//
961// CallBrInst Implementation
962//===----------------------------------------------------------------------===//
963
964void CallBrInst::init(FunctionType *FTy, Value *Fn, BasicBlock *Fallthrough,
965 ArrayRef<BasicBlock *> IndirectDests,
966 ArrayRef<Value *> Args,
967 ArrayRef<OperandBundleDef> Bundles,
968 const Twine &NameStr) {
969 this->FTy = FTy;
970
971 assert(getNumOperands() == ComputeNumOperands(Args.size(),
972 IndirectDests.size(),
973 CountBundleInputs(Bundles)) &&
974 "NumOperands not set up?");
975
976#ifndef NDEBUG
977 assert(((Args.size() == FTy->getNumParams()) ||
978 (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&
979 "Calling a function with bad signature");
980
981 for (unsigned i = 0, e = Args.size(); i != e; i++)
982 assert((i >= FTy->getNumParams() ||
983 FTy->getParamType(i) == Args[i]->getType()) &&
984 "Calling a function with a bad signature!");
985#endif
986
987 // Set operands in order of their index to match use-list-order
988 // prediction.
989 llvm::copy(Range&: Args, Out: op_begin());
990 NumIndirectDests = IndirectDests.size();
991 setDefaultDest(Fallthrough);
992 for (unsigned i = 0; i != NumIndirectDests; ++i)
993 setIndirectDest(i, B: IndirectDests[i]);
994 setCalledOperand(Fn);
995
996 auto It = populateBundleOperandInfos(Bundles, BeginIndex: Args.size());
997 (void)It;
998 assert(It + 2 + IndirectDests.size() == op_end() && "Should add up!");
999
1000 setName(NameStr);
1001}
1002
1003CallBrInst::CallBrInst(const CallBrInst &CBI, AllocInfo AllocInfo)
1004 : CallBase(CBI.Attrs, CBI.FTy, CBI.getType(), Instruction::CallBr,
1005 AllocInfo) {
1006 assert(getNumOperands() == CBI.getNumOperands() &&
1007 "Wrong number of operands allocated");
1008 setCallingConv(CBI.getCallingConv());
1009 std::copy(first: CBI.op_begin(), last: CBI.op_end(), result: op_begin());
1010 std::copy(first: CBI.bundle_op_info_begin(), last: CBI.bundle_op_info_end(),
1011 result: bundle_op_info_begin());
1012 SubclassOptionalData = CBI.SubclassOptionalData;
1013 NumIndirectDests = CBI.NumIndirectDests;
1014}
1015
1016CallBrInst *CallBrInst::Create(CallBrInst *CBI, ArrayRef<OperandBundleDef> OpB,
1017 InsertPosition InsertPt) {
1018 std::vector<Value *> Args(CBI->arg_begin(), CBI->arg_end());
1019
1020 auto *NewCBI = CallBrInst::Create(
1021 Ty: CBI->getFunctionType(), Func: CBI->getCalledOperand(), DefaultDest: CBI->getDefaultDest(),
1022 IndirectDests: CBI->getIndirectDests(), Args, Bundles: OpB, NameStr: CBI->getName(), InsertBefore: InsertPt);
1023 NewCBI->setCallingConv(CBI->getCallingConv());
1024 NewCBI->SubclassOptionalData = CBI->SubclassOptionalData;
1025 NewCBI->setAttributes(CBI->getAttributes());
1026 NewCBI->setDebugLoc(CBI->getDebugLoc());
1027 NewCBI->NumIndirectDests = CBI->NumIndirectDests;
1028 return NewCBI;
1029}
1030
1031//===----------------------------------------------------------------------===//
1032// ReturnInst Implementation
1033//===----------------------------------------------------------------------===//
1034
1035ReturnInst::ReturnInst(const ReturnInst &RI, AllocInfo AllocInfo)
1036 : Instruction(Type::getVoidTy(C&: RI.getContext()), Instruction::Ret,
1037 AllocInfo) {
1038 assert(getNumOperands() == RI.getNumOperands() &&
1039 "Wrong number of operands allocated");
1040 if (RI.getNumOperands())
1041 Op<0>() = RI.Op<0>();
1042 SubclassOptionalData = RI.SubclassOptionalData;
1043}
1044
1045ReturnInst::ReturnInst(LLVMContext &C, Value *retVal, AllocInfo AllocInfo,
1046 InsertPosition InsertBefore)
1047 : Instruction(Type::getVoidTy(C), Instruction::Ret, AllocInfo,
1048 InsertBefore) {
1049 if (retVal)
1050 Op<0>() = retVal;
1051}
1052
1053//===----------------------------------------------------------------------===//
1054// ResumeInst Implementation
1055//===----------------------------------------------------------------------===//
1056
1057ResumeInst::ResumeInst(const ResumeInst &RI)
1058 : Instruction(Type::getVoidTy(C&: RI.getContext()), Instruction::Resume,
1059 AllocMarker) {
1060 Op<0>() = RI.Op<0>();
1061}
1062
1063ResumeInst::ResumeInst(Value *Exn, InsertPosition InsertBefore)
1064 : Instruction(Type::getVoidTy(C&: Exn->getContext()), Instruction::Resume,
1065 AllocMarker, InsertBefore) {
1066 Op<0>() = Exn;
1067}
1068
1069//===----------------------------------------------------------------------===//
1070// CleanupReturnInst Implementation
1071//===----------------------------------------------------------------------===//
1072
1073CleanupReturnInst::CleanupReturnInst(const CleanupReturnInst &CRI,
1074 AllocInfo AllocInfo)
1075 : Instruction(CRI.getType(), Instruction::CleanupRet, AllocInfo) {
1076 assert(getNumOperands() == CRI.getNumOperands() &&
1077 "Wrong number of operands allocated");
1078 setSubclassData<Instruction::OpaqueField>(
1079 CRI.getSubclassData<Instruction::OpaqueField>());
1080 Op<0>() = CRI.Op<0>();
1081 if (CRI.hasUnwindDest())
1082 Op<1>() = CRI.Op<1>();
1083}
1084
1085void CleanupReturnInst::init(Value *CleanupPad, BasicBlock *UnwindBB) {
1086 if (UnwindBB)
1087 setSubclassData<UnwindDestField>(true);
1088
1089 Op<0>() = CleanupPad;
1090 if (UnwindBB)
1091 Op<1>() = UnwindBB;
1092}
1093
1094CleanupReturnInst::CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB,
1095 AllocInfo AllocInfo,
1096 InsertPosition InsertBefore)
1097 : Instruction(Type::getVoidTy(C&: CleanupPad->getContext()),
1098 Instruction::CleanupRet, AllocInfo, InsertBefore) {
1099 init(CleanupPad, UnwindBB);
1100}
1101
1102//===----------------------------------------------------------------------===//
1103// CatchReturnInst Implementation
1104//===----------------------------------------------------------------------===//
1105void CatchReturnInst::init(Value *CatchPad, BasicBlock *BB) {
1106 Op<0>() = CatchPad;
1107 Op<1>() = BB;
1108}
1109
1110CatchReturnInst::CatchReturnInst(const CatchReturnInst &CRI)
1111 : Instruction(Type::getVoidTy(C&: CRI.getContext()), Instruction::CatchRet,
1112 AllocMarker) {
1113 Op<0>() = CRI.Op<0>();
1114 Op<1>() = CRI.Op<1>();
1115}
1116
1117CatchReturnInst::CatchReturnInst(Value *CatchPad, BasicBlock *BB,
1118 InsertPosition InsertBefore)
1119 : Instruction(Type::getVoidTy(C&: BB->getContext()), Instruction::CatchRet,
1120 AllocMarker, InsertBefore) {
1121 init(CatchPad, BB);
1122}
1123
1124//===----------------------------------------------------------------------===//
1125// CatchSwitchInst Implementation
1126//===----------------------------------------------------------------------===//
1127
1128CatchSwitchInst::CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
1129 unsigned NumReservedValues,
1130 const Twine &NameStr,
1131 InsertPosition InsertBefore)
1132 : Instruction(ParentPad->getType(), Instruction::CatchSwitch, AllocMarker,
1133 InsertBefore) {
1134 if (UnwindDest)
1135 ++NumReservedValues;
1136 init(ParentPad, UnwindDest, NumReserved: NumReservedValues + 1);
1137 setName(NameStr);
1138}
1139
1140CatchSwitchInst::CatchSwitchInst(const CatchSwitchInst &CSI)
1141 : Instruction(CSI.getType(), Instruction::CatchSwitch, AllocMarker) {
1142 NumUserOperands = CSI.NumUserOperands;
1143 init(ParentPad: CSI.getParentPad(), UnwindDest: CSI.getUnwindDest(), NumReserved: CSI.getNumOperands());
1144 setNumHungOffUseOperands(ReservedSpace);
1145 Use *OL = getOperandList();
1146 const Use *InOL = CSI.getOperandList();
1147 for (unsigned I = 1, E = ReservedSpace; I != E; ++I)
1148 OL[I] = InOL[I];
1149}
1150
1151void CatchSwitchInst::init(Value *ParentPad, BasicBlock *UnwindDest,
1152 unsigned NumReservedValues) {
1153 assert(ParentPad && NumReservedValues);
1154
1155 ReservedSpace = NumReservedValues;
1156 allocHungoffUses(N: ReservedSpace);
1157 setNumHungOffUseOperands(UnwindDest ? 2 : 1);
1158
1159 Op<0>() = ParentPad;
1160 if (UnwindDest) {
1161 setSubclassData<UnwindDestField>(true);
1162 setUnwindDest(UnwindDest);
1163 }
1164}
1165
1166/// growOperands - grow operands - This grows the operand list in response to a
1167/// push_back style of operation. This grows the number of ops by 2 times.
1168void CatchSwitchInst::growOperands(unsigned Size) {
1169 unsigned NumOperands = getNumOperands();
1170 assert(NumOperands >= 1);
1171 if (ReservedSpace >= NumOperands + Size)
1172 return;
1173 ReservedSpace = (NumOperands + Size / 2) * 2;
1174 growHungoffUses(N: ReservedSpace);
1175}
1176
1177void CatchSwitchInst::addHandler(BasicBlock *Handler) {
1178 unsigned OpNo = getNumOperands();
1179 growOperands(Size: 1);
1180 assert(OpNo < ReservedSpace && "Growing didn't work!");
1181 setNumHungOffUseOperands(getNumOperands() + 1);
1182 getOperandList()[OpNo] = Handler;
1183}
1184
1185void CatchSwitchInst::removeHandler(handler_iterator HI) {
1186 // Move all subsequent handlers up one.
1187 Use *EndDst = op_end() - 1;
1188 for (Use *CurDst = HI.getCurrent(); CurDst != EndDst; ++CurDst)
1189 *CurDst = *(CurDst + 1);
1190 // Null out the last handler use.
1191 *EndDst = nullptr;
1192
1193 setNumHungOffUseOperands(getNumOperands() - 1);
1194}
1195
1196//===----------------------------------------------------------------------===//
1197// FuncletPadInst Implementation
1198//===----------------------------------------------------------------------===//
1199void FuncletPadInst::init(Value *ParentPad, ArrayRef<Value *> Args,
1200 const Twine &NameStr) {
1201 assert(getNumOperands() == 1 + Args.size() && "NumOperands not set up?");
1202 llvm::copy(Range&: Args, Out: op_begin());
1203 setParentPad(ParentPad);
1204 setName(NameStr);
1205}
1206
1207FuncletPadInst::FuncletPadInst(const FuncletPadInst &FPI, AllocInfo AllocInfo)
1208 : Instruction(FPI.getType(), FPI.getOpcode(), AllocInfo) {
1209 assert(getNumOperands() == FPI.getNumOperands() &&
1210 "Wrong number of operands allocated");
1211 std::copy(first: FPI.op_begin(), last: FPI.op_end(), result: op_begin());
1212 setParentPad(FPI.getParentPad());
1213}
1214
1215FuncletPadInst::FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
1216 ArrayRef<Value *> Args, AllocInfo AllocInfo,
1217 const Twine &NameStr,
1218 InsertPosition InsertBefore)
1219 : Instruction(ParentPad->getType(), Op, AllocInfo, InsertBefore) {
1220 init(ParentPad, Args, NameStr);
1221}
1222
1223//===----------------------------------------------------------------------===//
1224// UnreachableInst Implementation
1225//===----------------------------------------------------------------------===//
1226
1227UnreachableInst::UnreachableInst(LLVMContext &Context,
1228 InsertPosition InsertBefore)
1229 : Instruction(Type::getVoidTy(C&: Context), Instruction::Unreachable,
1230 AllocMarker, InsertBefore) {}
1231
1232//===----------------------------------------------------------------------===//
1233// UncondBrInst Implementation
1234//===----------------------------------------------------------------------===//
1235
1236UncondBrInst::UncondBrInst(BasicBlock *Target, InsertPosition InsertBefore)
1237 : Instruction(Type::getVoidTy(C&: Target->getContext()), Instruction::UncondBr,
1238 AllocMarker, InsertBefore) {
1239 Op<-1>() = Target;
1240}
1241
1242UncondBrInst::UncondBrInst(const UncondBrInst &BI)
1243 : Instruction(Type::getVoidTy(C&: BI.getContext()), Instruction::UncondBr,
1244 AllocMarker) {
1245 Op<-1>() = BI.Op<-1>();
1246 SubclassOptionalData = BI.SubclassOptionalData;
1247}
1248
1249//===----------------------------------------------------------------------===//
1250// CondBrInst Implementation
1251//===----------------------------------------------------------------------===//
1252
1253void CondBrInst::AssertOK() {
1254 assert(getCondition()->getType()->isIntegerTy(1) &&
1255 "May only branch on boolean predicates!");
1256}
1257
1258CondBrInst::CondBrInst(Value *Cond, BasicBlock *IfTrue, BasicBlock *IfFalse,
1259 InsertPosition InsertBefore)
1260 : Instruction(Type::getVoidTy(C&: IfTrue->getContext()), Instruction::CondBr,
1261 AllocMarker, InsertBefore) {
1262 // Assign in order of operand index to make use-list order predictable.
1263 Op<-3>() = Cond;
1264 Op<-2>() = IfTrue;
1265 Op<-1>() = IfFalse;
1266#ifndef NDEBUG
1267 AssertOK();
1268#endif
1269}
1270
1271CondBrInst::CondBrInst(const CondBrInst &BI)
1272 : Instruction(Type::getVoidTy(C&: BI.getContext()), Instruction::CondBr,
1273 AllocMarker) {
1274 // Assign in order of operand index to make use-list order predictable.
1275 Op<-3>() = BI.Op<-3>();
1276 Op<-2>() = BI.Op<-2>();
1277 Op<-1>() = BI.Op<-1>();
1278 SubclassOptionalData = BI.SubclassOptionalData;
1279}
1280
1281void CondBrInst::swapSuccessors() {
1282 Op<-1>().swap(RHS&: Op<-2>());
1283
1284 // Update profile metadata if present and it matches our structural
1285 // expectations.
1286 swapProfMetadata();
1287}
1288
1289//===----------------------------------------------------------------------===//
1290// AllocaInst Implementation
1291//===----------------------------------------------------------------------===//
1292
1293static Value *getAISize(LLVMContext &Context, Value *Amt) {
1294 if (!Amt)
1295 Amt = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 1);
1296 else {
1297 assert(!isa<BasicBlock>(Amt) &&
1298 "Passed basic block into allocation size parameter! Use other ctor");
1299 assert(Amt->getType()->isIntegerTy() &&
1300 "Allocation array size is not an integer!");
1301 }
1302 return Amt;
1303}
1304
1305static Align computeAllocaDefaultAlign(Type *Ty, InsertPosition Pos) {
1306 assert(Pos.isValid() &&
1307 "Insertion position cannot be null when alignment not provided!");
1308 BasicBlock *BB = Pos.getBasicBlock();
1309 assert(BB->getParent() &&
1310 "BB must be in a Function when alignment not provided!");
1311 const DataLayout &DL = BB->getDataLayout();
1312 return DL.getPrefTypeAlign(Ty);
1313}
1314
1315AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,
1316 InsertPosition InsertBefore)
1317 : AllocaInst(Ty, AddrSpace, /*ArraySize=*/nullptr, Name, InsertBefore) {}
1318
1319AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
1320 const Twine &Name, InsertPosition InsertBefore)
1321 : AllocaInst(Ty, AddrSpace, ArraySize,
1322 computeAllocaDefaultAlign(Ty, Pos: InsertBefore), Name,
1323 InsertBefore) {}
1324
1325AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
1326 Align Align, const Twine &Name,
1327 InsertPosition InsertBefore)
1328 : UnaryInstruction(PointerType::get(C&: Ty->getContext(), AddressSpace: AddrSpace), Alloca,
1329 getAISize(Context&: Ty->getContext(), Amt: ArraySize), InsertBefore),
1330 AllocatedType(Ty) {
1331 setAlignment(Align);
1332 assert(!Ty->isVoidTy() && "Cannot allocate void!");
1333 setName(Name);
1334}
1335
1336bool AllocaInst::isArrayAllocation() const {
1337 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: getOperand(i_nocapture: 0)))
1338 return !CI->isOne();
1339 return true;
1340}
1341
1342/// isStaticAlloca - Return true if this alloca is in the entry block of the
1343/// function and is a constant size. If so, the code generator will fold it
1344/// into the prolog/epilog code, so it is basically free.
1345bool AllocaInst::isStaticAlloca() const {
1346 // Must be constant size.
1347 if (!isa<ConstantInt>(Val: getArraySize())) return false;
1348
1349 // Must be in the entry block.
1350 const BasicBlock *Parent = getParent();
1351 return Parent->isEntryBlock() && !isUsedWithInAlloca();
1352}
1353
1354//===----------------------------------------------------------------------===//
1355// LoadInst Implementation
1356//===----------------------------------------------------------------------===//
1357
1358void LoadInst::AssertOK() {
1359 assert(getOperand(0)->getType()->isPointerTy() &&
1360 "Ptr must have pointer type.");
1361}
1362
1363static Align computeLoadStoreDefaultAlign(Type *Ty, InsertPosition Pos) {
1364 assert(Pos.isValid() &&
1365 "Insertion position cannot be null when alignment not provided!");
1366 BasicBlock *BB = Pos.getBasicBlock();
1367 assert(BB->getParent() &&
1368 "BB must be in a Function when alignment not provided!");
1369 const DataLayout &DL = BB->getDataLayout();
1370 return DL.getABITypeAlign(Ty);
1371}
1372
1373LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
1374 InsertPosition InsertBef)
1375 : LoadInst(Ty, Ptr, Name, /*isVolatile=*/false, InsertBef) {}
1376
1377LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
1378 InsertPosition InsertBef)
1379 : LoadInst(Ty, Ptr, Name, isVolatile,
1380 computeLoadStoreDefaultAlign(Ty, Pos: InsertBef), InsertBef) {}
1381
1382LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
1383 Align Align, InsertPosition InsertBef)
1384 : LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
1385 SyncScope::System, InsertBef) {}
1386
1387LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
1388 const LoadStoreInstProperties &Props,
1389 InsertPosition InsertBef)
1390 : LoadInst(Ty, Ptr, Name, Props.IsVolatile, Props.Alignment, Props.Ordering,
1391 Props.SSID, InsertBef) {
1392 setElementwise(Props.IsElementwise);
1393}
1394
1395LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
1396 Align Align, AtomicOrdering Order, SyncScope::ID SSID,
1397 InsertPosition InsertBef)
1398 : UnaryInstruction(Ty, Load, Ptr, InsertBef) {
1399 setVolatile(isVolatile);
1400 setAlignment(Align);
1401 setAtomic(Ordering: Order, SSID);
1402 AssertOK();
1403 setName(Name);
1404}
1405
1406//===----------------------------------------------------------------------===//
1407// StoreInst Implementation
1408//===----------------------------------------------------------------------===//
1409
1410void StoreInst::AssertOK() {
1411 assert(getOperand(0) && getOperand(1) && "Both operands must be non-null!");
1412 assert(getOperand(1)->getType()->isPointerTy() &&
1413 "Ptr must have pointer type!");
1414}
1415
1416StoreInst::StoreInst(Value *val, Value *addr, InsertPosition InsertBefore)
1417 : StoreInst(val, addr, /*isVolatile=*/false, InsertBefore) {}
1418
1419StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,
1420 InsertPosition InsertBefore)
1421 : StoreInst(val, addr, isVolatile,
1422 computeLoadStoreDefaultAlign(Ty: val->getType(), Pos: InsertBefore),
1423 InsertBefore) {}
1424
1425StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align,
1426 InsertPosition InsertBefore)
1427 : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic,
1428 SyncScope::System, InsertBefore) {}
1429
1430StoreInst::StoreInst(Value *Val, Value *Ptr,
1431 const LoadStoreInstProperties &Props,
1432 InsertPosition InsertBefore)
1433 : StoreInst(Val, Ptr, Props.IsVolatile, Props.Alignment, Props.Ordering,
1434 Props.SSID, InsertBefore) {
1435 setElementwise(Props.IsElementwise);
1436}
1437
1438StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align,
1439 AtomicOrdering Order, SyncScope::ID SSID,
1440 InsertPosition InsertBefore)
1441 : Instruction(Type::getVoidTy(C&: val->getContext()), Store, AllocMarker,
1442 InsertBefore) {
1443 Op<0>() = val;
1444 Op<1>() = addr;
1445 setVolatile(isVolatile);
1446 setAlignment(Align);
1447 setAtomic(Ordering: Order, SSID);
1448 AssertOK();
1449}
1450
1451//===----------------------------------------------------------------------===//
1452// AtomicCmpXchgInst Implementation
1453//===----------------------------------------------------------------------===//
1454
1455void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal,
1456 Align Alignment, AtomicOrdering SuccessOrdering,
1457 AtomicOrdering FailureOrdering,
1458 SyncScope::ID SSID) {
1459 Op<0>() = Ptr;
1460 Op<1>() = Cmp;
1461 Op<2>() = NewVal;
1462 setSuccessOrdering(SuccessOrdering);
1463 setFailureOrdering(FailureOrdering);
1464 setSyncScopeID(SSID);
1465 setAlignment(Alignment);
1466
1467 assert(getOperand(0) && getOperand(1) && getOperand(2) &&
1468 "All operands must be non-null!");
1469 assert(getOperand(0)->getType()->isPointerTy() &&
1470 "Ptr must have pointer type!");
1471 assert(getOperand(1)->getType() == getOperand(2)->getType() &&
1472 "Cmp type and NewVal type must be same!");
1473}
1474
1475AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
1476 Align Alignment,
1477 AtomicOrdering SuccessOrdering,
1478 AtomicOrdering FailureOrdering,
1479 SyncScope::ID SSID,
1480 InsertPosition InsertBefore)
1481 : Instruction(
1482 StructType::get(elt1: Cmp->getType(), elts: Type::getInt1Ty(C&: Cmp->getContext())),
1483 AtomicCmpXchg, AllocMarker, InsertBefore) {
1484 Init(Ptr, Cmp, NewVal, Alignment, SuccessOrdering, FailureOrdering, SSID);
1485}
1486
1487//===----------------------------------------------------------------------===//
1488// AtomicRMWInst Implementation
1489//===----------------------------------------------------------------------===//
1490
1491void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
1492 Align Alignment, AtomicOrdering Ordering,
1493 SyncScope::ID SSID, bool Elementwise) {
1494 assert(Ordering != AtomicOrdering::NotAtomic &&
1495 "atomicrmw instructions can only be atomic.");
1496 assert(Ordering != AtomicOrdering::Unordered &&
1497 "atomicrmw instructions cannot be unordered.");
1498 Op<0>() = Ptr;
1499 Op<1>() = Val;
1500 setOperation(Operation);
1501 setOrdering(Ordering);
1502 setSyncScopeID(SSID);
1503 setElementwise(Elementwise);
1504 setAlignment(Alignment);
1505
1506 assert(getOperand(0) && getOperand(1) && "All operands must be non-null!");
1507 assert(getOperand(0)->getType()->isPointerTy() &&
1508 "Ptr must have pointer type!");
1509 assert(Ordering != AtomicOrdering::NotAtomic &&
1510 "AtomicRMW instructions must be atomic!");
1511}
1512
1513AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
1514 Align Alignment, AtomicOrdering Ordering,
1515 SyncScope::ID SSID, bool Elementwise,
1516 InsertPosition InsertBefore)
1517 : Instruction(Val->getType(), AtomicRMW, AllocMarker, InsertBefore) {
1518 Init(Operation, Ptr, Val, Alignment, Ordering, SSID, Elementwise);
1519}
1520
1521StringRef AtomicRMWInst::getOperationName(BinOp Op) {
1522 switch (Op) {
1523 case AtomicRMWInst::Xchg:
1524 return "xchg";
1525 case AtomicRMWInst::Add:
1526 return "add";
1527 case AtomicRMWInst::Sub:
1528 return "sub";
1529 case AtomicRMWInst::And:
1530 return "and";
1531 case AtomicRMWInst::Nand:
1532 return "nand";
1533 case AtomicRMWInst::Or:
1534 return "or";
1535 case AtomicRMWInst::Xor:
1536 return "xor";
1537 case AtomicRMWInst::Max:
1538 return "max";
1539 case AtomicRMWInst::Min:
1540 return "min";
1541 case AtomicRMWInst::UMax:
1542 return "umax";
1543 case AtomicRMWInst::UMin:
1544 return "umin";
1545 case AtomicRMWInst::FAdd:
1546 return "fadd";
1547 case AtomicRMWInst::FSub:
1548 return "fsub";
1549 case AtomicRMWInst::FMax:
1550 return "fmax";
1551 case AtomicRMWInst::FMin:
1552 return "fmin";
1553 case AtomicRMWInst::FMaximum:
1554 return "fmaximum";
1555 case AtomicRMWInst::FMinimum:
1556 return "fminimum";
1557 case AtomicRMWInst::FMaximumNum:
1558 return "fmaximumnum";
1559 case AtomicRMWInst::FMinimumNum:
1560 return "fminimumnum";
1561 case AtomicRMWInst::UIncWrap:
1562 return "uinc_wrap";
1563 case AtomicRMWInst::UDecWrap:
1564 return "udec_wrap";
1565 case AtomicRMWInst::USubCond:
1566 return "usub_cond";
1567 case AtomicRMWInst::USubSat:
1568 return "usub_sat";
1569 case AtomicRMWInst::BAD_BINOP:
1570 return "<invalid operation>";
1571 }
1572
1573 llvm_unreachable("invalid atomicrmw operation");
1574}
1575
1576//===----------------------------------------------------------------------===//
1577// FenceInst Implementation
1578//===----------------------------------------------------------------------===//
1579
1580FenceInst::FenceInst(LLVMContext &C, AtomicOrdering Ordering,
1581 SyncScope::ID SSID, InsertPosition InsertBefore)
1582 : Instruction(Type::getVoidTy(C), Fence, AllocMarker, InsertBefore) {
1583 setOrdering(Ordering);
1584 setSyncScopeID(SSID);
1585}
1586
1587//===----------------------------------------------------------------------===//
1588// GetElementPtrInst Implementation
1589//===----------------------------------------------------------------------===//
1590
1591void GetElementPtrInst::init(Value *Ptr, ArrayRef<Value *> IdxList,
1592 const Twine &Name) {
1593 assert(getNumOperands() == 1 + IdxList.size() &&
1594 "NumOperands not initialized?");
1595 Op<0>() = Ptr;
1596 llvm::copy(Range&: IdxList, Out: op_begin() + 1);
1597 setName(Name);
1598}
1599
1600GetElementPtrInst::GetElementPtrInst(const GetElementPtrInst &GEPI,
1601 AllocInfo AllocInfo)
1602 : Instruction(GEPI.getType(), GetElementPtr, AllocInfo),
1603 SourceElementType(GEPI.SourceElementType),
1604 ResultElementType(GEPI.ResultElementType) {
1605 assert(getNumOperands() == GEPI.getNumOperands() &&
1606 "Wrong number of operands allocated");
1607 std::copy(first: GEPI.op_begin(), last: GEPI.op_end(), result: op_begin());
1608 SubclassOptionalData = GEPI.SubclassOptionalData;
1609}
1610
1611Type *GetElementPtrInst::getTypeAtIndex(Type *Ty, Value *Idx) {
1612 if (auto *Struct = dyn_cast<StructType>(Val: Ty)) {
1613 if (!Struct->indexValid(V: Idx))
1614 return nullptr;
1615 return Struct->getTypeAtIndex(V: Idx);
1616 }
1617 if (!Idx->getType()->isIntOrIntVectorTy())
1618 return nullptr;
1619 if (auto *Array = dyn_cast<ArrayType>(Val: Ty))
1620 return Array->getElementType();
1621 if (auto *Vector = dyn_cast<VectorType>(Val: Ty))
1622 return Vector->getElementType();
1623 return nullptr;
1624}
1625
1626Type *GetElementPtrInst::getTypeAtIndex(Type *Ty, uint64_t Idx) {
1627 if (auto *Struct = dyn_cast<StructType>(Val: Ty)) {
1628 if (Idx >= Struct->getNumElements())
1629 return nullptr;
1630 return Struct->getElementType(N: Idx);
1631 }
1632 if (auto *Array = dyn_cast<ArrayType>(Val: Ty))
1633 return Array->getElementType();
1634 if (auto *Vector = dyn_cast<VectorType>(Val: Ty))
1635 return Vector->getElementType();
1636 return nullptr;
1637}
1638
1639template <typename IndexTy>
1640static Type *getIndexedTypeInternal(Type *Ty, ArrayRef<IndexTy> IdxList) {
1641 if (IdxList.empty())
1642 return Ty;
1643 for (IndexTy V : IdxList.slice(1)) {
1644 Ty = GetElementPtrInst::getTypeAtIndex(Ty, V);
1645 if (!Ty)
1646 return Ty;
1647 }
1648 return Ty;
1649}
1650
1651Type *GetElementPtrInst::getIndexedType(Type *Ty, ArrayRef<Value *> IdxList) {
1652 return getIndexedTypeInternal(Ty, IdxList);
1653}
1654
1655Type *GetElementPtrInst::getIndexedType(Type *Ty,
1656 ArrayRef<Constant *> IdxList) {
1657 return getIndexedTypeInternal(Ty, IdxList);
1658}
1659
1660Type *GetElementPtrInst::getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList) {
1661 return getIndexedTypeInternal(Ty, IdxList);
1662}
1663
1664/// hasAllZeroIndices - Return true if all of the indices of this GEP are
1665/// zeros. If so, the result pointer and the first operand have the same
1666/// value, just potentially different types.
1667bool GetElementPtrInst::hasAllZeroIndices() const {
1668 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) {
1669 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: getOperand(i_nocapture: i))) {
1670 if (!CI->isZero()) return false;
1671 } else {
1672 return false;
1673 }
1674 }
1675 return true;
1676}
1677
1678/// hasAllConstantIndices - Return true if all of the indices of this GEP are
1679/// constant integers. If so, the result pointer and the first operand have
1680/// a constant offset between them.
1681bool GetElementPtrInst::hasAllConstantIndices() const {
1682 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) {
1683 if (!isa<ConstantInt>(Val: getOperand(i_nocapture: i)))
1684 return false;
1685 }
1686 return true;
1687}
1688
1689void GetElementPtrInst::setNoWrapFlags(GEPNoWrapFlags NW) {
1690 SubclassOptionalData = NW.getRaw();
1691}
1692
1693void GetElementPtrInst::setIsInBounds(bool B) {
1694 GEPNoWrapFlags NW = cast<GEPOperator>(Val: this)->getNoWrapFlags();
1695 if (B)
1696 NW |= GEPNoWrapFlags::inBounds();
1697 else
1698 NW = NW.withoutInBounds();
1699 setNoWrapFlags(NW);
1700}
1701
1702GEPNoWrapFlags GetElementPtrInst::getNoWrapFlags() const {
1703 return cast<GEPOperator>(Val: this)->getNoWrapFlags();
1704}
1705
1706bool GetElementPtrInst::isInBounds() const {
1707 return cast<GEPOperator>(Val: this)->isInBounds();
1708}
1709
1710bool GetElementPtrInst::hasNoUnsignedSignedWrap() const {
1711 return cast<GEPOperator>(Val: this)->hasNoUnsignedSignedWrap();
1712}
1713
1714bool GetElementPtrInst::hasNoUnsignedWrap() const {
1715 return cast<GEPOperator>(Val: this)->hasNoUnsignedWrap();
1716}
1717
1718bool GetElementPtrInst::accumulateConstantOffset(const DataLayout &DL,
1719 APInt &Offset) const {
1720 // Delegate to the generic GEPOperator implementation.
1721 return cast<GEPOperator>(Val: this)->accumulateConstantOffset(DL, Offset);
1722}
1723
1724bool GetElementPtrInst::collectOffset(
1725 const DataLayout &DL, unsigned BitWidth,
1726 SmallMapVector<Value *, APInt, 4> &VariableOffsets,
1727 APInt &ConstantOffset) const {
1728 // Delegate to the generic GEPOperator implementation.
1729 return cast<GEPOperator>(Val: this)->collectOffset(DL, BitWidth, VariableOffsets,
1730 ConstantOffset);
1731}
1732
1733//===----------------------------------------------------------------------===//
1734// ExtractElementInst Implementation
1735//===----------------------------------------------------------------------===//
1736
1737ExtractElementInst::ExtractElementInst(Value *Val, Value *Index,
1738 const Twine &Name,
1739 InsertPosition InsertBef)
1740 : Instruction(cast<VectorType>(Val: Val->getType())->getElementType(),
1741 ExtractElement, AllocMarker, InsertBef) {
1742 assert(isValidOperands(Val, Index) &&
1743 "Invalid extractelement instruction operands!");
1744 Op<0>() = Val;
1745 Op<1>() = Index;
1746 setName(Name);
1747}
1748
1749bool ExtractElementInst::isValidOperands(const Value *Val, const Value *Index) {
1750 if (!Val->getType()->isVectorTy() || !Index->getType()->isIntegerTy())
1751 return false;
1752 return true;
1753}
1754
1755//===----------------------------------------------------------------------===//
1756// InsertElementInst Implementation
1757//===----------------------------------------------------------------------===//
1758
1759InsertElementInst::InsertElementInst(Value *Vec, Value *Elt, Value *Index,
1760 const Twine &Name,
1761 InsertPosition InsertBef)
1762 : Instruction(Vec->getType(), InsertElement, AllocMarker, InsertBef) {
1763 assert(isValidOperands(Vec, Elt, Index) &&
1764 "Invalid insertelement instruction operands!");
1765 Op<0>() = Vec;
1766 Op<1>() = Elt;
1767 Op<2>() = Index;
1768 setName(Name);
1769}
1770
1771bool InsertElementInst::isValidOperands(const Value *Vec, const Value *Elt,
1772 const Value *Index) {
1773 if (!Vec->getType()->isVectorTy())
1774 return false; // First operand of insertelement must be vector type.
1775
1776 if (Elt->getType() != cast<VectorType>(Val: Vec->getType())->getElementType())
1777 return false;// Second operand of insertelement must be vector element type.
1778
1779 if (!Index->getType()->isIntegerTy())
1780 return false; // Third operand of insertelement must be an integer.
1781 return true;
1782}
1783
1784//===----------------------------------------------------------------------===//
1785// ShuffleVectorInst Implementation
1786//===----------------------------------------------------------------------===//
1787
1788static Value *createPlaceholderForShuffleVector(Value *V) {
1789 assert(V && "Cannot create placeholder of nullptr V");
1790 return PoisonValue::get(T: V->getType());
1791}
1792
1793ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *Mask, const Twine &Name,
1794 InsertPosition InsertBefore)
1795 : ShuffleVectorInst(V1, createPlaceholderForShuffleVector(V: V1), Mask, Name,
1796 InsertBefore) {}
1797
1798ShuffleVectorInst::ShuffleVectorInst(Value *V1, ArrayRef<int> Mask,
1799 const Twine &Name,
1800 InsertPosition InsertBefore)
1801 : ShuffleVectorInst(V1, createPlaceholderForShuffleVector(V: V1), Mask, Name,
1802 InsertBefore) {}
1803
1804ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
1805 const Twine &Name,
1806 InsertPosition InsertBefore)
1807 : Instruction(
1808 VectorType::get(ElementType: cast<VectorType>(Val: V1->getType())->getElementType(),
1809 EC: cast<VectorType>(Val: Mask->getType())->getElementCount()),
1810 ShuffleVector, AllocMarker, InsertBefore) {
1811 assert(isValidOperands(V1, V2, Mask) &&
1812 "Invalid shuffle vector instruction operands!");
1813
1814 Op<0>() = V1;
1815 Op<1>() = V2;
1816 SmallVector<int, 16> MaskArr;
1817 getShuffleMask(Mask: cast<Constant>(Val: Mask), Result&: MaskArr);
1818 setShuffleMask(MaskArr);
1819 setName(Name);
1820}
1821
1822ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
1823 const Twine &Name,
1824 InsertPosition InsertBefore)
1825 : Instruction(
1826 VectorType::get(ElementType: cast<VectorType>(Val: V1->getType())->getElementType(),
1827 NumElements: Mask.size(), Scalable: isa<ScalableVectorType>(Val: V1->getType())),
1828 ShuffleVector, AllocMarker, InsertBefore) {
1829 assert(isValidOperands(V1, V2, Mask) &&
1830 "Invalid shuffle vector instruction operands!");
1831 Op<0>() = V1;
1832 Op<1>() = V2;
1833 setShuffleMask(Mask);
1834 setName(Name);
1835}
1836
1837void ShuffleVectorInst::commute() {
1838 int NumOpElts = cast<FixedVectorType>(Val: Op<0>()->getType())->getNumElements();
1839 int NumMaskElts = ShuffleMask.size();
1840 SmallVector<int, 16> NewMask(NumMaskElts);
1841 for (int i = 0; i != NumMaskElts; ++i) {
1842 int MaskElt = getMaskValue(Elt: i);
1843 if (MaskElt == PoisonMaskElem) {
1844 NewMask[i] = PoisonMaskElem;
1845 continue;
1846 }
1847 assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
1848 MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts;
1849 NewMask[i] = MaskElt;
1850 }
1851 setShuffleMask(NewMask);
1852 Op<0>().swap(RHS&: Op<1>());
1853}
1854
1855bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
1856 ArrayRef<int> Mask) {
1857 // V1 and V2 must be vectors of the same type.
1858 if (!isa<VectorType>(Val: V1->getType()) || V1->getType() != V2->getType())
1859 return false;
1860
1861 if (isa<ScalableVectorType>(Val: V1->getType()))
1862 return (Mask[0] == 0 || Mask[0] == PoisonMaskElem) && all_equal(Range&: Mask);
1863
1864 // Make sure the mask elements make sense.
1865 int64_t MaxMaskSz =
1866 cast<FixedVectorType>(Val: V1->getType())->getNumElements() * 2LL;
1867 for (int Elem : Mask)
1868 if (Elem != PoisonMaskElem && Elem >= MaxMaskSz)
1869 return false;
1870 return true;
1871}
1872
1873bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
1874 const Value *Mask) {
1875 // V1 and V2 must be vectors of the same type.
1876 if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
1877 return false;
1878
1879 // Mask must be vector of i32, and must be the same kind of vector as the
1880 // input vectors
1881 auto *MaskTy = dyn_cast<VectorType>(Val: Mask->getType());
1882 if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(BitWidth: 32) ||
1883 isa<ScalableVectorType>(Val: MaskTy) != isa<ScalableVectorType>(Val: V1->getType()))
1884 return false;
1885
1886 // Check to see if Mask is valid.
1887 if (isa<UndefValue>(Val: Mask) || isa<ConstantAggregateZero>(Val: Mask))
1888 return true;
1889
1890 // NOTE: Through vector ConstantInt we have the potential to support more
1891 // than just zero splat masks but that requires a LangRef change.
1892 if (isa<ScalableVectorType>(Val: MaskTy))
1893 return false;
1894
1895 unsigned V1Size = cast<FixedVectorType>(Val: V1->getType())->getNumElements();
1896
1897 if (const auto *CI = dyn_cast<ConstantInt>(Val: Mask))
1898 return !CI->uge(Num: V1Size * 2);
1899
1900 if (const auto *MV = dyn_cast<ConstantVector>(Val: Mask)) {
1901 for (Value *Op : MV->operands()) {
1902 if (auto *CI = dyn_cast<ConstantInt>(Val: Op)) {
1903 if (CI->uge(Num: V1Size*2))
1904 return false;
1905 } else if (!isa<UndefValue>(Val: Op)) {
1906 return false;
1907 }
1908 }
1909 return true;
1910 }
1911
1912 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: Mask)) {
1913 for (unsigned i = 0, e = cast<FixedVectorType>(Val: MaskTy)->getNumElements();
1914 i != e; ++i)
1915 if (CDS->getElementAsInteger(i) >= V1Size*2)
1916 return false;
1917 return true;
1918 }
1919
1920 return false;
1921}
1922
1923void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
1924 SmallVectorImpl<int> &Result) {
1925 ElementCount EC = cast<VectorType>(Val: Mask->getType())->getElementCount();
1926
1927 if (isa<ConstantAggregateZero>(Val: Mask) || isa<UndefValue>(Val: Mask)) {
1928 int MaskVal = isa<UndefValue>(Val: Mask) ? -1 : 0;
1929 Result.append(NumInputs: EC.getKnownMinValue(), Elt: MaskVal);
1930 return;
1931 }
1932
1933 assert(!EC.isScalable() &&
1934 "Scalable vector shuffle mask must be undef or zeroinitializer");
1935
1936 unsigned NumElts = EC.getFixedValue();
1937
1938 Result.reserve(N: NumElts);
1939
1940 if (auto *CDS = dyn_cast<ConstantDataSequential>(Val: Mask)) {
1941 for (unsigned i = 0; i != NumElts; ++i)
1942 Result.push_back(Elt: CDS->getElementAsInteger(i));
1943 return;
1944 }
1945 for (unsigned i = 0; i != NumElts; ++i) {
1946 Constant *C = Mask->getAggregateElement(Elt: i);
1947 Result.push_back(Elt: isa<UndefValue>(Val: C) ? -1 :
1948 cast<ConstantInt>(Val: C)->getZExtValue());
1949 }
1950}
1951
1952void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
1953 ShuffleMask.assign(in_start: Mask.begin(), in_end: Mask.end());
1954 ShuffleMaskForBitcode = convertShuffleMaskForBitcode(Mask, ResultTy: getType());
1955}
1956
1957Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
1958 Type *ResultTy) {
1959 Type *Int32Ty = Type::getInt32Ty(C&: ResultTy->getContext());
1960 if (isa<ScalableVectorType>(Val: ResultTy)) {
1961 assert(all_equal(Mask) && "Unexpected shuffle");
1962 Type *VecTy = VectorType::get(ElementType: Int32Ty, NumElements: Mask.size(), Scalable: true);
1963 if (Mask[0] == 0)
1964 return Constant::getNullValue(Ty: VecTy);
1965 return PoisonValue::get(T: VecTy);
1966 }
1967 SmallVector<Constant *, 16> MaskConst;
1968 for (int Elem : Mask) {
1969 if (Elem == PoisonMaskElem)
1970 MaskConst.push_back(Elt: PoisonValue::get(T: Int32Ty));
1971 else
1972 MaskConst.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Elem));
1973 }
1974 return ConstantVector::get(V: MaskConst);
1975}
1976
1977static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int64_t NumOpElts) {
1978 assert(!Mask.empty() && "Shuffle mask must contain elements");
1979 bool UsesLHS = false;
1980 bool UsesRHS = false;
1981 for (int I : Mask) {
1982 if (I == -1)
1983 continue;
1984 assert(I >= 0 && I < (NumOpElts * 2) &&
1985 "Out-of-bounds shuffle mask element");
1986 UsesLHS |= (I < NumOpElts);
1987 UsesRHS |= (I >= NumOpElts);
1988 if (UsesLHS && UsesRHS)
1989 return false;
1990 }
1991 // Allow for degenerate case: completely undef mask means neither source is used.
1992 return UsesLHS || UsesRHS;
1993}
1994
1995bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts) {
1996 // We don't have vector operand size information, so assume operands are the
1997 // same size as the mask.
1998 return isSingleSourceMaskImpl(Mask, NumOpElts: NumSrcElts);
1999}
2000
2001static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
2002 if (!isSingleSourceMaskImpl(Mask, NumOpElts))
2003 return false;
2004 for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
2005 if (Mask[i] == -1)
2006 continue;
2007 if (Mask[i] != i && Mask[i] != (NumOpElts + i))
2008 return false;
2009 }
2010 return true;
2011}
2012
2013bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask, int NumSrcElts) {
2014 if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2015 return false;
2016 // We don't have vector operand size information, so assume operands are the
2017 // same size as the mask.
2018 return isIdentityMaskImpl(Mask, NumOpElts: NumSrcElts);
2019}
2020
2021bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask, int NumSrcElts) {
2022 if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2023 return false;
2024 if (!isSingleSourceMask(Mask, NumSrcElts))
2025 return false;
2026
2027 // The number of elements in the mask must be at least 2.
2028 if (NumSrcElts < 2)
2029 return false;
2030
2031 for (int I = 0, E = Mask.size(); I < E; ++I) {
2032 if (Mask[I] == -1)
2033 continue;
2034 if (Mask[I] != (NumSrcElts - 1 - I) &&
2035 Mask[I] != (NumSrcElts + NumSrcElts - 1 - I))
2036 return false;
2037 }
2038 return true;
2039}
2040
2041bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts) {
2042 if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2043 return false;
2044 if (!isSingleSourceMask(Mask, NumSrcElts))
2045 return false;
2046 for (int I = 0, E = Mask.size(); I < E; ++I) {
2047 if (Mask[I] == -1)
2048 continue;
2049 if (Mask[I] != 0 && Mask[I] != NumSrcElts)
2050 return false;
2051 }
2052 return true;
2053}
2054
2055bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask, int NumSrcElts) {
2056 if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2057 return false;
2058 // Select is differentiated from identity. It requires using both sources.
2059 if (isSingleSourceMask(Mask, NumSrcElts))
2060 return false;
2061 for (int I = 0, E = Mask.size(); I < E; ++I) {
2062 if (Mask[I] == -1)
2063 continue;
2064 if (Mask[I] != I && Mask[I] != (NumSrcElts + I))
2065 return false;
2066 }
2067 return true;
2068}
2069
2070bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask, int NumSrcElts) {
2071 // Example masks that will return true:
2072 // v1 = <a, b, c, d>
2073 // v2 = <e, f, g, h>
2074 // trn1 = shufflevector v1, v2 <0, 4, 2, 6> = <a, e, c, g>
2075 // trn2 = shufflevector v1, v2 <1, 5, 3, 7> = <b, f, d, h>
2076
2077 if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2078 return false;
2079 // 1. The number of elements in the mask must be a power-of-2 and at least 2.
2080 int Sz = Mask.size();
2081 if (Sz < 2 || !isPowerOf2_32(Value: Sz))
2082 return false;
2083
2084 // 2. The first element of the mask must be either a 0 or a 1.
2085 if (Mask[0] != 0 && Mask[0] != 1)
2086 return false;
2087
2088 // 3. The difference between the first 2 elements must be equal to the
2089 // number of elements in the mask.
2090 if ((Mask[1] - Mask[0]) != NumSrcElts)
2091 return false;
2092
2093 // 4. The difference between consecutive even-numbered and odd-numbered
2094 // elements must be equal to 2.
2095 for (int I = 2; I < Sz; ++I) {
2096 int MaskEltVal = Mask[I];
2097 if (MaskEltVal == -1)
2098 return false;
2099 int MaskEltPrevVal = Mask[I - 2];
2100 if (MaskEltVal - MaskEltPrevVal != 2)
2101 return false;
2102 }
2103 return true;
2104}
2105
2106bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int NumSrcElts,
2107 int &Index) {
2108 if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2109 return false;
2110 // Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
2111 int StartIndex = -1;
2112 for (int I = 0, E = Mask.size(); I != E; ++I) {
2113 int MaskEltVal = Mask[I];
2114 if (MaskEltVal == -1)
2115 continue;
2116
2117 if (StartIndex == -1) {
2118 // Don't support a StartIndex that begins in the second input, or if the
2119 // first non-undef index would access below the StartIndex.
2120 if (MaskEltVal < I || NumSrcElts <= (MaskEltVal - I))
2121 return false;
2122
2123 StartIndex = MaskEltVal - I;
2124 continue;
2125 }
2126
2127 // Splice is sequential starting from StartIndex.
2128 if (MaskEltVal != (StartIndex + I))
2129 return false;
2130 }
2131
2132 if (StartIndex == -1)
2133 return false;
2134
2135 // NOTE: This accepts StartIndex == 0 (COPY).
2136 Index = StartIndex;
2137 return true;
2138}
2139
2140bool ShuffleVectorInst::isExtractSubvectorMask(ArrayRef<int> Mask,
2141 int NumSrcElts, int &Index) {
2142 // Must extract from a single source.
2143 if (!isSingleSourceMaskImpl(Mask, NumOpElts: NumSrcElts))
2144 return false;
2145
2146 // Must be smaller (else this is an Identity shuffle).
2147 if (NumSrcElts <= (int)Mask.size())
2148 return false;
2149
2150 // Find start of extraction, accounting that we may start with an UNDEF.
2151 int SubIndex = -1;
2152 for (int i = 0, e = Mask.size(); i != e; ++i) {
2153 int M = Mask[i];
2154 if (M < 0)
2155 continue;
2156 int Offset = (M % NumSrcElts) - i;
2157 if (0 <= SubIndex && SubIndex != Offset)
2158 return false;
2159 SubIndex = Offset;
2160 }
2161
2162 if (0 <= SubIndex && SubIndex + (int)Mask.size() <= NumSrcElts) {
2163 Index = SubIndex;
2164 return true;
2165 }
2166 return false;
2167}
2168
2169bool ShuffleVectorInst::isInsertSubvectorMask(ArrayRef<int> Mask,
2170 int NumSrcElts, int &NumSubElts,
2171 int &Index) {
2172 int NumMaskElts = Mask.size();
2173
2174 // Don't try to match if we're shuffling to a smaller size.
2175 if (NumMaskElts < NumSrcElts)
2176 return false;
2177
2178 // TODO: We don't recognize self-insertion/widening.
2179 if (isSingleSourceMaskImpl(Mask, NumOpElts: NumSrcElts))
2180 return false;
2181
2182 // Determine which mask elements are attributed to which source.
2183 APInt UndefElts = APInt::getZero(numBits: NumMaskElts);
2184 APInt Src0Elts = APInt::getZero(numBits: NumMaskElts);
2185 APInt Src1Elts = APInt::getZero(numBits: NumMaskElts);
2186 bool Src0Identity = true;
2187 bool Src1Identity = true;
2188
2189 for (int i = 0; i != NumMaskElts; ++i) {
2190 int M = Mask[i];
2191 if (M < 0) {
2192 UndefElts.setBit(i);
2193 continue;
2194 }
2195 if (M < NumSrcElts) {
2196 Src0Elts.setBit(i);
2197 Src0Identity &= (M == i);
2198 continue;
2199 }
2200 Src1Elts.setBit(i);
2201 Src1Identity &= (M == (i + NumSrcElts));
2202 }
2203 assert((Src0Elts | Src1Elts | UndefElts).isAllOnes() &&
2204 "unknown shuffle elements");
2205 assert(!Src0Elts.isZero() && !Src1Elts.isZero() &&
2206 "2-source shuffle not found");
2207
2208 // Determine lo/hi span ranges.
2209 // TODO: How should we handle undefs at the start of subvector insertions?
2210 int Src0Lo = Src0Elts.countr_zero();
2211 int Src1Lo = Src1Elts.countr_zero();
2212 int Src0Hi = NumMaskElts - Src0Elts.countl_zero();
2213 int Src1Hi = NumMaskElts - Src1Elts.countl_zero();
2214
2215 // If src0 is in place, see if the src1 elements is inplace within its own
2216 // span.
2217 if (Src0Identity) {
2218 int NumSub1Elts = Src1Hi - Src1Lo;
2219 ArrayRef<int> Sub1Mask = Mask.slice(N: Src1Lo, M: NumSub1Elts);
2220 if (isIdentityMaskImpl(Mask: Sub1Mask, NumOpElts: NumSrcElts)) {
2221 NumSubElts = NumSub1Elts;
2222 Index = Src1Lo;
2223 return true;
2224 }
2225 }
2226
2227 // If src1 is in place, see if the src0 elements is inplace within its own
2228 // span.
2229 if (Src1Identity) {
2230 int NumSub0Elts = Src0Hi - Src0Lo;
2231 ArrayRef<int> Sub0Mask = Mask.slice(N: Src0Lo, M: NumSub0Elts);
2232 if (isIdentityMaskImpl(Mask: Sub0Mask, NumOpElts: NumSrcElts)) {
2233 NumSubElts = NumSub0Elts;
2234 Index = Src0Lo;
2235 return true;
2236 }
2237 }
2238
2239 return false;
2240}
2241
2242bool ShuffleVectorInst::isIdentityWithPadding() const {
2243 // FIXME: Not currently possible to express a shuffle mask for a scalable
2244 // vector for this case.
2245 if (isa<ScalableVectorType>(Val: getType()))
2246 return false;
2247
2248 int NumOpElts = cast<FixedVectorType>(Val: Op<0>()->getType())->getNumElements();
2249 int NumMaskElts = cast<FixedVectorType>(Val: getType())->getNumElements();
2250 if (NumMaskElts <= NumOpElts)
2251 return false;
2252
2253 // The first part of the mask must choose elements from exactly 1 source op.
2254 ArrayRef<int> Mask = getShuffleMask();
2255 if (!isIdentityMaskImpl(Mask, NumOpElts))
2256 return false;
2257
2258 // All extending must be with undef elements.
2259 for (int i = NumOpElts; i < NumMaskElts; ++i)
2260 if (Mask[i] != -1)
2261 return false;
2262
2263 return true;
2264}
2265
2266bool ShuffleVectorInst::isIdentityWithExtract() const {
2267 // FIXME: Not currently possible to express a shuffle mask for a scalable
2268 // vector for this case.
2269 if (isa<ScalableVectorType>(Val: getType()))
2270 return false;
2271
2272 int NumOpElts = cast<FixedVectorType>(Val: Op<0>()->getType())->getNumElements();
2273 int NumMaskElts = cast<FixedVectorType>(Val: getType())->getNumElements();
2274 if (NumMaskElts >= NumOpElts)
2275 return false;
2276
2277 return isIdentityMaskImpl(Mask: getShuffleMask(), NumOpElts);
2278}
2279
2280bool ShuffleVectorInst::isConcat() const {
2281 // Vector concatenation is differentiated from identity with padding.
2282 if (isa<UndefValue>(Val: Op<0>()) || isa<UndefValue>(Val: Op<1>()))
2283 return false;
2284
2285 // FIXME: Not currently possible to express a shuffle mask for a scalable
2286 // vector for this case.
2287 if (isa<ScalableVectorType>(Val: getType()))
2288 return false;
2289
2290 int NumOpElts = cast<FixedVectorType>(Val: Op<0>()->getType())->getNumElements();
2291 int NumMaskElts = cast<FixedVectorType>(Val: getType())->getNumElements();
2292 if (NumMaskElts != NumOpElts * 2)
2293 return false;
2294
2295 // Use the mask length rather than the operands' vector lengths here. We
2296 // already know that the shuffle returns a vector twice as long as the inputs,
2297 // and neither of the inputs are undef vectors. If the mask picks consecutive
2298 // elements from both inputs, then this is a concatenation of the inputs.
2299 return isIdentityMaskImpl(Mask: getShuffleMask(), NumOpElts: NumMaskElts);
2300}
2301
2302static bool isReplicationMaskWithParams(ArrayRef<int> Mask,
2303 int ReplicationFactor, int VF) {
2304 assert(Mask.size() == (unsigned)ReplicationFactor * VF &&
2305 "Unexpected mask size.");
2306
2307 for (int CurrElt : seq(Size: VF)) {
2308 ArrayRef<int> CurrSubMask = Mask.take_front(N: ReplicationFactor);
2309 assert(CurrSubMask.size() == (unsigned)ReplicationFactor &&
2310 "Run out of mask?");
2311 Mask = Mask.drop_front(N: ReplicationFactor);
2312 if (!all_of(Range&: CurrSubMask, P: [CurrElt](int MaskElt) {
2313 return MaskElt == PoisonMaskElem || MaskElt == CurrElt;
2314 }))
2315 return false;
2316 }
2317 assert(Mask.empty() && "Did not consume the whole mask?");
2318
2319 return true;
2320}
2321
2322bool ShuffleVectorInst::isReplicationMask(ArrayRef<int> Mask,
2323 int &ReplicationFactor, int &VF) {
2324 // undef-less case is trivial.
2325 if (!llvm::is_contained(Range&: Mask, Element: PoisonMaskElem)) {
2326 ReplicationFactor =
2327 Mask.take_while(Pred: [](int MaskElt) { return MaskElt == 0; }).size();
2328 if (ReplicationFactor == 0 || Mask.size() % ReplicationFactor != 0)
2329 return false;
2330 VF = Mask.size() / ReplicationFactor;
2331 return isReplicationMaskWithParams(Mask, ReplicationFactor, VF);
2332 }
2333
2334 // However, if the mask contains undef's, we have to enumerate possible tuples
2335 // and pick one. There are bounds on replication factor: [1, mask size]
2336 // (where RF=1 is an identity shuffle, RF=mask size is a broadcast shuffle)
2337 // Additionally, mask size is a replication factor multiplied by vector size,
2338 // which further significantly reduces the search space.
2339
2340 // Before doing that, let's perform basic correctness checking first.
2341 int Largest = -1;
2342 for (int MaskElt : Mask) {
2343 if (MaskElt == PoisonMaskElem)
2344 continue;
2345 // Elements must be in non-decreasing order.
2346 if (MaskElt < Largest)
2347 return false;
2348 Largest = std::max(a: Largest, b: MaskElt);
2349 }
2350
2351 // Prefer larger replication factor if all else equal.
2352 for (int PossibleReplicationFactor :
2353 reverse(C: seq_inclusive<unsigned>(Begin: 1, End: Mask.size()))) {
2354 if (Mask.size() % PossibleReplicationFactor != 0)
2355 continue;
2356 int PossibleVF = Mask.size() / PossibleReplicationFactor;
2357 if (!isReplicationMaskWithParams(Mask, ReplicationFactor: PossibleReplicationFactor,
2358 VF: PossibleVF))
2359 continue;
2360 ReplicationFactor = PossibleReplicationFactor;
2361 VF = PossibleVF;
2362 return true;
2363 }
2364
2365 return false;
2366}
2367
2368bool ShuffleVectorInst::isReplicationMask(int &ReplicationFactor,
2369 int &VF) const {
2370 // Not possible to express a shuffle mask for a scalable vector for this
2371 // case.
2372 if (isa<ScalableVectorType>(Val: getType()))
2373 return false;
2374
2375 VF = cast<FixedVectorType>(Val: Op<0>()->getType())->getNumElements();
2376 if (ShuffleMask.size() % VF != 0)
2377 return false;
2378 ReplicationFactor = ShuffleMask.size() / VF;
2379
2380 return isReplicationMaskWithParams(Mask: ShuffleMask, ReplicationFactor, VF);
2381}
2382
2383bool ShuffleVectorInst::isOneUseSingleSourceMask(ArrayRef<int> Mask, int VF) {
2384 if (VF <= 0 || Mask.size() < static_cast<unsigned>(VF) ||
2385 Mask.size() % VF != 0)
2386 return false;
2387 for (unsigned K = 0, Sz = Mask.size(); K < Sz; K += VF) {
2388 ArrayRef<int> SubMask = Mask.slice(N: K, M: VF);
2389 if (all_of(Range&: SubMask, P: equal_to(Arg: PoisonMaskElem)))
2390 continue;
2391 SmallBitVector Used(VF, false);
2392 for (int Idx : SubMask) {
2393 if (Idx != PoisonMaskElem && Idx < VF)
2394 Used.set(Idx);
2395 }
2396 if (!Used.all())
2397 return false;
2398 }
2399 return true;
2400}
2401
2402/// Return true if this shuffle mask is a replication mask.
2403bool ShuffleVectorInst::isOneUseSingleSourceMask(int VF) const {
2404 // Not possible to express a shuffle mask for a scalable vector for this
2405 // case.
2406 if (isa<ScalableVectorType>(Val: getType()))
2407 return false;
2408 if (!isSingleSourceMask(Mask: ShuffleMask, NumSrcElts: VF))
2409 return false;
2410
2411 return isOneUseSingleSourceMask(Mask: ShuffleMask, VF);
2412}
2413
2414bool ShuffleVectorInst::isInterleave(unsigned Factor) {
2415 FixedVectorType *OpTy = dyn_cast<FixedVectorType>(Val: getOperand(i_nocapture: 0)->getType());
2416 // shuffle_vector can only interleave fixed length vectors - for scalable
2417 // vectors, see the @llvm.vector.interleave2 intrinsic
2418 if (!OpTy)
2419 return false;
2420 unsigned OpNumElts = OpTy->getNumElements();
2421
2422 return isInterleaveMask(Mask: ShuffleMask, Factor, NumInputElts: OpNumElts * 2);
2423}
2424
2425bool ShuffleVectorInst::isInterleaveMask(
2426 ArrayRef<int> Mask, unsigned Factor, unsigned NumInputElts,
2427 SmallVectorImpl<unsigned> &StartIndexes) {
2428 unsigned NumElts = Mask.size();
2429 if (NumElts % Factor)
2430 return false;
2431
2432 unsigned LaneLen = NumElts / Factor;
2433 if (!isPowerOf2_32(Value: LaneLen))
2434 return false;
2435
2436 StartIndexes.resize(N: Factor);
2437
2438 // Check whether each element matches the general interleaved rule.
2439 // Ignore undef elements, as long as the defined elements match the rule.
2440 // Outer loop processes all factors (x, y, z in the above example)
2441 unsigned I = 0, J;
2442 for (; I < Factor; I++) {
2443 unsigned SavedLaneValue;
2444 unsigned SavedNoUndefs = 0;
2445
2446 // Inner loop processes consecutive accesses (x, x+1... in the example)
2447 for (J = 0; J < LaneLen - 1; J++) {
2448 // Lane computes x's position in the Mask
2449 unsigned Lane = J * Factor + I;
2450 unsigned NextLane = Lane + Factor;
2451 int LaneValue = Mask[Lane];
2452 int NextLaneValue = Mask[NextLane];
2453
2454 // If both are defined, values must be sequential
2455 if (LaneValue >= 0 && NextLaneValue >= 0 &&
2456 LaneValue + 1 != NextLaneValue)
2457 break;
2458
2459 // If the next value is undef, save the current one as reference
2460 if (LaneValue >= 0 && NextLaneValue < 0) {
2461 SavedLaneValue = LaneValue;
2462 SavedNoUndefs = 1;
2463 }
2464
2465 // Undefs are allowed, but defined elements must still be consecutive:
2466 // i.e.: x,..., undef,..., x + 2,..., undef,..., undef,..., x + 5, ....
2467 // Verify this by storing the last non-undef followed by an undef
2468 // Check that following non-undef masks are incremented with the
2469 // corresponding distance.
2470 if (SavedNoUndefs > 0 && LaneValue < 0) {
2471 SavedNoUndefs++;
2472 if (NextLaneValue >= 0 &&
2473 SavedLaneValue + SavedNoUndefs != (unsigned)NextLaneValue)
2474 break;
2475 }
2476 }
2477
2478 if (J < LaneLen - 1)
2479 return false;
2480
2481 int StartMask = 0;
2482 if (Mask[I] >= 0) {
2483 // Check that the start of the I range (J=0) is greater than 0
2484 StartMask = Mask[I];
2485 } else if (Mask[(LaneLen - 1) * Factor + I] >= 0) {
2486 // StartMask defined by the last value in lane
2487 StartMask = Mask[(LaneLen - 1) * Factor + I] - J;
2488 } else if (SavedNoUndefs > 0) {
2489 // StartMask defined by some non-zero value in the j loop
2490 StartMask = SavedLaneValue - (LaneLen - 1 - SavedNoUndefs);
2491 }
2492 // else StartMask remains set to 0, i.e. all elements are undefs
2493
2494 if (StartMask < 0)
2495 return false;
2496 // We must stay within the vectors; This case can happen with undefs.
2497 if (StartMask + LaneLen > NumInputElts)
2498 return false;
2499
2500 StartIndexes[I] = StartMask;
2501 }
2502
2503 return true;
2504}
2505
2506/// Check if the mask is a DE-interleave mask of the given factor
2507/// \p Factor like:
2508/// <Index, Index+Factor, ..., Index+(NumElts-1)*Factor>
2509bool ShuffleVectorInst::isDeInterleaveMaskOfFactor(ArrayRef<int> Mask,
2510 unsigned Factor,
2511 unsigned &Index) {
2512 // Check all potential start indices from 0 to (Factor - 1).
2513 for (unsigned Idx = 0; Idx < Factor; Idx++) {
2514 unsigned I = 0;
2515
2516 // Check that elements are in ascending order by Factor. Ignore undef
2517 // elements.
2518 for (; I < Mask.size(); I++)
2519 if (Mask[I] >= 0 && static_cast<unsigned>(Mask[I]) != Idx + I * Factor)
2520 break;
2521
2522 if (I == Mask.size()) {
2523 Index = Idx;
2524 return true;
2525 }
2526 }
2527
2528 return false;
2529}
2530
2531/// Try to lower a vector shuffle as a bit rotation.
2532///
2533/// Look for a repeated rotation pattern in each sub group.
2534/// Returns an element-wise left bit rotation amount or -1 if failed.
2535static int matchShuffleAsBitRotate(ArrayRef<int> Mask, int NumSubElts) {
2536 int NumElts = Mask.size();
2537 assert((NumElts % NumSubElts) == 0 && "Illegal shuffle mask");
2538
2539 int RotateAmt = -1;
2540 for (int i = 0; i != NumElts; i += NumSubElts) {
2541 for (int j = 0; j != NumSubElts; ++j) {
2542 int M = Mask[i + j];
2543 if (M < 0)
2544 continue;
2545 if (M < i || M >= i + NumSubElts)
2546 return -1;
2547 int Offset = (NumSubElts - (M - (i + j))) % NumSubElts;
2548 if (0 <= RotateAmt && Offset != RotateAmt)
2549 return -1;
2550 RotateAmt = Offset;
2551 }
2552 }
2553 return RotateAmt;
2554}
2555
2556bool ShuffleVectorInst::isBitRotateMask(
2557 ArrayRef<int> Mask, unsigned EltSizeInBits, unsigned MinSubElts,
2558 unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt) {
2559 for (NumSubElts = MinSubElts; NumSubElts <= MaxSubElts; NumSubElts *= 2) {
2560 int EltRotateAmt = matchShuffleAsBitRotate(Mask, NumSubElts);
2561 if (EltRotateAmt < 0)
2562 continue;
2563 RotateAmt = EltRotateAmt * EltSizeInBits;
2564 return true;
2565 }
2566
2567 return false;
2568}
2569
2570//===----------------------------------------------------------------------===//
2571// InsertValueInst Class
2572//===----------------------------------------------------------------------===//
2573
2574void InsertValueInst::init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2575 const Twine &Name) {
2576 assert(getNumOperands() == 2 && "NumOperands not initialized?");
2577
2578 // There's no fundamental reason why we require at least one index
2579 // (other than weirdness with &*IdxBegin being invalid; see
2580 // getelementptr's init routine for example). But there's no
2581 // present need to support it.
2582 assert(!Idxs.empty() && "InsertValueInst must have at least one index");
2583
2584 assert(ExtractValueInst::getIndexedType(Agg->getType(), Idxs) ==
2585 Val->getType() && "Inserted value must match indexed type!");
2586 Op<0>() = Agg;
2587 Op<1>() = Val;
2588
2589 Indices.append(in_start: Idxs.begin(), in_end: Idxs.end());
2590 setName(Name);
2591}
2592
2593InsertValueInst::InsertValueInst(const InsertValueInst &IVI)
2594 : Instruction(IVI.getType(), InsertValue, AllocMarker),
2595 Indices(IVI.Indices) {
2596 Op<0>() = IVI.getOperand(i_nocapture: 0);
2597 Op<1>() = IVI.getOperand(i_nocapture: 1);
2598 SubclassOptionalData = IVI.SubclassOptionalData;
2599}
2600
2601//===----------------------------------------------------------------------===//
2602// ExtractValueInst Class
2603//===----------------------------------------------------------------------===//
2604
2605void ExtractValueInst::init(ArrayRef<unsigned> Idxs, const Twine &Name) {
2606 assert(getNumOperands() == 1 && "NumOperands not initialized?");
2607
2608 // There's no fundamental reason why we require at least one index.
2609 // But there's no present need to support it.
2610 assert(!Idxs.empty() && "ExtractValueInst must have at least one index");
2611
2612 Indices.append(in_start: Idxs.begin(), in_end: Idxs.end());
2613 setName(Name);
2614}
2615
2616ExtractValueInst::ExtractValueInst(const ExtractValueInst &EVI)
2617 : UnaryInstruction(EVI.getType(), ExtractValue, EVI.getOperand(i_nocapture: 0),
2618 (BasicBlock *)nullptr),
2619 Indices(EVI.Indices) {
2620 SubclassOptionalData = EVI.SubclassOptionalData;
2621}
2622
2623// getIndexedType - Returns the type of the element that would be extracted
2624// with an extractvalue instruction with the specified parameters.
2625//
2626// A null type is returned if the indices are invalid for the specified
2627// pointer type.
2628//
2629Type *ExtractValueInst::getIndexedType(Type *Agg,
2630 ArrayRef<unsigned> Idxs) {
2631 for (unsigned Index : Idxs) {
2632 // We can't use CompositeType::indexValid(Index) here.
2633 // indexValid() always returns true for arrays because getelementptr allows
2634 // out-of-bounds indices. Since we don't allow those for extractvalue and
2635 // insertvalue we need to check array indexing manually.
2636 // Since the only other types we can index into are struct types it's just
2637 // as easy to check those manually as well.
2638 if (ArrayType *AT = dyn_cast<ArrayType>(Val: Agg)) {
2639 if (Index >= AT->getNumElements())
2640 return nullptr;
2641 Agg = AT->getElementType();
2642 } else if (StructType *ST = dyn_cast<StructType>(Val: Agg)) {
2643 if (Index >= ST->getNumElements())
2644 return nullptr;
2645 Agg = ST->getElementType(N: Index);
2646 } else {
2647 // Not a valid type to index into.
2648 return nullptr;
2649 }
2650 }
2651 return Agg;
2652}
2653
2654//===----------------------------------------------------------------------===//
2655// BitInsert Class
2656//===----------------------------------------------------------------------===//
2657BitInsertInst::BitInsertInst(Value *Base, Value *Val, Value *Offset,
2658 const Twine &Name, InsertPosition InsertBef)
2659 : Instruction(Base->getType(), BitInsert, AllocMarker, InsertBef) {
2660 assert(!areInvalidOperands(Base, Val, Offset) &&
2661 "Invalid bitinsert instruction operands!");
2662 Op<0>() = Base;
2663 Op<1>() = Val;
2664 Op<2>() = Offset;
2665 setName(Name);
2666}
2667
2668const char *BitInsertInst::areInvalidOperands(Value *Base, Value *Val,
2669 Value *Offset) {
2670 Type *BaseTy = Base->getType();
2671 if (!BaseTy->isByteOrByteVectorTy())
2672 return "bitinsert base must be a byte type or a vector of byte type";
2673 Type *ValTy = Val->getType()->getScalarType();
2674 if (!(ValTy->isFloatingPointTy() || ValTy->isIntegerTy() ||
2675 ValTy->isPointerTy() || ValTy->isByteTy()))
2676 return "bitinsert value must be an integer, floating-point, pointer, or "
2677 "byte type, or a vector of those types";
2678 if (!Offset->getType()->isIntOrIntVectorTy(BitWidth: 32))
2679 return "bitinsert offset must be i32 or a vector of i32";
2680 if (BaseTy->getWithNewType(EltTy: ValTy) != Val->getType() ||
2681 BaseTy->getWithNewType(EltTy: Offset->getType()->getScalarType()) !=
2682 Offset->getType())
2683 return "bitinsert operands must all be scalars or all be vectors with the "
2684 "same number of elements";
2685 return nullptr;
2686}
2687
2688//===----------------------------------------------------------------------===//
2689// BitExtract Class
2690//===----------------------------------------------------------------------===//
2691BitExtractInst::BitExtractInst(Type *Ty, Value *Src, Value *Offset,
2692 const Twine &Name, InsertPosition InsertBef)
2693 : Instruction(Ty, BitExtract, AllocMarker, InsertBef) {
2694 assert(!areInvalidOperands(Ty, Src, Offset) &&
2695 "Invalid bitextract instruction operands!");
2696 Op<0>() = Src;
2697 Op<1>() = Offset;
2698 setName(Name);
2699}
2700
2701const char *BitExtractInst::areInvalidOperands(const Type *Ty, Value *Src,
2702 Value *Offset) {
2703 const Type *EltTy = Ty->getScalarType();
2704 if (!(EltTy->isFloatingPointTy() || EltTy->isIntegerTy() ||
2705 EltTy->isPointerTy() || EltTy->isByteTy()))
2706 return "bitextract result must be an integer, floating-point, pointer, or "
2707 "byte type, or a vector of those types";
2708 if (!Src->getType()->isByteOrByteVectorTy())
2709 return "bitextract source must be a byte type or a vector of byte type";
2710 if (!Offset->getType()->isIntOrIntVectorTy(BitWidth: 32))
2711 return "bitextract offset must be i32 or a vector of i32";
2712 if (Ty->getWithNewType(EltTy: Src->getType()->getScalarType()) != Src->getType() ||
2713 Ty->getWithNewType(EltTy: Offset->getType()->getScalarType()) !=
2714 Offset->getType())
2715 return "bitextract operands must all be scalars or all be vectors with the "
2716 "same number of elements";
2717 return nullptr;
2718}
2719
2720//===----------------------------------------------------------------------===//
2721// UnaryOperator Class
2722//===----------------------------------------------------------------------===//
2723
2724UnaryOperator::UnaryOperator(UnaryOps iType, Value *S, Type *Ty,
2725 const Twine &Name, InsertPosition InsertBefore)
2726 : UnaryInstruction(Ty, iType, S, InsertBefore) {
2727 Op<0>() = S;
2728 setName(Name);
2729 AssertOK();
2730}
2731
2732UnaryOperator *UnaryOperator::Create(UnaryOps Op, Value *S, const Twine &Name,
2733 InsertPosition InsertBefore) {
2734 switch (Op) {
2735 case UnaryOps::FNeg:
2736 return new FPUnaryOperator(Op, S, S->getType(), Name, InsertBefore);
2737 default:
2738 return new UnaryOperator(Op, S, S->getType(), Name, InsertBefore);
2739 }
2740}
2741
2742void UnaryOperator::AssertOK() {
2743 Value *LHS = getOperand(i_nocapture: 0);
2744 (void)LHS; // Silence warnings.
2745#ifndef NDEBUG
2746 switch (getOpcode()) {
2747 case FNeg:
2748 assert(getType() == LHS->getType() &&
2749 "Unary operation should return same type as operand!");
2750 assert(getType()->isFPOrFPVectorTy() &&
2751 "Tried to create a floating-point operation on a "
2752 "non-floating-point type!");
2753 break;
2754 default: llvm_unreachable("Invalid opcode provided");
2755 }
2756#endif
2757}
2758
2759//===----------------------------------------------------------------------===//
2760// BinaryOperator Class
2761//===----------------------------------------------------------------------===//
2762
2763BinaryOperator::BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
2764 const Twine &Name, InsertPosition InsertBefore)
2765 : Instruction(Ty, iType, AllocMarker, InsertBefore) {
2766 Op<0>() = S1;
2767 Op<1>() = S2;
2768 setName(Name);
2769 AssertOK();
2770}
2771
2772void BinaryOperator::AssertOK() {
2773 Value *LHS = getOperand(i_nocapture: 0), *RHS = getOperand(i_nocapture: 1);
2774 (void)LHS; (void)RHS; // Silence warnings.
2775 assert(LHS->getType() == RHS->getType() &&
2776 "Binary operator operand types must match!");
2777#ifndef NDEBUG
2778 switch (getOpcode()) {
2779 case Add: case Sub:
2780 case Mul:
2781 assert(getType() == LHS->getType() &&
2782 "Arithmetic operation should return same type as operands!");
2783 assert(getType()->isIntOrIntVectorTy() &&
2784 "Tried to create an integer operation on a non-integer type!");
2785 break;
2786 case FAdd: case FSub:
2787 case FMul:
2788 assert(getType() == LHS->getType() &&
2789 "Arithmetic operation should return same type as operands!");
2790 assert(getType()->isFPOrFPVectorTy() &&
2791 "Tried to create a floating-point operation on a "
2792 "non-floating-point type!");
2793 break;
2794 case UDiv:
2795 case SDiv:
2796 assert(getType() == LHS->getType() &&
2797 "Arithmetic operation should return same type as operands!");
2798 assert(getType()->isIntOrIntVectorTy() &&
2799 "Incorrect operand type (not integer) for S/UDIV");
2800 break;
2801 case FDiv:
2802 assert(getType() == LHS->getType() &&
2803 "Arithmetic operation should return same type as operands!");
2804 assert(getType()->isFPOrFPVectorTy() &&
2805 "Incorrect operand type (not floating point) for FDIV");
2806 break;
2807 case URem:
2808 case SRem:
2809 assert(getType() == LHS->getType() &&
2810 "Arithmetic operation should return same type as operands!");
2811 assert(getType()->isIntOrIntVectorTy() &&
2812 "Incorrect operand type (not integer) for S/UREM");
2813 break;
2814 case FRem:
2815 assert(getType() == LHS->getType() &&
2816 "Arithmetic operation should return same type as operands!");
2817 assert(getType()->isFPOrFPVectorTy() &&
2818 "Incorrect operand type (not floating point) for FREM");
2819 break;
2820 case Shl:
2821 case LShr:
2822 case AShr:
2823 assert(getType() == LHS->getType() &&
2824 "Shift operation should return same type as operands!");
2825 assert(getType()->isIntOrIntVectorTy() &&
2826 "Tried to create a shift operation on a non-integral type!");
2827 break;
2828 case And: case Or:
2829 case Xor:
2830 assert(getType() == LHS->getType() &&
2831 "Logical operation should return same type as operands!");
2832 assert(getType()->isIntOrIntVectorTy() &&
2833 "Tried to create a logical operation on a non-integral type!");
2834 break;
2835 default: llvm_unreachable("Invalid opcode provided");
2836 }
2837#endif
2838}
2839
2840BinaryOperator *BinaryOperator::Create(BinaryOps Op, Value *S1, Value *S2,
2841 const Twine &Name,
2842 InsertPosition InsertBefore) {
2843 assert(S1->getType() == S2->getType() &&
2844 "Cannot create binary operator with two operands of differing type!");
2845 switch (Op) {
2846 case BinaryOps::FAdd:
2847 case BinaryOps::FSub:
2848 case BinaryOps::FMul:
2849 case BinaryOps::FDiv:
2850 case BinaryOps::FRem:
2851 return new FPBinaryOperator(Op, S1, S2, S1->getType(), Name, InsertBefore);
2852 default:
2853 return new BinaryOperator(Op, S1, S2, S1->getType(), Name, InsertBefore);
2854 }
2855}
2856
2857BinaryOperator *BinaryOperator::CreateNeg(Value *Op, const Twine &Name,
2858 InsertPosition InsertBefore) {
2859 Value *Zero = ConstantInt::get(Ty: Op->getType(), V: 0);
2860 return new BinaryOperator(Instruction::Sub, Zero, Op, Op->getType(), Name,
2861 InsertBefore);
2862}
2863
2864BinaryOperator *BinaryOperator::CreateNSWNeg(Value *Op, const Twine &Name,
2865 InsertPosition InsertBefore) {
2866 Value *Zero = ConstantInt::get(Ty: Op->getType(), V: 0);
2867 return BinaryOperator::CreateNSWSub(V1: Zero, V2: Op, Name, InsertBefore);
2868}
2869
2870BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name,
2871 InsertPosition InsertBefore) {
2872 Constant *C = Constant::getAllOnesValue(Ty: Op->getType());
2873 return new BinaryOperator(Instruction::Xor, Op, C,
2874 Op->getType(), Name, InsertBefore);
2875}
2876
2877// Exchange the two operands to this instruction. This instruction is safe to
2878// use on any binary instruction and does not modify the semantics of the
2879// instruction.
2880bool BinaryOperator::swapOperands() {
2881 if (!isCommutative())
2882 return true; // Can't commute operands
2883 Op<0>().swap(RHS&: Op<1>());
2884 return false;
2885}
2886
2887//===----------------------------------------------------------------------===//
2888// FPMathOperator Class
2889//===----------------------------------------------------------------------===//
2890
2891float FPMathOperator::getFPAccuracy() const {
2892 const MDNode *MD =
2893 cast<Instruction>(Val: this)->getMetadata(KindID: LLVMContext::MD_fpmath);
2894 if (!MD)
2895 return 0.0;
2896 ConstantFP *Accuracy = mdconst::extract<ConstantFP>(MD: MD->getOperand(I: 0));
2897 return Accuracy->getValueAPF().convertToFloat();
2898}
2899
2900//===----------------------------------------------------------------------===//
2901// CastInst Class
2902//===----------------------------------------------------------------------===//
2903
2904// Just determine if this cast only deals with integral->integral conversion.
2905bool CastInst::isIntegerCast() const {
2906 switch (getOpcode()) {
2907 default: return false;
2908 case Instruction::ZExt:
2909 case Instruction::SExt:
2910 case Instruction::Trunc:
2911 return true;
2912 case Instruction::BitCast:
2913 return getOperand(i_nocapture: 0)->getType()->isIntegerTy() &&
2914 getType()->isIntegerTy();
2915 }
2916}
2917
2918/// This function determines if the CastInst does not require any bits to be
2919/// changed in order to effect the cast. Essentially, it identifies cases where
2920/// no code gen is necessary for the cast, hence the name no-op cast. For
2921/// example, the following are all no-op casts:
2922/// # bitcast i32* %x to i8*
2923/// # bitcast <2 x i32> %x to <4 x i16>
2924/// # ptrtoint i32* %x to i32 ; on 32-bit plaforms only
2925/// Determine if the described cast is a no-op.
2926bool CastInst::isNoopCast(Instruction::CastOps Opcode,
2927 Type *SrcTy,
2928 Type *DestTy,
2929 const DataLayout &DL) {
2930 assert(castIsValid(Opcode, SrcTy, DestTy) && "method precondition");
2931 switch (Opcode) {
2932 default: llvm_unreachable("Invalid CastOp");
2933 case Instruction::Trunc:
2934 case Instruction::ZExt:
2935 case Instruction::SExt:
2936 case Instruction::FPTrunc:
2937 case Instruction::FPExt:
2938 case Instruction::UIToFP:
2939 case Instruction::SIToFP:
2940 case Instruction::FPToUI:
2941 case Instruction::FPToSI:
2942 case Instruction::AddrSpaceCast:
2943 // TODO: Target informations may give a more accurate answer here.
2944 return false;
2945 case Instruction::BitCast:
2946 return true; // BitCast never modifies bits.
2947 case Instruction::PtrToAddr:
2948 case Instruction::PtrToInt:
2949 return DL.getIntPtrType(SrcTy)->getScalarSizeInBits() ==
2950 DestTy->getScalarSizeInBits();
2951 case Instruction::IntToPtr:
2952 return DL.getIntPtrType(DestTy)->getScalarSizeInBits() ==
2953 SrcTy->getScalarSizeInBits();
2954 }
2955}
2956
2957bool CastInst::isNoopCast(const DataLayout &DL) const {
2958 return isNoopCast(Opcode: getOpcode(), SrcTy: getOperand(i_nocapture: 0)->getType(), DestTy: getType(), DL);
2959}
2960
2961/// This function determines if a pair of casts can be eliminated and what
2962/// opcode should be used in the elimination. This assumes that there are two
2963/// instructions like this:
2964/// * %F = firstOpcode SrcTy %x to MidTy
2965/// * %S = secondOpcode MidTy %F to DstTy
2966/// The function returns a resultOpcode so these two casts can be replaced with:
2967/// * %Replacement = resultOpcode %SrcTy %x to DstTy
2968/// If no such cast is permitted, the function returns 0.
2969unsigned CastInst::isEliminableCastPair(Instruction::CastOps firstOp,
2970 Instruction::CastOps secondOp,
2971 Type *SrcTy, Type *MidTy, Type *DstTy,
2972 const DataLayout *DL) {
2973 // Define the 144 possibilities for these two cast instructions. The values
2974 // in this matrix determine what to do in a given situation and select the
2975 // case in the switch below. The rows correspond to firstOp, the columns
2976 // correspond to secondOp. In looking at the table below, keep in mind
2977 // the following cast properties:
2978 //
2979 // Size Compare Source Destination
2980 // Operator Src ? Size Type Sign Type Sign
2981 // -------- ------------ ------------------- ---------------------
2982 // TRUNC > Integer Any Integral Any
2983 // ZEXT < Integral Unsigned Integer Any
2984 // SEXT < Integral Signed Integer Any
2985 // FPTOUI n/a FloatPt n/a Integral Unsigned
2986 // FPTOSI n/a FloatPt n/a Integral Signed
2987 // UITOFP n/a Integral Unsigned FloatPt n/a
2988 // SITOFP n/a Integral Signed FloatPt n/a
2989 // FPTRUNC > FloatPt n/a FloatPt n/a
2990 // FPEXT < FloatPt n/a FloatPt n/a
2991 // PTRTOINT n/a Pointer n/a Integral Unsigned
2992 // PTRTOADDR n/a Pointer n/a Integral Unsigned
2993 // INTTOPTR n/a Integral Unsigned Pointer n/a
2994 // BITCAST = FirstClass n/a FirstClass n/a
2995 // ADDRSPCST n/a Pointer n/a Pointer n/a
2996 //
2997 // NOTE: some transforms are safe, but we consider them to be non-profitable.
2998 // For example, we could merge "fptoui double to i32" + "zext i32 to i64",
2999 // into "fptoui double to i64", but this loses information about the range
3000 // of the produced value (we no longer know the top-part is all zeros).
3001 // Further this conversion is often much more expensive for typical hardware,
3002 // and causes issues when building libgcc. We disallow fptosi+sext for the
3003 // same reason.
3004 const unsigned numCastOps =
3005 Instruction::CastOpsEnd - Instruction::CastOpsBegin;
3006 // clang-format off
3007 static const uint8_t CastResults[numCastOps][numCastOps] = {
3008 // T F F U S F F P P I B A -+
3009 // R Z S P P I I T P 2 2 N T S |
3010 // U E E 2 2 2 2 R E I A T C C +- secondOp
3011 // N X X U S F F N X N D 2 V V |
3012 // C T T I I P P C T T R P T T -+
3013 { 1, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // Trunc -+
3014 { 8, 1, 9,99,99, 2,17,99,99,99,99, 2, 3, 0}, // ZExt |
3015 { 8, 0, 1,99,99, 0, 2,99,99,99,99, 0, 3, 0}, // SExt |
3016 { 0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // FPToUI |
3017 { 0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // FPToSI |
3018 { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // UIToFP +- firstOp
3019 { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // SIToFP |
3020 { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // FPTrunc |
3021 { 99,99,99, 2, 2,99,99, 8, 2,99,99,99, 4, 0}, // FPExt |
3022 { 1, 0, 0,99,99, 0, 0,99,99,99,99, 7, 3, 0}, // PtrToInt |
3023 { 0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // PtrToAddr |
3024 { 99,99,99,99,99,99,99,99,99,11,11,99,15, 0}, // IntToPtr |
3025 { 5, 5, 5, 0, 0, 5, 5, 0, 0,16,16, 5,18,14}, // BitCast |
3026 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,13,12}, // AddrSpaceCast -+
3027 };
3028 // clang-format on
3029
3030 // TODO: This logic could be encoded into the table above and handled in the
3031 // switch below.
3032 // If either of the casts are a bitcast from scalar to vector, disallow the
3033 // merging. However, any pair of bitcasts are allowed.
3034 bool IsFirstBitcast = (firstOp == Instruction::BitCast);
3035 bool IsSecondBitcast = (secondOp == Instruction::BitCast);
3036 bool AreBothBitcasts = IsFirstBitcast && IsSecondBitcast;
3037
3038 // Check if any of the casts convert scalars <-> vectors.
3039 if ((IsFirstBitcast && isa<VectorType>(Val: SrcTy) != isa<VectorType>(Val: MidTy)) ||
3040 (IsSecondBitcast && isa<VectorType>(Val: MidTy) != isa<VectorType>(Val: DstTy)))
3041 if (!AreBothBitcasts)
3042 return 0;
3043
3044 int ElimCase = CastResults[firstOp-Instruction::CastOpsBegin]
3045 [secondOp-Instruction::CastOpsBegin];
3046 switch (ElimCase) {
3047 case 0:
3048 // Categorically disallowed.
3049 return 0;
3050 case 1:
3051 // Allowed, use first cast's opcode.
3052 return firstOp;
3053 case 2:
3054 // Allowed, use second cast's opcode.
3055 return secondOp;
3056 case 3:
3057 // No-op cast in second op implies firstOp as long as the DestTy
3058 // is integer and we are not converting between a vector and a
3059 // non-vector type.
3060 if (!SrcTy->isVectorTy() && DstTy->isIntegerTy())
3061 return firstOp;
3062 return 0;
3063 case 4:
3064 // No-op cast in second op implies firstOp as long as the DestTy
3065 // matches MidTy.
3066 if (DstTy == MidTy)
3067 return firstOp;
3068 return 0;
3069 case 5:
3070 // No-op cast in first op implies secondOp as long as the SrcTy
3071 // is an integer.
3072 if (SrcTy->isIntegerTy())
3073 return secondOp;
3074 return 0;
3075 case 7: {
3076 // Skip the inttoptr/ptrtoint optimization if disabled.
3077 if (!getIROptions(C: SrcTy->getContext()).i2p_p2i_opt)
3078 return 0;
3079
3080 // Cannot simplify if address spaces are different!
3081 if (SrcTy != DstTy)
3082 return 0;
3083
3084 // Cannot simplify if the intermediate integer size is smaller than the
3085 // pointer size.
3086 unsigned MidSize = MidTy->getScalarSizeInBits();
3087 if (!DL || MidSize < DL->getPointerTypeSizeInBits(SrcTy))
3088 return 0;
3089
3090 return Instruction::BitCast;
3091 }
3092 case 8: {
3093 // ext, trunc -> bitcast, if the SrcTy and DstTy are the same
3094 // ext, trunc -> ext, if sizeof(SrcTy) < sizeof(DstTy)
3095 // ext, trunc -> trunc, if sizeof(SrcTy) > sizeof(DstTy)
3096 unsigned SrcSize = SrcTy->getScalarSizeInBits();
3097 unsigned DstSize = DstTy->getScalarSizeInBits();
3098 if (SrcTy == DstTy)
3099 return Instruction::BitCast;
3100 if (SrcSize < DstSize)
3101 return firstOp;
3102 if (SrcSize > DstSize)
3103 return secondOp;
3104 return 0;
3105 }
3106 case 9:
3107 // zext, sext -> zext, because sext can't sign extend after zext
3108 return Instruction::ZExt;
3109 case 11: {
3110 // inttoptr, ptrtoint/ptrtoaddr -> integer cast
3111 if (!DL)
3112 return 0;
3113 unsigned MidSize = secondOp == Instruction::PtrToAddr
3114 ? DL->getAddressSizeInBits(Ty: MidTy)
3115 : DL->getPointerTypeSizeInBits(MidTy);
3116 unsigned SrcSize = SrcTy->getScalarSizeInBits();
3117 unsigned DstSize = DstTy->getScalarSizeInBits();
3118 // If the middle size is smaller than both source and destination,
3119 // an additional masking operation would be required.
3120 if (MidSize < SrcSize && MidSize < DstSize)
3121 return 0;
3122 if (DstSize < SrcSize)
3123 return Instruction::Trunc;
3124 if (DstSize > SrcSize)
3125 return Instruction::ZExt;
3126 return Instruction::BitCast;
3127 }
3128 case 12:
3129 // addrspacecast, addrspacecast -> bitcast, if SrcAS == DstAS
3130 // addrspacecast, addrspacecast -> addrspacecast, if SrcAS != DstAS
3131 if (SrcTy->getPointerAddressSpace() != DstTy->getPointerAddressSpace())
3132 return Instruction::AddrSpaceCast;
3133 return Instruction::BitCast;
3134 case 13:
3135 // FIXME: this state can be merged with (1), but the following assert
3136 // is useful to check the correcteness of the sequence due to semantic
3137 // change of bitcast.
3138 // addrspacecast can only fold through a bitcast if the result remains a
3139 // pointer. A pointer-to-byte bitcast must stay as a separate bitcast.
3140 if (!DstTy->isPtrOrPtrVectorTy())
3141 return 0;
3142 assert(
3143 SrcTy->isPtrOrPtrVectorTy() &&
3144 MidTy->isPtrOrPtrVectorTy() &&
3145 DstTy->isPtrOrPtrVectorTy() &&
3146 SrcTy->getPointerAddressSpace() != MidTy->getPointerAddressSpace() &&
3147 MidTy->getPointerAddressSpace() == DstTy->getPointerAddressSpace() &&
3148 "Illegal addrspacecast, bitcast sequence!");
3149 // Allowed, use first cast's opcode
3150 return firstOp;
3151 case 14:
3152 // bitcast, addrspacecast -> addrspacecast
3153 // addrspacecast can only fold through a bitcast if the source was already
3154 // a pointer. A byte-to-pointer bitcast must stay as a separate bitcast.
3155 if (!SrcTy->isPtrOrPtrVectorTy())
3156 return 0;
3157 return Instruction::AddrSpaceCast;
3158 case 15:
3159 // FIXME: this state can be merged with (1), but the following assert
3160 // is useful to check the correcteness of the sequence due to semantic
3161 // change of bitcast.
3162 // inttoptr can only fold through a bitcast if the result remains a
3163 // pointer. A pointer-to-byte bitcast must stay as a separate bitcast.
3164 if (!DstTy->isPtrOrPtrVectorTy())
3165 return 0;
3166 assert(
3167 SrcTy->isIntOrIntVectorTy() &&
3168 MidTy->isPtrOrPtrVectorTy() &&
3169 DstTy->isPtrOrPtrVectorTy() &&
3170 MidTy->getPointerAddressSpace() == DstTy->getPointerAddressSpace() &&
3171 "Illegal inttoptr, bitcast sequence!");
3172 // Allowed, use first cast's opcode
3173 return firstOp;
3174 case 16:
3175 // FIXME: this state can be merged with (2), but the following assert
3176 // is useful to check the correcteness of the sequence due to semantic
3177 // change of bitcast.
3178 // bitcast, ptrtoaddr -> bitcast, if SrcTy is a byte and the address is
3179 // the whole pointer
3180 // ptrtoint can only fold through a bitcast if the source was already a
3181 // pointer, as ptrtoint exposes the provenance.
3182 if (!SrcTy->isPtrOrPtrVectorTy()) {
3183 // ptrtoaddr yields the address width, so the sizes only match if the
3184 // address is the whole pointer.
3185 if (secondOp == Instruction::PtrToAddr &&
3186 SrcTy->getPrimitiveSizeInBits() == DstTy->getPrimitiveSizeInBits())
3187 return Instruction::BitCast;
3188 return 0;
3189 }
3190 assert(
3191 SrcTy->isPtrOrPtrVectorTy() &&
3192 MidTy->isPtrOrPtrVectorTy() &&
3193 DstTy->isIntOrIntVectorTy() &&
3194 SrcTy->getPointerAddressSpace() == MidTy->getPointerAddressSpace() &&
3195 "Illegal bitcast, ptrtoint sequence!");
3196 // Allowed, use second cast's opcode
3197 return secondOp;
3198 case 17:
3199 // (sitofp (zext x)) -> (uitofp x)
3200 return Instruction::UIToFP;
3201 case 18:
3202 // bitcast, bitcast -> bitcast, if neither SrcTy nor DstTy is a pointer,
3203 // both are pointers in the same addrspace,
3204 // or one is a pointer and the other a byte
3205 // bitcast, bitcast -> ptrtoint, if a pointer goes through a byte to an
3206 // integer of the pointer width
3207 // bitcast, bitcast -> inttoptr, if an integer of the pointer width goes
3208 // through a byte to a pointer
3209 // A pair of bitcasts through a byte must stay separate otherwise.
3210 if (!SrcTy->isPtrOrPtrVectorTy() && !DstTy->isPtrOrPtrVectorTy())
3211 return Instruction::BitCast;
3212 if (SrcTy->isPtrOrPtrVectorTy() && DstTy->isPtrOrPtrVectorTy() &&
3213 SrcTy->getPointerAddressSpace() == DstTy->getPointerAddressSpace())
3214 return Instruction::BitCast;
3215 if (SrcTy->isByteOrByteVectorTy() || DstTy->isByteOrByteVectorTy())
3216 return Instruction::BitCast;
3217 if (SrcTy->isVectorTy() == DstTy->isVectorTy() && DL) {
3218 if (SrcTy->isPtrOrPtrVectorTy() && DstTy->isIntOrIntVectorTy() &&
3219 DL->getPointerTypeSizeInBits(SrcTy) == DstTy->getScalarSizeInBits())
3220 return Instruction::PtrToInt;
3221 if (SrcTy->isIntOrIntVectorTy() && DstTy->isPtrOrPtrVectorTy() &&
3222 SrcTy->getScalarSizeInBits() == DL->getPointerTypeSizeInBits(DstTy))
3223 return Instruction::IntToPtr;
3224 }
3225 return 0;
3226 case 99:
3227 // Cast combination can't happen (error in input). This is for all cases
3228 // where the MidTy is not the same for the two cast instructions.
3229 llvm_unreachable("Invalid Cast Combination");
3230 default:
3231 llvm_unreachable("Error in CastResults table!!!");
3232 }
3233}
3234
3235CastInst *CastInst::Create(Instruction::CastOps op, Value *S, Type *Ty,
3236 const Twine &Name, InsertPosition InsertBefore) {
3237 assert(castIsValid(op, S, Ty) && "Invalid cast!");
3238 // Construct and return the appropriate CastInst subclass
3239 switch (op) {
3240 case Trunc: return new TruncInst (S, Ty, Name, InsertBefore);
3241 case ZExt: return new ZExtInst (S, Ty, Name, InsertBefore);
3242 case SExt: return new SExtInst (S, Ty, Name, InsertBefore);
3243 case FPTrunc: return new FPTruncInst (S, Ty, Name, InsertBefore);
3244 case FPExt: return new FPExtInst (S, Ty, Name, InsertBefore);
3245 case UIToFP: return new UIToFPInst (S, Ty, Name, InsertBefore);
3246 case SIToFP: return new SIToFPInst (S, Ty, Name, InsertBefore);
3247 case FPToUI: return new FPToUIInst (S, Ty, Name, InsertBefore);
3248 case FPToSI: return new FPToSIInst (S, Ty, Name, InsertBefore);
3249 case PtrToAddr: return new PtrToAddrInst (S, Ty, Name, InsertBefore);
3250 case PtrToInt: return new PtrToIntInst (S, Ty, Name, InsertBefore);
3251 case IntToPtr: return new IntToPtrInst (S, Ty, Name, InsertBefore);
3252 case BitCast:
3253 return new BitCastInst(S, Ty, Name, InsertBefore);
3254 case AddrSpaceCast:
3255 return new AddrSpaceCastInst(S, Ty, Name, InsertBefore);
3256 default:
3257 llvm_unreachable("Invalid opcode provided");
3258 }
3259}
3260
3261CastInst *CastInst::CreateZExtOrBitCast(Value *S, Type *Ty, const Twine &Name,
3262 InsertPosition InsertBefore) {
3263 if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
3264 return Create(op: Instruction::BitCast, S, Ty, Name, InsertBefore);
3265 return Create(op: Instruction::ZExt, S, Ty, Name, InsertBefore);
3266}
3267
3268CastInst *CastInst::CreateSExtOrBitCast(Value *S, Type *Ty, const Twine &Name,
3269 InsertPosition InsertBefore) {
3270 if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
3271 return Create(op: Instruction::BitCast, S, Ty, Name, InsertBefore);
3272 return Create(op: Instruction::SExt, S, Ty, Name, InsertBefore);
3273}
3274
3275CastInst *CastInst::CreateTruncOrBitCast(Value *S, Type *Ty, const Twine &Name,
3276 InsertPosition InsertBefore) {
3277 if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())
3278 return Create(op: Instruction::BitCast, S, Ty, Name, InsertBefore);
3279 return Create(op: Instruction::Trunc, S, Ty, Name, InsertBefore);
3280}
3281
3282/// Create a BitCast or a PtrToInt cast instruction
3283CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty, const Twine &Name,
3284 InsertPosition InsertBefore) {
3285 assert(S->getType()->isPtrOrPtrVectorTy() && "Invalid cast");
3286 assert((Ty->isIntOrIntVectorTy() || Ty->isPtrOrPtrVectorTy()) &&
3287 "Invalid cast");
3288 assert(Ty->isVectorTy() == S->getType()->isVectorTy() && "Invalid cast");
3289 assert((!Ty->isVectorTy() ||
3290 cast<VectorType>(Ty)->getElementCount() ==
3291 cast<VectorType>(S->getType())->getElementCount()) &&
3292 "Invalid cast");
3293
3294 if (Ty->isIntOrIntVectorTy())
3295 return Create(op: Instruction::PtrToInt, S, Ty, Name, InsertBefore);
3296
3297 return CreatePointerBitCastOrAddrSpaceCast(S, Ty, Name, InsertBefore);
3298}
3299
3300CastInst *CastInst::CreatePointerBitCastOrAddrSpaceCast(
3301 Value *S, Type *Ty, const Twine &Name, InsertPosition InsertBefore) {
3302 assert(S->getType()->isPtrOrPtrVectorTy() && "Invalid cast");
3303 assert(Ty->isPtrOrPtrVectorTy() && "Invalid cast");
3304
3305 if (S->getType()->getPointerAddressSpace() != Ty->getPointerAddressSpace())
3306 return Create(op: Instruction::AddrSpaceCast, S, Ty, Name, InsertBefore);
3307
3308 return Create(op: Instruction::BitCast, S, Ty, Name, InsertBefore);
3309}
3310
3311CastInst *CastInst::CreateBitOrPointerCast(Value *S, Type *Ty,
3312 const Twine &Name,
3313 InsertPosition InsertBefore) {
3314 if (S->getType()->isPointerTy() && Ty->isIntegerTy())
3315 return Create(op: Instruction::PtrToInt, S, Ty, Name, InsertBefore);
3316 if (S->getType()->isIntegerTy() && Ty->isPointerTy())
3317 return Create(op: Instruction::IntToPtr, S, Ty, Name, InsertBefore);
3318
3319 return Create(op: Instruction::BitCast, S, Ty, Name, InsertBefore);
3320}
3321
3322CastInst *CastInst::CreateIntegerCast(Value *C, Type *Ty, bool isSigned,
3323 const Twine &Name,
3324 InsertPosition InsertBefore) {
3325 assert(C->getType()->isIntOrIntVectorTy() && Ty->isIntOrIntVectorTy() &&
3326 "Invalid integer cast");
3327 unsigned SrcBits = C->getType()->getScalarSizeInBits();
3328 unsigned DstBits = Ty->getScalarSizeInBits();
3329 Instruction::CastOps opcode =
3330 (SrcBits == DstBits ? Instruction::BitCast :
3331 (SrcBits > DstBits ? Instruction::Trunc :
3332 (isSigned ? Instruction::SExt : Instruction::ZExt)));
3333 return Create(op: opcode, S: C, Ty, Name, InsertBefore);
3334}
3335
3336CastInst *CastInst::CreateFPCast(Value *C, Type *Ty, const Twine &Name,
3337 InsertPosition InsertBefore) {
3338 assert(C->getType()->isFPOrFPVectorTy() && Ty->isFPOrFPVectorTy() &&
3339 "Invalid cast");
3340 unsigned SrcBits = C->getType()->getScalarSizeInBits();
3341 unsigned DstBits = Ty->getScalarSizeInBits();
3342 assert((C->getType() == Ty || SrcBits != DstBits) && "Invalid cast");
3343 Instruction::CastOps opcode =
3344 (SrcBits == DstBits ? Instruction::BitCast :
3345 (SrcBits > DstBits ? Instruction::FPTrunc : Instruction::FPExt));
3346 return Create(op: opcode, S: C, Ty, Name, InsertBefore);
3347}
3348
3349bool CastInst::isBitCastable(Type *SrcTy, Type *DestTy) {
3350 if (!SrcTy->isFirstClassType() || !DestTy->isFirstClassType())
3351 return false;
3352
3353 if (SrcTy == DestTy)
3354 return true;
3355
3356 if (VectorType *SrcVecTy = dyn_cast<VectorType>(Val: SrcTy)) {
3357 if (VectorType *DestVecTy = dyn_cast<VectorType>(Val: DestTy)) {
3358 if (SrcVecTy->getElementCount() == DestVecTy->getElementCount()) {
3359 // An element by element cast. Valid if casting the elements is valid.
3360 SrcTy = SrcVecTy->getElementType();
3361 DestTy = DestVecTy->getElementType();
3362 }
3363 }
3364 }
3365
3366 if (PointerType *DestPtrTy = dyn_cast<PointerType>(Val: DestTy)) {
3367 if (PointerType *SrcPtrTy = dyn_cast<PointerType>(Val: SrcTy)) {
3368 return SrcPtrTy->getAddressSpace() == DestPtrTy->getAddressSpace();
3369 }
3370 }
3371
3372 TypeSize SrcBits = SrcTy->getPrimitiveSizeInBits(); // 0 for ptr
3373 TypeSize DestBits = DestTy->getPrimitiveSizeInBits(); // 0 for ptr
3374
3375 // Could still have vectors of pointers if the number of elements doesn't
3376 // match
3377 if (SrcBits.getKnownMinValue() == 0 || DestBits.getKnownMinValue() == 0)
3378 return false;
3379
3380 if (SrcBits != DestBits)
3381 return false;
3382
3383 return true;
3384}
3385
3386bool CastInst::isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy,
3387 const DataLayout &DL) {
3388 // ptrtoint and inttoptr are not allowed on non-integral pointers
3389 if (auto *PtrTy = dyn_cast<PointerType>(Val: SrcTy))
3390 if (auto *IntTy = dyn_cast<IntegerType>(Val: DestTy))
3391 return (IntTy->getBitWidth() == DL.getPointerTypeSizeInBits(PtrTy) &&
3392 !DL.isNonIntegralPointerType(PT: PtrTy));
3393 if (auto *PtrTy = dyn_cast<PointerType>(Val: DestTy))
3394 if (auto *IntTy = dyn_cast<IntegerType>(Val: SrcTy))
3395 return (IntTy->getBitWidth() == DL.getPointerTypeSizeInBits(PtrTy) &&
3396 !DL.isNonIntegralPointerType(PT: PtrTy));
3397
3398 return isBitCastable(SrcTy, DestTy);
3399}
3400
3401// Provide a way to get a "cast" where the cast opcode is inferred from the
3402// types and size of the operand. This, basically, is a parallel of the
3403// logic in the castIsValid function below. This axiom should hold:
3404// castIsValid( getCastOpcode(Val, Ty), Val, Ty)
3405// should not assert in castIsValid. In other words, this produces a "correct"
3406// casting opcode for the arguments passed to it.
3407Instruction::CastOps
3408CastInst::getCastOpcode(
3409 const Value *Src, bool SrcIsSigned, Type *DestTy, bool DestIsSigned) {
3410 Type *SrcTy = Src->getType();
3411
3412 assert(SrcTy->isFirstClassType() && DestTy->isFirstClassType() &&
3413 "Only first class types are castable!");
3414
3415 if (SrcTy == DestTy)
3416 return BitCast;
3417
3418 // FIXME: Check address space sizes here
3419 if (VectorType *SrcVecTy = dyn_cast<VectorType>(Val: SrcTy))
3420 if (VectorType *DestVecTy = dyn_cast<VectorType>(Val: DestTy))
3421 if (SrcVecTy->getElementCount() == DestVecTy->getElementCount()) {
3422 // An element by element cast. Find the appropriate opcode based on the
3423 // element types.
3424 SrcTy = SrcVecTy->getElementType();
3425 DestTy = DestVecTy->getElementType();
3426 }
3427
3428 // Get the bit sizes, we'll need these
3429 // FIXME: This doesn't work for scalable vector types with different element
3430 // counts that don't call getElementType above.
3431 unsigned SrcBits =
3432 SrcTy->getPrimitiveSizeInBits().getFixedValue(); // 0 for ptr
3433 unsigned DestBits =
3434 DestTy->getPrimitiveSizeInBits().getFixedValue(); // 0 for ptr
3435
3436 // Run through the possibilities ...
3437 if (DestTy->isByteTy()) { // Casting to byte
3438 if (SrcTy->isIntegerTy()) { // Casting from integral
3439 assert(DestBits == SrcBits && "Illegal cast from integer to byte type");
3440 return BitCast;
3441 } else if (SrcTy->isPointerTy()) { // Casting from pointer
3442 assert(DestBits == SrcBits && "Illegal cast from pointer to byte type");
3443 return BitCast;
3444 }
3445 llvm_unreachable("Illegal cast to byte type");
3446 } else if (DestTy->isIntegerTy()) { // Casting to integral
3447 if (SrcTy->isIntegerTy()) { // Casting from integral
3448 if (DestBits < SrcBits)
3449 return Trunc; // int -> smaller int
3450 else if (DestBits > SrcBits) { // its an extension
3451 if (SrcIsSigned)
3452 return SExt; // signed -> SEXT
3453 else
3454 return ZExt; // unsigned -> ZEXT
3455 } else {
3456 return BitCast; // Same size, No-op cast
3457 }
3458 } else if (SrcTy->isFloatingPointTy()) { // Casting from floating pt
3459 if (DestIsSigned)
3460 return FPToSI; // FP -> sint
3461 else
3462 return FPToUI; // FP -> uint
3463 } else if (SrcTy->isVectorTy()) {
3464 assert(DestBits == SrcBits &&
3465 "Casting vector to integer of different width");
3466 return BitCast; // Same size, no-op cast
3467 } else {
3468 assert(SrcTy->isPointerTy() &&
3469 "Casting from a value that is not first-class type");
3470 return PtrToInt; // ptr -> int
3471 }
3472 } else if (DestTy->isFloatingPointTy()) { // Casting to floating pt
3473 if (SrcTy->isIntegerTy()) { // Casting from integral
3474 if (SrcIsSigned)
3475 return SIToFP; // sint -> FP
3476 else
3477 return UIToFP; // uint -> FP
3478 } else if (SrcTy->isFloatingPointTy()) { // Casting from floating pt
3479 if (DestBits < SrcBits) {
3480 return FPTrunc; // FP -> smaller FP
3481 } else if (DestBits > SrcBits) {
3482 return FPExt; // FP -> larger FP
3483 } else {
3484 return BitCast; // same size, no-op cast
3485 }
3486 } else if (SrcTy->isVectorTy()) {
3487 assert(DestBits == SrcBits &&
3488 "Casting vector to floating point of different width");
3489 return BitCast; // same size, no-op cast
3490 }
3491 llvm_unreachable("Casting pointer or non-first class to float");
3492 } else if (DestTy->isVectorTy()) {
3493 assert(DestBits == SrcBits &&
3494 "Illegal cast to vector (wrong type or size)");
3495 return BitCast;
3496 } else if (DestTy->isPointerTy()) {
3497 if (SrcTy->isPointerTy()) {
3498 if (DestTy->getPointerAddressSpace() != SrcTy->getPointerAddressSpace())
3499 return AddrSpaceCast;
3500 return BitCast; // ptr -> ptr
3501 } else if (SrcTy->isIntegerTy()) {
3502 return IntToPtr; // int -> ptr
3503 }
3504 llvm_unreachable("Casting pointer to other than pointer or int");
3505 }
3506 llvm_unreachable("Casting to type that is not first-class");
3507}
3508
3509//===----------------------------------------------------------------------===//
3510// CastInst SubClass Constructors
3511//===----------------------------------------------------------------------===//
3512
3513/// Check that the construction parameters for a CastInst are correct. This
3514/// could be broken out into the separate constructors but it is useful to have
3515/// it in one place and to eliminate the redundant code for getting the sizes
3516/// of the types involved.
3517bool
3518CastInst::castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy) {
3519 if (!SrcTy->isFirstClassType() || !DstTy->isFirstClassType() ||
3520 SrcTy->isAggregateType() || DstTy->isAggregateType())
3521 return false;
3522
3523 // Get the size of the types in bits, and whether we are dealing
3524 // with vector types, we'll need this later.
3525 bool SrcIsVec = isa<VectorType>(Val: SrcTy);
3526 bool DstIsVec = isa<VectorType>(Val: DstTy);
3527 unsigned SrcScalarBitSize = SrcTy->getScalarSizeInBits();
3528 unsigned DstScalarBitSize = DstTy->getScalarSizeInBits();
3529
3530 // If these are vector types, get the lengths of the vectors (using zero for
3531 // scalar types means that checking that vector lengths match also checks that
3532 // scalars are not being converted to vectors or vectors to scalars).
3533 ElementCount SrcEC = SrcIsVec ? cast<VectorType>(Val: SrcTy)->getElementCount()
3534 : ElementCount::getFixed(MinVal: 0);
3535 ElementCount DstEC = DstIsVec ? cast<VectorType>(Val: DstTy)->getElementCount()
3536 : ElementCount::getFixed(MinVal: 0);
3537
3538 // Switch on the opcode provided
3539 switch (op) {
3540 default: return false; // This is an input error
3541 case Instruction::Trunc:
3542 return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&
3543 SrcEC == DstEC && SrcScalarBitSize > DstScalarBitSize;
3544 case Instruction::ZExt:
3545 return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&
3546 SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;
3547 case Instruction::SExt:
3548 return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&
3549 SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;
3550 case Instruction::FPTrunc:
3551 return SrcTy->isFPOrFPVectorTy() && DstTy->isFPOrFPVectorTy() &&
3552 SrcEC == DstEC && SrcScalarBitSize > DstScalarBitSize;
3553 case Instruction::FPExt:
3554 return SrcTy->isFPOrFPVectorTy() && DstTy->isFPOrFPVectorTy() &&
3555 SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;
3556 case Instruction::UIToFP:
3557 case Instruction::SIToFP:
3558 return SrcTy->isIntOrIntVectorTy() && DstTy->isFPOrFPVectorTy() &&
3559 SrcEC == DstEC;
3560 case Instruction::FPToUI:
3561 case Instruction::FPToSI:
3562 return SrcTy->isFPOrFPVectorTy() && DstTy->isIntOrIntVectorTy() &&
3563 SrcEC == DstEC;
3564 case Instruction::PtrToAddr:
3565 case Instruction::PtrToInt:
3566 if (SrcEC != DstEC)
3567 return false;
3568 return SrcTy->isPtrOrPtrVectorTy() && DstTy->isIntOrIntVectorTy();
3569 case Instruction::IntToPtr:
3570 if (SrcEC != DstEC)
3571 return false;
3572 return SrcTy->isIntOrIntVectorTy() && DstTy->isPtrOrPtrVectorTy();
3573 case Instruction::BitCast: {
3574 Type *SrcScalarTy = SrcTy->getScalarType();
3575 if (SrcScalarTy->isTargetExtTy() || !SrcScalarTy->isSingleValueType())
3576 return false;
3577 Type *DstScalarTy = DstTy->getScalarType();
3578 if (DstScalarTy->isTargetExtTy() || !DstScalarTy->isSingleValueType())
3579 return false;
3580
3581 PointerType *SrcPtrTy = dyn_cast<PointerType>(Val: SrcScalarTy);
3582 PointerType *DstPtrTy = dyn_cast<PointerType>(Val: DstScalarTy);
3583
3584 // BitCast implies a no-op cast of type only. No bits change.
3585 // However, you can't cast pointers to anything but pointers/bytes.
3586 if ((SrcPtrTy && DstTy->isByteOrByteVectorTy()) ||
3587 (SrcTy->isByteOrByteVectorTy() && DstPtrTy))
3588 return true;
3589 if (!SrcPtrTy != !DstPtrTy)
3590 return false;
3591
3592 // For non-pointer cases, the cast is okay if the source and destination bit
3593 // widths are identical.
3594 if (!SrcPtrTy)
3595 return SrcTy->getPrimitiveSizeInBits() == DstTy->getPrimitiveSizeInBits();
3596
3597 // If both are pointers then the address spaces must match.
3598 if (SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace())
3599 return false;
3600
3601 // A vector of pointers must have the same number of elements.
3602 if (SrcIsVec && DstIsVec)
3603 return SrcEC == DstEC;
3604 if (SrcIsVec)
3605 return SrcEC == ElementCount::getFixed(MinVal: 1);
3606 if (DstIsVec)
3607 return DstEC == ElementCount::getFixed(MinVal: 1);
3608
3609 return true;
3610 }
3611 case Instruction::AddrSpaceCast: {
3612 PointerType *SrcPtrTy = dyn_cast<PointerType>(Val: SrcTy->getScalarType());
3613 if (!SrcPtrTy)
3614 return false;
3615
3616 PointerType *DstPtrTy = dyn_cast<PointerType>(Val: DstTy->getScalarType());
3617 if (!DstPtrTy)
3618 return false;
3619
3620 if (SrcPtrTy->getAddressSpace() == DstPtrTy->getAddressSpace())
3621 return false;
3622
3623 return SrcEC == DstEC;
3624 }
3625 }
3626}
3627
3628TruncInst::TruncInst(Value *S, Type *Ty, const Twine &Name,
3629 InsertPosition InsertBefore)
3630 : CastInst(Ty, Trunc, S, Name, InsertBefore) {
3631 assert(castIsValid(getOpcode(), S, Ty) && "Illegal Trunc");
3632}
3633
3634ZExtInst::ZExtInst(Value *S, Type *Ty, const Twine &Name,
3635 InsertPosition InsertBefore)
3636 : CastInst(Ty, ZExt, S, Name, InsertBefore) {
3637 assert(castIsValid(getOpcode(), S, Ty) && "Illegal ZExt");
3638}
3639
3640SExtInst::SExtInst(Value *S, Type *Ty, const Twine &Name,
3641 InsertPosition InsertBefore)
3642 : CastInst(Ty, SExt, S, Name, InsertBefore) {
3643 assert(castIsValid(getOpcode(), S, Ty) && "Illegal SExt");
3644}
3645
3646FPTruncInst::FPTruncInst(Value *S, Type *Ty, const Twine &Name,
3647 InsertPosition InsertBefore)
3648 : CastInst(Ty, FPTrunc, S, Name, InsertBefore) {
3649 assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPTrunc");
3650}
3651
3652FPExtInst::FPExtInst(Value *S, Type *Ty, const Twine &Name,
3653 InsertPosition InsertBefore)
3654 : CastInst(Ty, FPExt, S, Name, InsertBefore) {
3655 assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPExt");
3656}
3657
3658UIToFPInst::UIToFPInst(Value *S, Type *Ty, const Twine &Name,
3659 InsertPosition InsertBefore)
3660 : CastInst(Ty, UIToFP, S, Name, InsertBefore) {
3661 assert(castIsValid(getOpcode(), S, Ty) && "Illegal UIToFP");
3662}
3663
3664SIToFPInst::SIToFPInst(Value *S, Type *Ty, const Twine &Name,
3665 InsertPosition InsertBefore)
3666 : CastInst(Ty, SIToFP, S, Name, InsertBefore) {
3667 assert(castIsValid(getOpcode(), S, Ty) && "Illegal SIToFP");
3668}
3669
3670FPToUIInst::FPToUIInst(Value *S, Type *Ty, const Twine &Name,
3671 InsertPosition InsertBefore)
3672 : CastInst(Ty, FPToUI, S, Name, InsertBefore) {
3673 assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPToUI");
3674}
3675
3676FPToSIInst::FPToSIInst(Value *S, Type *Ty, const Twine &Name,
3677 InsertPosition InsertBefore)
3678 : CastInst(Ty, FPToSI, S, Name, InsertBefore) {
3679 assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPToSI");
3680}
3681
3682PtrToIntInst::PtrToIntInst(Value *S, Type *Ty, const Twine &Name,
3683 InsertPosition InsertBefore)
3684 : CastInst(Ty, PtrToInt, S, Name, InsertBefore) {
3685 assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToInt");
3686}
3687
3688PtrToAddrInst::PtrToAddrInst(Value *S, Type *Ty, const Twine &Name,
3689 InsertPosition InsertBefore)
3690 : CastInst(Ty, PtrToAddr, S, Name, InsertBefore) {
3691 assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToAddr");
3692}
3693
3694IntToPtrInst::IntToPtrInst(Value *S, Type *Ty, const Twine &Name,
3695 InsertPosition InsertBefore)
3696 : CastInst(Ty, IntToPtr, S, Name, InsertBefore) {
3697 assert(castIsValid(getOpcode(), S, Ty) && "Illegal IntToPtr");
3698}
3699
3700BitCastInst::BitCastInst(Value *S, Type *Ty, const Twine &Name,
3701 InsertPosition InsertBefore)
3702 : CastInst(Ty, BitCast, S, Name, InsertBefore) {
3703 assert(castIsValid(getOpcode(), S, Ty) && "Illegal BitCast");
3704}
3705
3706AddrSpaceCastInst::AddrSpaceCastInst(Value *S, Type *Ty, const Twine &Name,
3707 InsertPosition InsertBefore)
3708 : CastInst(Ty, AddrSpaceCast, S, Name, InsertBefore) {
3709 assert(castIsValid(getOpcode(), S, Ty) && "Illegal AddrSpaceCast");
3710}
3711
3712//===----------------------------------------------------------------------===//
3713// CmpInst Classes
3714//===----------------------------------------------------------------------===//
3715
3716CmpInst::CmpInst(Type *ty, OtherOps op, Predicate predicate, Value *LHS,
3717 Value *RHS, const Twine &Name, InsertPosition InsertBefore)
3718 : Instruction(ty, op, AllocMarker, InsertBefore) {
3719 Op<0>() = LHS;
3720 Op<1>() = RHS;
3721 setPredicate(predicate);
3722 setName(Name);
3723}
3724
3725CmpInst *CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2,
3726 const Twine &Name, InsertPosition InsertBefore) {
3727 if (Op == Instruction::ICmp) {
3728 if (InsertBefore.isValid())
3729 return new ICmpInst(InsertBefore, CmpInst::Predicate(predicate),
3730 S1, S2, Name);
3731 else
3732 return new ICmpInst(CmpInst::Predicate(predicate),
3733 S1, S2, Name);
3734 }
3735
3736 if (InsertBefore.isValid())
3737 return new FCmpInst(InsertBefore, CmpInst::Predicate(predicate),
3738 S1, S2, Name);
3739 else
3740 return new FCmpInst(CmpInst::Predicate(predicate),
3741 S1, S2, Name);
3742}
3743
3744CmpInst *CmpInst::CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1,
3745 Value *S2,
3746 const Instruction *FlagsSource,
3747 const Twine &Name,
3748 InsertPosition InsertBefore) {
3749 CmpInst *Inst = Create(Op, predicate: Pred, S1, S2, Name, InsertBefore);
3750 Inst->copyIRFlags(V: FlagsSource);
3751 return Inst;
3752}
3753
3754void CmpInst::swapOperands() {
3755 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: this))
3756 IC->swapOperands();
3757 else
3758 cast<FCmpInst>(Val: this)->swapOperands();
3759}
3760
3761bool CmpInst::isCommutative() const {
3762 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: this))
3763 return IC->isCommutative();
3764 return cast<FCmpInst>(Val: this)->isCommutative();
3765}
3766
3767bool CmpInst::isEquality(Predicate P) {
3768 if (ICmpInst::isIntPredicate(P))
3769 return ICmpInst::isEquality(P);
3770 if (FCmpInst::isFPPredicate(P))
3771 return FCmpInst::isEquality(Pred: P);
3772 llvm_unreachable("Unsupported predicate kind");
3773}
3774
3775// Returns true if either operand of CmpInst is a provably non-zero
3776// floating-point constant.
3777static bool hasNonZeroFPOperands(const CmpInst *Cmp) {
3778 auto *LHS = dyn_cast<Constant>(Val: Cmp->getOperand(i_nocapture: 0));
3779 auto *RHS = dyn_cast<Constant>(Val: Cmp->getOperand(i_nocapture: 1));
3780 if (auto *Const = LHS ? LHS : RHS) {
3781 using namespace llvm::PatternMatch;
3782 return match(V: Const, P: m_NonZeroNotDenormalFP());
3783 }
3784 return false;
3785}
3786
3787// Floating-point equality is not an equivalence when comparing +0.0 with
3788// -0.0, when comparing NaN with another value, or when flushing
3789// denormals-to-zero.
3790bool CmpInst::isEquivalence(bool Invert) const {
3791 switch (Invert ? getInversePredicate() : getPredicate()) {
3792 case CmpInst::Predicate::ICMP_EQ:
3793 return true;
3794 case CmpInst::Predicate::FCMP_UEQ:
3795 if (!hasNoNaNs())
3796 return false;
3797 [[fallthrough]];
3798 case CmpInst::Predicate::FCMP_OEQ:
3799 return hasNonZeroFPOperands(Cmp: this);
3800 default:
3801 return false;
3802 }
3803}
3804
3805CmpInst::Predicate CmpInst::getInversePredicate(Predicate pred) {
3806 switch (pred) {
3807 default: llvm_unreachable("Unknown cmp predicate!");
3808 case ICMP_EQ: return ICMP_NE;
3809 case ICMP_NE: return ICMP_EQ;
3810 case ICMP_UGT: return ICMP_ULE;
3811 case ICMP_ULT: return ICMP_UGE;
3812 case ICMP_UGE: return ICMP_ULT;
3813 case ICMP_ULE: return ICMP_UGT;
3814 case ICMP_SGT: return ICMP_SLE;
3815 case ICMP_SLT: return ICMP_SGE;
3816 case ICMP_SGE: return ICMP_SLT;
3817 case ICMP_SLE: return ICMP_SGT;
3818
3819 case FCMP_OEQ: return FCMP_UNE;
3820 case FCMP_ONE: return FCMP_UEQ;
3821 case FCMP_OGT: return FCMP_ULE;
3822 case FCMP_OLT: return FCMP_UGE;
3823 case FCMP_OGE: return FCMP_ULT;
3824 case FCMP_OLE: return FCMP_UGT;
3825 case FCMP_UEQ: return FCMP_ONE;
3826 case FCMP_UNE: return FCMP_OEQ;
3827 case FCMP_UGT: return FCMP_OLE;
3828 case FCMP_ULT: return FCMP_OGE;
3829 case FCMP_UGE: return FCMP_OLT;
3830 case FCMP_ULE: return FCMP_OGT;
3831 case FCMP_ORD: return FCMP_UNO;
3832 case FCMP_UNO: return FCMP_ORD;
3833 case FCMP_TRUE: return FCMP_FALSE;
3834 case FCMP_FALSE: return FCMP_TRUE;
3835 }
3836}
3837
3838StringRef CmpInst::getPredicateName(Predicate Pred) {
3839 switch (Pred) {
3840 default: return "unknown";
3841 case FCmpInst::FCMP_FALSE: return "false";
3842 case FCmpInst::FCMP_OEQ: return "oeq";
3843 case FCmpInst::FCMP_OGT: return "ogt";
3844 case FCmpInst::FCMP_OGE: return "oge";
3845 case FCmpInst::FCMP_OLT: return "olt";
3846 case FCmpInst::FCMP_OLE: return "ole";
3847 case FCmpInst::FCMP_ONE: return "one";
3848 case FCmpInst::FCMP_ORD: return "ord";
3849 case FCmpInst::FCMP_UNO: return "uno";
3850 case FCmpInst::FCMP_UEQ: return "ueq";
3851 case FCmpInst::FCMP_UGT: return "ugt";
3852 case FCmpInst::FCMP_UGE: return "uge";
3853 case FCmpInst::FCMP_ULT: return "ult";
3854 case FCmpInst::FCMP_ULE: return "ule";
3855 case FCmpInst::FCMP_UNE: return "une";
3856 case FCmpInst::FCMP_TRUE: return "true";
3857 case ICmpInst::ICMP_EQ: return "eq";
3858 case ICmpInst::ICMP_NE: return "ne";
3859 case ICmpInst::ICMP_SGT: return "sgt";
3860 case ICmpInst::ICMP_SGE: return "sge";
3861 case ICmpInst::ICMP_SLT: return "slt";
3862 case ICmpInst::ICMP_SLE: return "sle";
3863 case ICmpInst::ICMP_UGT: return "ugt";
3864 case ICmpInst::ICMP_UGE: return "uge";
3865 case ICmpInst::ICMP_ULT: return "ult";
3866 case ICmpInst::ICMP_ULE: return "ule";
3867 }
3868}
3869
3870raw_ostream &llvm::operator<<(raw_ostream &OS, CmpInst::Predicate Pred) {
3871 OS << CmpInst::getPredicateName(Pred);
3872 return OS;
3873}
3874
3875ICmpInst::Predicate ICmpInst::getSignedPredicate(Predicate pred) {
3876 switch (pred) {
3877 default: llvm_unreachable("Unknown icmp predicate!");
3878 case ICMP_EQ: case ICMP_NE:
3879 case ICMP_SGT: case ICMP_SLT: case ICMP_SGE: case ICMP_SLE:
3880 return pred;
3881 case ICMP_UGT: return ICMP_SGT;
3882 case ICMP_ULT: return ICMP_SLT;
3883 case ICMP_UGE: return ICMP_SGE;
3884 case ICMP_ULE: return ICMP_SLE;
3885 }
3886}
3887
3888ICmpInst::Predicate ICmpInst::getUnsignedPredicate(Predicate pred) {
3889 switch (pred) {
3890 default: llvm_unreachable("Unknown icmp predicate!");
3891 case ICMP_EQ: case ICMP_NE:
3892 case ICMP_UGT: case ICMP_ULT: case ICMP_UGE: case ICMP_ULE:
3893 return pred;
3894 case ICMP_SGT: return ICMP_UGT;
3895 case ICMP_SLT: return ICMP_ULT;
3896 case ICMP_SGE: return ICMP_UGE;
3897 case ICMP_SLE: return ICMP_ULE;
3898 }
3899}
3900
3901CmpInst::Predicate CmpInst::getSwappedPredicate(Predicate pred) {
3902 switch (pred) {
3903 default: llvm_unreachable("Unknown cmp predicate!");
3904 case ICMP_EQ: case ICMP_NE:
3905 return pred;
3906 case ICMP_SGT: return ICMP_SLT;
3907 case ICMP_SLT: return ICMP_SGT;
3908 case ICMP_SGE: return ICMP_SLE;
3909 case ICMP_SLE: return ICMP_SGE;
3910 case ICMP_UGT: return ICMP_ULT;
3911 case ICMP_ULT: return ICMP_UGT;
3912 case ICMP_UGE: return ICMP_ULE;
3913 case ICMP_ULE: return ICMP_UGE;
3914
3915 case FCMP_FALSE: case FCMP_TRUE:
3916 case FCMP_OEQ: case FCMP_ONE:
3917 case FCMP_UEQ: case FCMP_UNE:
3918 case FCMP_ORD: case FCMP_UNO:
3919 return pred;
3920 case FCMP_OGT: return FCMP_OLT;
3921 case FCMP_OLT: return FCMP_OGT;
3922 case FCMP_OGE: return FCMP_OLE;
3923 case FCMP_OLE: return FCMP_OGE;
3924 case FCMP_UGT: return FCMP_ULT;
3925 case FCMP_ULT: return FCMP_UGT;
3926 case FCMP_UGE: return FCMP_ULE;
3927 case FCMP_ULE: return FCMP_UGE;
3928 }
3929}
3930
3931bool CmpInst::isNonStrictPredicate(Predicate pred) {
3932 switch (pred) {
3933 case ICMP_SGE:
3934 case ICMP_SLE:
3935 case ICMP_UGE:
3936 case ICMP_ULE:
3937 case FCMP_OGE:
3938 case FCMP_OLE:
3939 case FCMP_UGE:
3940 case FCMP_ULE:
3941 return true;
3942 default:
3943 return false;
3944 }
3945}
3946
3947bool CmpInst::isStrictPredicate(Predicate pred) {
3948 switch (pred) {
3949 case ICMP_SGT:
3950 case ICMP_SLT:
3951 case ICMP_UGT:
3952 case ICMP_ULT:
3953 case FCMP_OGT:
3954 case FCMP_OLT:
3955 case FCMP_UGT:
3956 case FCMP_ULT:
3957 return true;
3958 default:
3959 return false;
3960 }
3961}
3962
3963CmpInst::Predicate CmpInst::getStrictPredicate(Predicate pred) {
3964 switch (pred) {
3965 case ICMP_SGE:
3966 return ICMP_SGT;
3967 case ICMP_SLE:
3968 return ICMP_SLT;
3969 case ICMP_UGE:
3970 return ICMP_UGT;
3971 case ICMP_ULE:
3972 return ICMP_ULT;
3973 case FCMP_OGE:
3974 return FCMP_OGT;
3975 case FCMP_OLE:
3976 return FCMP_OLT;
3977 case FCMP_UGE:
3978 return FCMP_UGT;
3979 case FCMP_ULE:
3980 return FCMP_ULT;
3981 default:
3982 return pred;
3983 }
3984}
3985
3986CmpInst::Predicate CmpInst::getNonStrictPredicate(Predicate pred) {
3987 switch (pred) {
3988 case ICMP_SGT:
3989 return ICMP_SGE;
3990 case ICMP_SLT:
3991 return ICMP_SLE;
3992 case ICMP_UGT:
3993 return ICMP_UGE;
3994 case ICMP_ULT:
3995 return ICMP_ULE;
3996 case FCMP_OGT:
3997 return FCMP_OGE;
3998 case FCMP_OLT:
3999 return FCMP_OLE;
4000 case FCMP_UGT:
4001 return FCMP_UGE;
4002 case FCMP_ULT:
4003 return FCMP_ULE;
4004 default:
4005 return pred;
4006 }
4007}
4008
4009CmpInst::Predicate CmpInst::getFlippedStrictnessPredicate(Predicate pred) {
4010 assert(CmpInst::isRelational(pred) && "Call only with relational predicate!");
4011
4012 if (isStrictPredicate(pred))
4013 return getNonStrictPredicate(pred);
4014 if (isNonStrictPredicate(pred))
4015 return getStrictPredicate(pred);
4016
4017 llvm_unreachable("Unknown predicate!");
4018}
4019
4020bool ICmpInst::compare(const APInt &LHS, const APInt &RHS,
4021 ICmpInst::Predicate Pred) {
4022 assert(ICmpInst::isIntPredicate(Pred) && "Only for integer predicates!");
4023 switch (Pred) {
4024 case ICmpInst::Predicate::ICMP_EQ:
4025 return LHS.eq(RHS);
4026 case ICmpInst::Predicate::ICMP_NE:
4027 return LHS.ne(RHS);
4028 case ICmpInst::Predicate::ICMP_UGT:
4029 return LHS.ugt(RHS);
4030 case ICmpInst::Predicate::ICMP_UGE:
4031 return LHS.uge(RHS);
4032 case ICmpInst::Predicate::ICMP_ULT:
4033 return LHS.ult(RHS);
4034 case ICmpInst::Predicate::ICMP_ULE:
4035 return LHS.ule(RHS);
4036 case ICmpInst::Predicate::ICMP_SGT:
4037 return LHS.sgt(RHS);
4038 case ICmpInst::Predicate::ICMP_SGE:
4039 return LHS.sge(RHS);
4040 case ICmpInst::Predicate::ICMP_SLT:
4041 return LHS.slt(RHS);
4042 case ICmpInst::Predicate::ICMP_SLE:
4043 return LHS.sle(RHS);
4044 default:
4045 llvm_unreachable("Unexpected non-integer predicate.");
4046 };
4047}
4048
4049bool FCmpInst::compare(const APFloat &LHS, const APFloat &RHS,
4050 FCmpInst::Predicate Pred) {
4051 APFloat::cmpResult R = LHS.compare(RHS);
4052 switch (Pred) {
4053 default:
4054 llvm_unreachable("Invalid FCmp Predicate");
4055 case FCmpInst::FCMP_FALSE:
4056 return false;
4057 case FCmpInst::FCMP_TRUE:
4058 return true;
4059 case FCmpInst::FCMP_UNO:
4060 return R == APFloat::cmpUnordered;
4061 case FCmpInst::FCMP_ORD:
4062 return R != APFloat::cmpUnordered;
4063 case FCmpInst::FCMP_UEQ:
4064 return R == APFloat::cmpUnordered || R == APFloat::cmpEqual;
4065 case FCmpInst::FCMP_OEQ:
4066 return R == APFloat::cmpEqual;
4067 case FCmpInst::FCMP_UNE:
4068 return R != APFloat::cmpEqual;
4069 case FCmpInst::FCMP_ONE:
4070 return R == APFloat::cmpLessThan || R == APFloat::cmpGreaterThan;
4071 case FCmpInst::FCMP_ULT:
4072 return R == APFloat::cmpUnordered || R == APFloat::cmpLessThan;
4073 case FCmpInst::FCMP_OLT:
4074 return R == APFloat::cmpLessThan;
4075 case FCmpInst::FCMP_UGT:
4076 return R == APFloat::cmpUnordered || R == APFloat::cmpGreaterThan;
4077 case FCmpInst::FCMP_OGT:
4078 return R == APFloat::cmpGreaterThan;
4079 case FCmpInst::FCMP_ULE:
4080 return R != APFloat::cmpGreaterThan;
4081 case FCmpInst::FCMP_OLE:
4082 return R == APFloat::cmpLessThan || R == APFloat::cmpEqual;
4083 case FCmpInst::FCMP_UGE:
4084 return R != APFloat::cmpLessThan;
4085 case FCmpInst::FCMP_OGE:
4086 return R == APFloat::cmpGreaterThan || R == APFloat::cmpEqual;
4087 }
4088}
4089
4090std::optional<bool> ICmpInst::compare(const KnownBits &LHS,
4091 const KnownBits &RHS,
4092 ICmpInst::Predicate Pred) {
4093 switch (Pred) {
4094 case ICmpInst::ICMP_EQ:
4095 return KnownBits::eq(LHS, RHS);
4096 case ICmpInst::ICMP_NE:
4097 return KnownBits::ne(LHS, RHS);
4098 case ICmpInst::ICMP_UGE:
4099 return KnownBits::uge(LHS, RHS);
4100 case ICmpInst::ICMP_UGT:
4101 return KnownBits::ugt(LHS, RHS);
4102 case ICmpInst::ICMP_ULE:
4103 return KnownBits::ule(LHS, RHS);
4104 case ICmpInst::ICMP_ULT:
4105 return KnownBits::ult(LHS, RHS);
4106 case ICmpInst::ICMP_SGE:
4107 return KnownBits::sge(LHS, RHS);
4108 case ICmpInst::ICMP_SGT:
4109 return KnownBits::sgt(LHS, RHS);
4110 case ICmpInst::ICMP_SLE:
4111 return KnownBits::sle(LHS, RHS);
4112 case ICmpInst::ICMP_SLT:
4113 return KnownBits::slt(LHS, RHS);
4114 default:
4115 llvm_unreachable("Unexpected non-integer predicate.");
4116 }
4117}
4118
4119CmpInst::Predicate ICmpInst::getFlippedSignednessPredicate(Predicate pred) {
4120 if (CmpInst::isEquality(P: pred))
4121 return pred;
4122 if (isSigned(Pred: pred))
4123 return getUnsignedPredicate(pred);
4124 if (isUnsigned(Pred: pred))
4125 return getSignedPredicate(pred);
4126
4127 llvm_unreachable("Unknown predicate!");
4128}
4129
4130bool CmpInst::isOrdered(Predicate predicate) {
4131 switch (predicate) {
4132 default: return false;
4133 case FCmpInst::FCMP_OEQ: case FCmpInst::FCMP_ONE: case FCmpInst::FCMP_OGT:
4134 case FCmpInst::FCMP_OLT: case FCmpInst::FCMP_OGE: case FCmpInst::FCMP_OLE:
4135 case FCmpInst::FCMP_ORD: return true;
4136 }
4137}
4138
4139bool CmpInst::isUnordered(Predicate predicate) {
4140 switch (predicate) {
4141 default: return false;
4142 case FCmpInst::FCMP_UEQ: case FCmpInst::FCMP_UNE: case FCmpInst::FCMP_UGT:
4143 case FCmpInst::FCMP_ULT: case FCmpInst::FCMP_UGE: case FCmpInst::FCMP_ULE:
4144 case FCmpInst::FCMP_UNO: return true;
4145 }
4146}
4147
4148bool CmpInst::isTrueWhenEqual(Predicate predicate) {
4149 switch(predicate) {
4150 default: return false;
4151 case ICMP_EQ: case ICMP_UGE: case ICMP_ULE: case ICMP_SGE: case ICMP_SLE:
4152 case FCMP_TRUE: case FCMP_UEQ: case FCMP_UGE: case FCMP_ULE: return true;
4153 }
4154}
4155
4156bool CmpInst::isFalseWhenEqual(Predicate predicate) {
4157 switch(predicate) {
4158 case ICMP_NE: case ICMP_UGT: case ICMP_ULT: case ICMP_SGT: case ICMP_SLT:
4159 case FCMP_FALSE: case FCMP_ONE: case FCMP_OGT: case FCMP_OLT: return true;
4160 default: return false;
4161 }
4162}
4163
4164static bool isImpliedTrueByMatchingCmp(CmpPredicate Pred1, CmpPredicate Pred2) {
4165 // If the predicates match, then we know the first condition implies the
4166 // second is true.
4167 if (CmpPredicate::getMatching(A: Pred1, B: Pred2))
4168 return true;
4169
4170 if (Pred1.hasSameSign() && CmpInst::isSigned(Pred: Pred2))
4171 Pred1 = ICmpInst::getFlippedSignednessPredicate(pred: Pred1);
4172 else if (Pred2.hasSameSign() && CmpInst::isSigned(Pred: Pred1))
4173 Pred2 = ICmpInst::getFlippedSignednessPredicate(pred: Pred2);
4174
4175 switch (Pred1) {
4176 default:
4177 break;
4178 case CmpInst::ICMP_EQ:
4179 // A == B implies A >=u B, A <=u B, A >=s B, and A <=s B are true.
4180 return Pred2 == CmpInst::ICMP_UGE || Pred2 == CmpInst::ICMP_ULE ||
4181 Pred2 == CmpInst::ICMP_SGE || Pred2 == CmpInst::ICMP_SLE;
4182 case CmpInst::ICMP_UGT: // A >u B implies A != B and A >=u B are true.
4183 return Pred2 == CmpInst::ICMP_NE || Pred2 == CmpInst::ICMP_UGE;
4184 case CmpInst::ICMP_ULT: // A <u B implies A != B and A <=u B are true.
4185 return Pred2 == CmpInst::ICMP_NE || Pred2 == CmpInst::ICMP_ULE;
4186 case CmpInst::ICMP_SGT: // A >s B implies A != B and A >=s B are true.
4187 return Pred2 == CmpInst::ICMP_NE || Pred2 == CmpInst::ICMP_SGE;
4188 case CmpInst::ICMP_SLT: // A <s B implies A != B and A <=s B are true.
4189 return Pred2 == CmpInst::ICMP_NE || Pred2 == CmpInst::ICMP_SLE;
4190 }
4191 return false;
4192}
4193
4194static bool isImpliedFalseByMatchingCmp(CmpPredicate Pred1,
4195 CmpPredicate Pred2) {
4196 return isImpliedTrueByMatchingCmp(Pred1,
4197 Pred2: ICmpInst::getInverseCmpPredicate(Pred: Pred2));
4198}
4199
4200std::optional<bool> ICmpInst::isImpliedByMatchingCmp(CmpPredicate Pred1,
4201 CmpPredicate Pred2) {
4202 if (isImpliedTrueByMatchingCmp(Pred1, Pred2))
4203 return true;
4204 if (isImpliedFalseByMatchingCmp(Pred1, Pred2))
4205 return false;
4206 return std::nullopt;
4207}
4208
4209//===----------------------------------------------------------------------===//
4210// CmpPredicate Implementation
4211//===----------------------------------------------------------------------===//
4212
4213std::optional<CmpPredicate> CmpPredicate::getMatching(CmpPredicate A,
4214 CmpPredicate B) {
4215 if (A.Pred == B.Pred)
4216 return A.HasSameSign == B.HasSameSign ? A : CmpPredicate(A.Pred);
4217 if (CmpInst::isFPPredicate(P: A) || CmpInst::isFPPredicate(P: B))
4218 return {};
4219 if (A.HasSameSign &&
4220 A.Pred == ICmpInst::getFlippedSignednessPredicate(pred: B.Pred))
4221 return B.Pred;
4222 if (B.HasSameSign &&
4223 B.Pred == ICmpInst::getFlippedSignednessPredicate(pred: A.Pred))
4224 return A.Pred;
4225 return {};
4226}
4227
4228CmpInst::Predicate CmpPredicate::getPreferredSignedPredicate() const {
4229 return HasSameSign ? ICmpInst::getSignedPredicate(pred: Pred) : Pred;
4230}
4231
4232CmpPredicate CmpPredicate::get(const CmpInst *Cmp) {
4233 if (auto *ICI = dyn_cast<ICmpInst>(Val: Cmp))
4234 return ICI->getCmpPredicate();
4235 return Cmp->getPredicate();
4236}
4237
4238CmpPredicate CmpPredicate::getInverse(CmpPredicate P) {
4239 return {CmpInst::getInversePredicate(pred: P), P.hasSameSign()};
4240}
4241
4242CmpPredicate CmpPredicate::getSwapped(CmpPredicate P) {
4243 return {CmpInst::getSwappedPredicate(pred: P), P.hasSameSign()};
4244}
4245
4246CmpPredicate CmpPredicate::getSwapped(const CmpInst *Cmp) {
4247 return getSwapped(P: get(Cmp));
4248}
4249
4250//===----------------------------------------------------------------------===//
4251// SwitchInst Implementation
4252//===----------------------------------------------------------------------===//
4253
4254void SwitchInst::init(Value *Value, BasicBlock *Default, unsigned NumReserved) {
4255 assert(Value && Default && NumReserved);
4256 ReservedSpace = NumReserved;
4257 allocHungoffUses(N: ReservedSpace);
4258 setNumHungOffUseOperands(2);
4259
4260 Op<0>() = Value;
4261 Op<1>() = Default;
4262}
4263
4264/// SwitchInst ctor - Create a new switch instruction, specifying a value to
4265/// switch on and a default destination. The number of additional cases can
4266/// be specified here to make memory allocation more efficient. This
4267/// constructor can also autoinsert before another instruction.
4268SwitchInst::SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
4269 InsertPosition InsertBefore)
4270 : Instruction(Type::getVoidTy(C&: Value->getContext()), Instruction::Switch,
4271 AllocMarker, InsertBefore) {
4272 init(Value, Default, NumReserved: 2 + NumCases);
4273}
4274
4275SwitchInst::SwitchInst(const SwitchInst &SI)
4276 : Instruction(SI.getType(), Instruction::Switch, AllocMarker) {
4277 init(Value: SI.getCondition(), Default: SI.getDefaultDest(), NumReserved: SI.getNumOperands());
4278 setNumHungOffUseOperands(SI.getNumOperands());
4279 Use *OL = getOperandList();
4280 ConstantInt **VL = case_values();
4281 const Use *InOL = SI.getOperandList();
4282 ConstantInt *const *InVL = SI.case_values();
4283 for (unsigned i = 2, E = SI.getNumOperands(); i != E; ++i) {
4284 OL[i] = InOL[i];
4285 VL[i - 2] = InVL[i - 2];
4286 }
4287 SubclassOptionalData = SI.SubclassOptionalData;
4288}
4289
4290/// addCase - Add an entry to the switch instruction...
4291///
4292void SwitchInst::addCase(ConstantInt *OnVal, BasicBlock *Dest) {
4293 unsigned NewCaseIdx = getNumCases();
4294 unsigned OpNo = getNumOperands();
4295 if (OpNo + 1 > ReservedSpace)
4296 growOperands(); // Get more space!
4297 // Initialize some new operands.
4298 assert(OpNo < ReservedSpace && "Growing didn't work!");
4299 setNumHungOffUseOperands(OpNo + 1);
4300 CaseHandle Case(this, NewCaseIdx);
4301 Case.setValue(OnVal);
4302 Case.setSuccessor(Dest);
4303}
4304
4305/// removeCase - This method removes the specified case and its successor
4306/// from the switch instruction.
4307SwitchInst::CaseIt SwitchInst::removeCase(CaseIt I) {
4308 unsigned idx = I->getCaseIndex();
4309
4310 assert(2 + idx < getNumOperands() && "Case index out of range!!!");
4311
4312 unsigned NumOps = getNumOperands();
4313 Use *OL = getOperandList();
4314 ConstantInt **VL = case_values();
4315
4316 // Overwrite this case with the end of the list.
4317 if (2 + idx + 1 != NumOps) {
4318 OL[2 + idx] = OL[NumOps - 1];
4319 VL[idx] = VL[NumOps - 2 - 1];
4320 }
4321
4322 // Nuke the last value.
4323 OL[NumOps - 1].set(nullptr);
4324 VL[NumOps - 2 - 1] = nullptr;
4325 setNumHungOffUseOperands(NumOps - 1);
4326
4327 return CaseIt(this, idx);
4328}
4329
4330/// growOperands - grow operands - This grows the operand list in response
4331/// to a push_back style of operation. This grows the number of ops by 3 times.
4332///
4333void SwitchInst::growOperands() {
4334 unsigned e = getNumOperands();
4335 unsigned NumOps = e*3;
4336
4337 ReservedSpace = NumOps;
4338 growHungoffUses(N: ReservedSpace, /*WithExtraValues=*/true);
4339}
4340
4341void SwitchInstProfUpdateWrapper::init() {
4342 MDNode *ProfileData = getBranchWeightMDNode(I: SI);
4343 if (!ProfileData)
4344 return;
4345
4346 if (getNumBranchWeights(ProfileData: *ProfileData) != SI.getNumSuccessors()) {
4347 llvm_unreachable("number of prof branch_weights metadata operands does "
4348 "not correspond to number of succesors");
4349 }
4350
4351 SmallVector<uint32_t, 8> Weights;
4352 if (!extractBranchWeights(ProfileData, Weights))
4353 return;
4354 this->Weights = std::move(Weights);
4355}
4356
4357SwitchInst::CaseIt
4358SwitchInstProfUpdateWrapper::removeCase(SwitchInst::CaseIt I) {
4359 if (Weights) {
4360 assert(SI.getNumSuccessors() == Weights->size() &&
4361 "num of prof branch_weights must accord with num of successors");
4362 Changed = true;
4363 // Copy the last case to the place of the removed one and shrink.
4364 // This is tightly coupled with the way SwitchInst::removeCase() removes
4365 // the cases in SwitchInst::removeCase(CaseIt).
4366 (*Weights)[I->getCaseIndex() + 1] = Weights->back();
4367 Weights->pop_back();
4368 }
4369 return SI.removeCase(I);
4370}
4371
4372void SwitchInstProfUpdateWrapper::replaceDefaultDest(SwitchInst::CaseIt I) {
4373 auto *DestBlock = I->getCaseSuccessor();
4374 if (Weights) {
4375 auto Weight = getSuccessorWeight(idx: I->getCaseIndex() + 1);
4376 (*Weights)[0] = Weight.value();
4377 }
4378
4379 SI.setDefaultDest(DestBlock);
4380}
4381
4382void SwitchInstProfUpdateWrapper::addCase(
4383 ConstantInt *OnVal, BasicBlock *Dest,
4384 SwitchInstProfUpdateWrapper::CaseWeightOpt W) {
4385 SI.addCase(OnVal, Dest);
4386
4387 if (!Weights && W && *W) {
4388 Changed = true;
4389 Weights = SmallVector<uint32_t, 8>(SI.getNumSuccessors(), 0);
4390 (*Weights)[SI.getNumSuccessors() - 1] = *W;
4391 } else if (Weights) {
4392 Changed = true;
4393 Weights->push_back(Elt: W.value_or(u: 0));
4394 }
4395 if (Weights)
4396 assert(SI.getNumSuccessors() == Weights->size() &&
4397 "num of prof branch_weights must accord with num of successors");
4398}
4399
4400Instruction::InstListType::iterator
4401SwitchInstProfUpdateWrapper::eraseFromParent() {
4402 // Instruction is erased. Mark as unchanged to not touch it in the destructor.
4403 Changed = false;
4404 if (Weights)
4405 Weights->resize(N: 0);
4406 return SI.eraseFromParent();
4407}
4408
4409SwitchInstProfUpdateWrapper::CaseWeightOpt
4410SwitchInstProfUpdateWrapper::getSuccessorWeight(unsigned idx) {
4411 if (!Weights)
4412 return std::nullopt;
4413 return (*Weights)[idx];
4414}
4415
4416void SwitchInstProfUpdateWrapper::setSuccessorWeight(
4417 unsigned idx, SwitchInstProfUpdateWrapper::CaseWeightOpt W) {
4418 if (!W)
4419 return;
4420
4421 if (!Weights && *W)
4422 Weights = SmallVector<uint32_t, 8>(SI.getNumSuccessors(), 0);
4423
4424 if (Weights) {
4425 auto &OldW = (*Weights)[idx];
4426 if (*W != OldW) {
4427 Changed = true;
4428 OldW = *W;
4429 }
4430 }
4431}
4432
4433SwitchInstProfUpdateWrapper::CaseWeightOpt
4434SwitchInstProfUpdateWrapper::getSuccessorWeight(const SwitchInst &SI,
4435 unsigned idx) {
4436 if (MDNode *ProfileData = getValidBranchWeightMDNode(I: SI)) {
4437 SmallVector<uint32_t> Weights;
4438 extractFromBranchWeightMD32(ProfileData, Weights);
4439 return Weights[idx];
4440 }
4441
4442 return std::nullopt;
4443}
4444
4445//===----------------------------------------------------------------------===//
4446// IndirectBrInst Implementation
4447//===----------------------------------------------------------------------===//
4448
4449void IndirectBrInst::init(Value *Address, unsigned NumDests) {
4450 assert(Address && Address->getType()->isPointerTy() &&
4451 "Address of indirectbr must be a pointer");
4452 ReservedSpace = 1 + NumDests;
4453 allocHungoffUses(N: ReservedSpace);
4454 setNumHungOffUseOperands(1);
4455
4456 Op<0>() = Address;
4457}
4458
4459
4460/// growOperands - grow operands - This grows the operand list in response
4461/// to a push_back style of operation. This grows the number of ops by 2 times.
4462///
4463void IndirectBrInst::growOperands() {
4464 unsigned e = getNumOperands();
4465 unsigned NumOps = e*2;
4466
4467 ReservedSpace = NumOps;
4468 growHungoffUses(N: ReservedSpace);
4469}
4470
4471IndirectBrInst::IndirectBrInst(Value *Address, unsigned NumCases,
4472 InsertPosition InsertBefore)
4473 : Instruction(Type::getVoidTy(C&: Address->getContext()),
4474 Instruction::IndirectBr, AllocMarker, InsertBefore) {
4475 init(Address, NumDests: NumCases);
4476}
4477
4478IndirectBrInst::IndirectBrInst(const IndirectBrInst &IBI)
4479 : Instruction(Type::getVoidTy(C&: IBI.getContext()), Instruction::IndirectBr,
4480 AllocMarker) {
4481 allocHungoffUses(N: IBI.getNumOperands());
4482 setNumHungOffUseOperands(IBI.NumUserOperands);
4483 Use *OL = getOperandList();
4484 const Use *InOL = IBI.getOperandList();
4485 for (unsigned i = 0, E = IBI.getNumOperands(); i != E; ++i)
4486 OL[i] = InOL[i];
4487 SubclassOptionalData = IBI.SubclassOptionalData;
4488}
4489
4490/// addDestination - Add a destination.
4491///
4492void IndirectBrInst::addDestination(BasicBlock *DestBB) {
4493 unsigned OpNo = getNumOperands();
4494 if (OpNo+1 > ReservedSpace)
4495 growOperands(); // Get more space!
4496 // Initialize some new operands.
4497 assert(OpNo < ReservedSpace && "Growing didn't work!");
4498 setNumHungOffUseOperands(OpNo+1);
4499 getOperandList()[OpNo] = DestBB;
4500}
4501
4502/// removeDestination - This method removes the specified successor from the
4503/// indirectbr instruction.
4504void IndirectBrInst::removeDestination(unsigned idx) {
4505 assert(idx < getNumOperands()-1 && "Successor index out of range!");
4506
4507 unsigned NumOps = getNumOperands();
4508 Use *OL = getOperandList();
4509
4510 // Replace this value with the last one.
4511 OL[idx+1] = OL[NumOps-1];
4512
4513 // Nuke the last value.
4514 OL[NumOps-1].set(nullptr);
4515 setNumHungOffUseOperands(NumOps-1);
4516}
4517
4518//===----------------------------------------------------------------------===//
4519// FreezeInst Implementation
4520//===----------------------------------------------------------------------===//
4521
4522FreezeInst::FreezeInst(Value *S, const Twine &Name, InsertPosition InsertBefore)
4523 : UnaryInstruction(S->getType(), Freeze, S, InsertBefore) {
4524 setName(Name);
4525}
4526
4527//===----------------------------------------------------------------------===//
4528// cloneImpl() implementations
4529//===----------------------------------------------------------------------===//
4530
4531// Define these methods here so vtables don't get emitted into every translation
4532// unit that uses these classes.
4533
4534GetElementPtrInst *GetElementPtrInst::cloneImpl() const {
4535 IntrusiveOperandsAllocMarker AllocMarker{.NumOps: getNumOperands()};
4536 return new (AllocMarker) GetElementPtrInst(*this, AllocMarker);
4537}
4538
4539UnaryOperator *UnaryOperator::cloneImpl() const {
4540 return Create(Op: getOpcode(), S: Op<0>());
4541}
4542
4543FPUnaryOperator *FPUnaryOperator::cloneImpl() const {
4544 auto *I = static_cast<FPUnaryOperator *>(Create(Op: getOpcode(), S: Op<0>()));
4545 I->FMF = FMF;
4546 return I;
4547}
4548
4549BinaryOperator *BinaryOperator::cloneImpl() const {
4550 assert(!isa<FPBinaryOperator>(this) &&
4551 "Should call FPBinaryOperator::cloneImpl!");
4552 return Create(Op: getOpcode(), S1: Op<0>(), S2: Op<1>());
4553}
4554
4555FPBinaryOperator *FPBinaryOperator::cloneImpl() const {
4556 auto *I =
4557 static_cast<FPBinaryOperator *>(Create(Op: getOpcode(), S1: Op<0>(), S2: Op<1>()));
4558 I->FMF = FMF;
4559 return I;
4560}
4561
4562FCmpInst *FCmpInst::cloneImpl() const {
4563 auto *I = new FCmpInst(getPredicate(), Op<0>(), Op<1>());
4564 I->FMF = FMF;
4565 return I;
4566}
4567
4568ICmpInst *ICmpInst::cloneImpl() const {
4569 auto *Result = new ICmpInst(getPredicate(), Op<0>(), Op<1>());
4570 Result->setSameSign(hasSameSign());
4571 return Result;
4572}
4573
4574ExtractValueInst *ExtractValueInst::cloneImpl() const {
4575 return new ExtractValueInst(*this);
4576}
4577
4578InsertValueInst *InsertValueInst::cloneImpl() const {
4579 return new InsertValueInst(*this);
4580}
4581
4582AllocaInst *AllocaInst::cloneImpl() const {
4583 AllocaInst *Result = new AllocaInst(getAllocatedType(), getAddressSpace(),
4584 getOperand(i_nocapture: 0), getAlign());
4585 Result->setUsedWithInAlloca(isUsedWithInAlloca());
4586 Result->setSwiftError(isSwiftError());
4587 return Result;
4588}
4589
4590LoadInst *LoadInst::cloneImpl() const {
4591 return new LoadInst(getType(), getOperand(i_nocapture: 0), Twine(), getProperties(),
4592 /*InsertBefore=*/nullptr);
4593}
4594
4595StoreInst *StoreInst::cloneImpl() const {
4596 return new StoreInst(getOperand(i_nocapture: 0), getOperand(i_nocapture: 1), getProperties(),
4597 /*InsertBefore=*/nullptr);
4598}
4599
4600AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const {
4601 AtomicCmpXchgInst *Result = new AtomicCmpXchgInst(
4602 getOperand(i_nocapture: 0), getOperand(i_nocapture: 1), getOperand(i_nocapture: 2), getAlign(),
4603 getSuccessOrdering(), getFailureOrdering(), getSyncScopeID());
4604 Result->setVolatile(isVolatile());
4605 Result->setWeak(isWeak());
4606 return Result;
4607}
4608
4609AtomicRMWInst *AtomicRMWInst::cloneImpl() const {
4610 AtomicRMWInst *Result = new AtomicRMWInst(
4611 getOperation(), getOperand(i_nocapture: 0), getOperand(i_nocapture: 1), getAlign(), getOrdering(),
4612 getSyncScopeID(), isElementwise());
4613 Result->setVolatile(isVolatile());
4614 return Result;
4615}
4616
4617FenceInst *FenceInst::cloneImpl() const {
4618 return new FenceInst(getContext(), getOrdering(), getSyncScopeID());
4619}
4620
4621TruncInst *TruncInst::cloneImpl() const {
4622 return new TruncInst(getOperand(i_nocapture: 0), getType());
4623}
4624
4625ZExtInst *ZExtInst::cloneImpl() const {
4626 return new ZExtInst(getOperand(i_nocapture: 0), getType());
4627}
4628
4629SExtInst *SExtInst::cloneImpl() const {
4630 return new SExtInst(getOperand(i_nocapture: 0), getType());
4631}
4632
4633FPTruncInst *FPTruncInst::cloneImpl() const {
4634 auto *I = new FPTruncInst(getOperand(i_nocapture: 0), getType());
4635 I->FMF = FMF;
4636 return I;
4637}
4638
4639FPExtInst *FPExtInst::cloneImpl() const {
4640 auto *I = new FPExtInst(getOperand(i_nocapture: 0), getType());
4641 I->FMF = FMF;
4642 return I;
4643}
4644
4645UIToFPInst *UIToFPInst::cloneImpl() const {
4646 auto *Result = new UIToFPInst(getOperand(i_nocapture: 0), getType());
4647 Result->FMF = FMF;
4648 return Result;
4649}
4650
4651SIToFPInst *SIToFPInst::cloneImpl() const {
4652 auto *Result = new SIToFPInst(getOperand(i_nocapture: 0), getType());
4653 Result->FMF = FMF;
4654 return Result;
4655}
4656
4657FPToUIInst *FPToUIInst::cloneImpl() const {
4658 return new FPToUIInst(getOperand(i_nocapture: 0), getType());
4659}
4660
4661FPToSIInst *FPToSIInst::cloneImpl() const {
4662 return new FPToSIInst(getOperand(i_nocapture: 0), getType());
4663}
4664
4665PtrToIntInst *PtrToIntInst::cloneImpl() const {
4666 return new PtrToIntInst(getOperand(i_nocapture: 0), getType());
4667}
4668
4669PtrToAddrInst *PtrToAddrInst::cloneImpl() const {
4670 return new PtrToAddrInst(getOperand(i_nocapture: 0), getType());
4671}
4672
4673IntToPtrInst *IntToPtrInst::cloneImpl() const {
4674 return new IntToPtrInst(getOperand(i_nocapture: 0), getType());
4675}
4676
4677BitCastInst *BitCastInst::cloneImpl() const {
4678 return new BitCastInst(getOperand(i_nocapture: 0), getType());
4679}
4680
4681AddrSpaceCastInst *AddrSpaceCastInst::cloneImpl() const {
4682 return new AddrSpaceCastInst(getOperand(i_nocapture: 0), getType());
4683}
4684
4685CallInst *CallInst::cloneImpl() const {
4686 if (hasOperandBundles()) {
4687 IntrusiveOperandsAndDescriptorAllocMarker AllocMarker{
4688 .NumOps: getNumOperands(),
4689 .DescBytes: getNumOperandBundles() * unsigned(sizeof(BundleOpInfo))};
4690 return new (AllocMarker) CallInst(*this, AllocMarker);
4691 }
4692 IntrusiveOperandsAllocMarker AllocMarker{.NumOps: getNumOperands()};
4693 return new (AllocMarker) CallInst(*this, AllocMarker);
4694}
4695
4696SelectInst *SelectInst::cloneImpl() const {
4697 auto *I = SelectInst::Create(C: getOperand(i_nocapture: 0), S1: getOperand(i_nocapture: 1), S2: getOperand(i_nocapture: 2));
4698 I->FMF = FMF;
4699 return I;
4700}
4701
4702VAArgInst *VAArgInst::cloneImpl() const {
4703 return new VAArgInst(getOperand(i_nocapture: 0), getType());
4704}
4705
4706ExtractElementInst *ExtractElementInst::cloneImpl() const {
4707 return ExtractElementInst::Create(Vec: getOperand(i_nocapture: 0), Idx: getOperand(i_nocapture: 1));
4708}
4709
4710InsertElementInst *InsertElementInst::cloneImpl() const {
4711 return InsertElementInst::Create(Vec: getOperand(i_nocapture: 0), NewElt: getOperand(i_nocapture: 1), Idx: getOperand(i_nocapture: 2));
4712}
4713
4714BitInsertInst *BitInsertInst::cloneImpl() const {
4715 return BitInsertInst::Create(Base: getOperand(i_nocapture: 0), Val: getOperand(i_nocapture: 1), Offset: getOperand(i_nocapture: 2));
4716}
4717
4718BitExtractInst *BitExtractInst::cloneImpl() const {
4719 return BitExtractInst::Create(Ty: getType(), Src: getOperand(i_nocapture: 0), Offset: getOperand(i_nocapture: 1));
4720}
4721
4722ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {
4723 return new ShuffleVectorInst(getOperand(i_nocapture: 0), getOperand(i_nocapture: 1), getShuffleMask());
4724}
4725
4726PHINode *PHINode::cloneImpl() const { return new (AllocMarker) PHINode(*this); }
4727
4728LandingPadInst *LandingPadInst::cloneImpl() const {
4729 return new LandingPadInst(*this);
4730}
4731
4732ReturnInst *ReturnInst::cloneImpl() const {
4733 IntrusiveOperandsAllocMarker AllocMarker{.NumOps: getNumOperands()};
4734 return new (AllocMarker) ReturnInst(*this, AllocMarker);
4735}
4736
4737UncondBrInst *UncondBrInst::cloneImpl() const {
4738 return new (AllocMarker) UncondBrInst(*this);
4739}
4740
4741CondBrInst *CondBrInst::cloneImpl() const {
4742 return new (AllocMarker) CondBrInst(*this);
4743}
4744
4745SwitchInst *SwitchInst::cloneImpl() const { return new SwitchInst(*this); }
4746
4747IndirectBrInst *IndirectBrInst::cloneImpl() const {
4748 return new IndirectBrInst(*this);
4749}
4750
4751InvokeInst *InvokeInst::cloneImpl() const {
4752 if (hasOperandBundles()) {
4753 IntrusiveOperandsAndDescriptorAllocMarker AllocMarker{
4754 .NumOps: getNumOperands(),
4755 .DescBytes: getNumOperandBundles() * unsigned(sizeof(BundleOpInfo))};
4756 return new (AllocMarker) InvokeInst(*this, AllocMarker);
4757 }
4758 IntrusiveOperandsAllocMarker AllocMarker{.NumOps: getNumOperands()};
4759 return new (AllocMarker) InvokeInst(*this, AllocMarker);
4760}
4761
4762CallBrInst *CallBrInst::cloneImpl() const {
4763 if (hasOperandBundles()) {
4764 IntrusiveOperandsAndDescriptorAllocMarker AllocMarker{
4765 .NumOps: getNumOperands(),
4766 .DescBytes: getNumOperandBundles() * unsigned(sizeof(BundleOpInfo))};
4767 return new (AllocMarker) CallBrInst(*this, AllocMarker);
4768 }
4769 IntrusiveOperandsAllocMarker AllocMarker{.NumOps: getNumOperands()};
4770 return new (AllocMarker) CallBrInst(*this, AllocMarker);
4771}
4772
4773ResumeInst *ResumeInst::cloneImpl() const {
4774 return new (AllocMarker) ResumeInst(*this);
4775}
4776
4777CleanupReturnInst *CleanupReturnInst::cloneImpl() const {
4778 IntrusiveOperandsAllocMarker AllocMarker{.NumOps: getNumOperands()};
4779 return new (AllocMarker) CleanupReturnInst(*this, AllocMarker);
4780}
4781
4782CatchReturnInst *CatchReturnInst::cloneImpl() const {
4783 return new (AllocMarker) CatchReturnInst(*this);
4784}
4785
4786CatchSwitchInst *CatchSwitchInst::cloneImpl() const {
4787 return new CatchSwitchInst(*this);
4788}
4789
4790FuncletPadInst *FuncletPadInst::cloneImpl() const {
4791 IntrusiveOperandsAllocMarker AllocMarker{.NumOps: getNumOperands()};
4792 return new (AllocMarker) FuncletPadInst(*this, AllocMarker);
4793}
4794
4795UnreachableInst *UnreachableInst::cloneImpl() const {
4796 LLVMContext &Context = getContext();
4797 return new UnreachableInst(Context);
4798}
4799
4800bool UnreachableInst::shouldLowerToTrap(bool TrapUnreachable,
4801 bool NoTrapAfterNoreturn) const {
4802 if (!TrapUnreachable)
4803 return false;
4804
4805 // We may be able to ignore unreachable behind a noreturn call.
4806 if (const CallInst *Call = dyn_cast_or_null<CallInst>(Val: getPrevNode());
4807 Call && Call->doesNotReturn()) {
4808 if (NoTrapAfterNoreturn)
4809 return false;
4810 // Do not emit an additional trap instruction.
4811 if (Call->isNonContinuableTrap())
4812 return false;
4813 }
4814
4815 if (getFunction()->hasFnAttribute(Kind: Attribute::Naked))
4816 return false;
4817
4818 return true;
4819}
4820
4821FreezeInst *FreezeInst::cloneImpl() const {
4822 return new FreezeInst(getOperand(i_nocapture: 0));
4823}
4824