1//===-- IntrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements methods that make it really easy to deal with intrinsic
10// functions.
11//
12// All intrinsic function calls are instances of the call instruction, so these
13// are all subclasses of the CallInst class. Note that none of these classes
14// has state or virtual methods, which is an important part of this gross/neat
15// hack working.
16//
17// In some cases, arguments to intrinsics need to be generic and are defined as
18// type pointer to empty struct { }*. To access the real item of interest the
19// cast instruction needs to be stripped away.
20//
21//===----------------------------------------------------------------------===//
22
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/Metadata.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/Operator.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/IR/Statepoint.h"
32#include <optional>
33
34using namespace llvm;
35
36bool IntrinsicInst::mayLowerToFunctionCall(Intrinsic::ID IID) {
37 switch (IID) {
38 case Intrinsic::objc_autorelease:
39 case Intrinsic::objc_autoreleasePoolPop:
40 case Intrinsic::objc_autoreleasePoolPush:
41 case Intrinsic::objc_autoreleaseReturnValue:
42 case Intrinsic::objc_claimAutoreleasedReturnValue:
43 case Intrinsic::objc_copyWeak:
44 case Intrinsic::objc_destroyWeak:
45 case Intrinsic::objc_initWeak:
46 case Intrinsic::objc_loadWeak:
47 case Intrinsic::objc_loadWeakRetained:
48 case Intrinsic::objc_moveWeak:
49 case Intrinsic::objc_release:
50 case Intrinsic::objc_retain:
51 case Intrinsic::objc_retainAutorelease:
52 case Intrinsic::objc_retainAutoreleaseReturnValue:
53 case Intrinsic::objc_retainAutoreleasedReturnValue:
54 case Intrinsic::objc_retainBlock:
55 case Intrinsic::objc_storeStrong:
56 case Intrinsic::objc_storeWeak:
57 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
58 case Intrinsic::objc_retainedObject:
59 case Intrinsic::objc_unretainedObject:
60 case Intrinsic::objc_unretainedPointer:
61 case Intrinsic::objc_retain_autorelease:
62 case Intrinsic::objc_sync_enter:
63 case Intrinsic::objc_sync_exit:
64 return true;
65 default:
66 return false;
67 }
68}
69
70//===----------------------------------------------------------------------===//
71/// DbgVariableIntrinsic - This is the common base class for debug info
72/// intrinsics for variables.
73///
74
75iterator_range<location_op_iterator> RawLocationWrapper::location_ops() const {
76 Metadata *MD = getRawLocation();
77 assert(MD && "First operand of DbgVariableIntrinsic should be non-null.");
78 // If operand is ValueAsMetadata, return a range over just that operand.
79 if (auto *VAM = dyn_cast<ValueAsMetadata>(Val: MD)) {
80 return {location_op_iterator(VAM), location_op_iterator(VAM + 1)};
81 }
82 // If operand is DIArgList, return a range over its args.
83 if (auto *AL = dyn_cast<DIArgList>(Val: MD))
84 return {location_op_iterator(AL->args_begin()),
85 location_op_iterator(AL->args_end())};
86 // Operand must be an empty metadata tuple, so return empty iterator.
87 return {location_op_iterator(static_cast<ValueAsMetadata *>(nullptr)),
88 location_op_iterator(static_cast<ValueAsMetadata *>(nullptr))};
89}
90
91iterator_range<location_op_iterator>
92DbgVariableIntrinsic::location_ops() const {
93 return getWrappedLocation().location_ops();
94}
95
96Value *DbgVariableIntrinsic::getVariableLocationOp(unsigned OpIdx) const {
97 return getWrappedLocation().getVariableLocationOp(OpIdx);
98}
99
100Value *RawLocationWrapper::getVariableLocationOp(unsigned OpIdx) const {
101 Metadata *MD = getRawLocation();
102 assert(MD && "First operand of DbgVariableIntrinsic should be non-null.");
103 if (auto *AL = dyn_cast<DIArgList>(Val: MD))
104 return AL->getArgs()[OpIdx]->getValue();
105 if (isa<MDNode>(Val: MD))
106 return nullptr;
107 assert(
108 isa<ValueAsMetadata>(MD) &&
109 "Attempted to get location operand from DbgVariableIntrinsic with none.");
110 auto *V = cast<ValueAsMetadata>(Val: MD);
111 assert(OpIdx == 0 && "Operand Index must be 0 for a debug intrinsic with a "
112 "single location operand.");
113 return V->getValue();
114}
115
116static ValueAsMetadata *getAsMetadata(Value *V) {
117 return isa<MetadataAsValue>(Val: V) ? dyn_cast<ValueAsMetadata>(
118 Val: cast<MetadataAsValue>(Val: V)->getMetadata())
119 : ValueAsMetadata::get(V);
120}
121
122void DbgVariableIntrinsic::replaceVariableLocationOp(Value *OldValue,
123 Value *NewValue,
124 bool AllowEmpty) {
125 // If OldValue is used as the address part of a dbg.assign intrinsic replace
126 // it with NewValue and return true.
127 auto ReplaceDbgAssignAddress = [this, OldValue, NewValue]() -> bool {
128 auto *DAI = dyn_cast<DbgAssignIntrinsic>(Val: this);
129 if (!DAI || OldValue != DAI->getAddress())
130 return false;
131 DAI->setAddress(NewValue);
132 return true;
133 };
134 bool DbgAssignAddrReplaced = ReplaceDbgAssignAddress();
135 (void)DbgAssignAddrReplaced;
136
137 assert(NewValue && "Values must be non-null");
138 auto Locations = location_ops();
139 auto OldIt = find(Range&: Locations, Val: OldValue);
140 if (OldIt == Locations.end()) {
141 if (AllowEmpty || DbgAssignAddrReplaced)
142 return;
143 assert(DbgAssignAddrReplaced &&
144 "OldValue must be dbg.assign addr if unused in DIArgList");
145 return;
146 }
147
148 assert(OldIt != Locations.end() && "OldValue must be a current location");
149 if (!hasArgList()) {
150 Value *NewOperand = isa<MetadataAsValue>(Val: NewValue)
151 ? NewValue
152 : MetadataAsValue::get(
153 Context&: getContext(), MD: ValueAsMetadata::get(V: NewValue));
154 return setArgOperand(i: 0, v: NewOperand);
155 }
156 SmallVector<ValueAsMetadata *, 4> MDs;
157 ValueAsMetadata *NewOperand = getAsMetadata(V: NewValue);
158 for (auto *VMD : Locations)
159 MDs.push_back(Elt: VMD == *OldIt ? NewOperand : getAsMetadata(V: VMD));
160 setArgOperand(
161 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
162}
163void DbgVariableIntrinsic::replaceVariableLocationOp(unsigned OpIdx,
164 Value *NewValue) {
165 assert(OpIdx < getNumVariableLocationOps() && "Invalid Operand Index");
166 if (!hasArgList()) {
167 Value *NewOperand = isa<MetadataAsValue>(Val: NewValue)
168 ? NewValue
169 : MetadataAsValue::get(
170 Context&: getContext(), MD: ValueAsMetadata::get(V: NewValue));
171 return setArgOperand(i: 0, v: NewOperand);
172 }
173 SmallVector<ValueAsMetadata *, 4> MDs;
174 ValueAsMetadata *NewOperand = getAsMetadata(V: NewValue);
175 for (unsigned Idx = 0; Idx < getNumVariableLocationOps(); ++Idx)
176 MDs.push_back(Elt: Idx == OpIdx ? NewOperand
177 : getAsMetadata(V: getVariableLocationOp(OpIdx: Idx)));
178 setArgOperand(
179 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
180}
181
182void DbgVariableIntrinsic::addVariableLocationOps(ArrayRef<Value *> NewValues,
183 DIExpression *NewExpr) {
184 assert(NewExpr->hasAllLocationOps(getNumVariableLocationOps() +
185 NewValues.size()) &&
186 "NewExpr for debug variable intrinsic does not reference every "
187 "location operand.");
188 assert(!is_contained(NewValues, nullptr) && "New values must be non-null");
189 setArgOperand(i: 2, v: MetadataAsValue::get(Context&: getContext(), MD: NewExpr));
190 SmallVector<ValueAsMetadata *, 4> MDs;
191 for (auto *VMD : location_ops())
192 MDs.push_back(Elt: getAsMetadata(V: VMD));
193 for (auto *VMD : NewValues)
194 MDs.push_back(Elt: getAsMetadata(V: VMD));
195 setArgOperand(
196 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
197}
198
199std::optional<uint64_t> DbgVariableIntrinsic::getFragmentSizeInBits() const {
200 if (auto Fragment = getExpression()->getFragmentInfo())
201 return Fragment->SizeInBits;
202 return getVariable()->getSizeInBits();
203}
204
205Value *DbgAssignIntrinsic::getAddress() const {
206 auto *MD = getRawAddress();
207 if (auto *V = dyn_cast<ValueAsMetadata>(Val: MD))
208 return V->getValue();
209
210 // When the value goes to null, it gets replaced by an empty MDNode.
211 assert(!cast<MDNode>(MD)->getNumOperands() && "Expected an empty MDNode");
212 return nullptr;
213}
214
215void DbgAssignIntrinsic::setAssignId(DIAssignID *New) {
216 setOperand(i: OpAssignID, v: MetadataAsValue::get(Context&: getContext(), MD: New));
217}
218
219void DbgAssignIntrinsic::setAddress(Value *V) {
220 setOperand(i: OpAddress,
221 v: MetadataAsValue::get(Context&: getContext(), MD: ValueAsMetadata::get(V)));
222}
223
224void DbgAssignIntrinsic::setKillAddress() {
225 if (isKillAddress())
226 return;
227 setAddress(PoisonValue::get(T: getAddress()->getType()));
228}
229
230bool DbgAssignIntrinsic::isKillAddress() const {
231 Value *Addr = getAddress();
232 return !Addr || isa<UndefValue>(Val: Addr);
233}
234
235void DbgAssignIntrinsic::setValue(Value *V) {
236 setOperand(i: OpValue,
237 v: MetadataAsValue::get(Context&: getContext(), MD: ValueAsMetadata::get(V)));
238}
239
240ConstantInt *InstrProfCntrInstBase::getNumCounters() const {
241 if (InstrProfValueProfileInst::classof(I: this))
242 llvm_unreachable("InstrProfValueProfileInst does not have counters!");
243 return cast<ConstantInt>(Val: getArgOperand(i: 2));
244}
245
246ConstantInt *InstrProfCntrInstBase::getIndex() const {
247 if (InstrProfValueProfileInst::classof(I: this))
248 llvm_unreachable("Please use InstrProfValueProfileInst::getIndex()");
249 return cast<ConstantInt>(Val: getArgOperand(i: 3));
250}
251
252void InstrProfCntrInstBase::setIndex(uint32_t Idx) {
253 assert(isa<InstrProfCntrInstBase>(this));
254 setArgOperand(i: 3, v: ConstantInt::get(Ty: Type::getInt32Ty(C&: getContext()), V: Idx));
255}
256
257Value *InstrProfIncrementInst::getStep() const {
258 if (InstrProfIncrementInstStep::classof(I: this)) {
259 return getArgOperand(i: 4);
260 }
261 const Module *M = getModule();
262 LLVMContext &Context = M->getContext();
263 return ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 1);
264}
265
266Value *InstrProfCallsite::getCallee() const { return getArgOperand(i: 4); }
267
268void InstrProfCallsite::setCallee(Value *Callee) {
269 assert(isa<InstrProfCallsite>(this));
270 setArgOperand(i: 4, v: Callee);
271}
272
273std::optional<RoundingMode> ConstrainedFPIntrinsic::getRoundingMode() const {
274 unsigned NumOperands = arg_size();
275 Metadata *MD = nullptr;
276 auto *MAV = dyn_cast<MetadataAsValue>(Val: getArgOperand(i: NumOperands - 2));
277 if (MAV)
278 MD = MAV->getMetadata();
279 if (!MD || !isa<MDString>(Val: MD))
280 return std::nullopt;
281 return convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
282}
283
284std::optional<fp::ExceptionBehavior>
285ConstrainedFPIntrinsic::getExceptionBehavior() const {
286 unsigned NumOperands = arg_size();
287 Metadata *MD = nullptr;
288 auto *MAV = dyn_cast<MetadataAsValue>(Val: getArgOperand(i: NumOperands - 1));
289 if (MAV)
290 MD = MAV->getMetadata();
291 if (!MD || !isa<MDString>(Val: MD))
292 return std::nullopt;
293 return convertStrToExceptionBehavior(cast<MDString>(Val: MD)->getString());
294}
295
296bool ConstrainedFPIntrinsic::isDefaultFPEnvironment() const {
297 std::optional<fp::ExceptionBehavior> Except = getExceptionBehavior();
298 if (Except) {
299 if (*Except != fp::ebIgnore)
300 return false;
301 }
302
303 std::optional<RoundingMode> Rounding = getRoundingMode();
304 if (Rounding) {
305 if (*Rounding != RoundingMode::NearestTiesToEven)
306 return false;
307 }
308
309 return true;
310}
311
312static FCmpInst::Predicate getFPPredicateFromMD(const Value *Op) {
313 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
314 if (!MD || !isa<MDString>(Val: MD))
315 return FCmpInst::BAD_FCMP_PREDICATE;
316 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
317 .Case(S: "oeq", Value: FCmpInst::FCMP_OEQ)
318 .Case(S: "ogt", Value: FCmpInst::FCMP_OGT)
319 .Case(S: "oge", Value: FCmpInst::FCMP_OGE)
320 .Case(S: "olt", Value: FCmpInst::FCMP_OLT)
321 .Case(S: "ole", Value: FCmpInst::FCMP_OLE)
322 .Case(S: "one", Value: FCmpInst::FCMP_ONE)
323 .Case(S: "ord", Value: FCmpInst::FCMP_ORD)
324 .Case(S: "uno", Value: FCmpInst::FCMP_UNO)
325 .Case(S: "ueq", Value: FCmpInst::FCMP_UEQ)
326 .Case(S: "ugt", Value: FCmpInst::FCMP_UGT)
327 .Case(S: "uge", Value: FCmpInst::FCMP_UGE)
328 .Case(S: "ult", Value: FCmpInst::FCMP_ULT)
329 .Case(S: "ule", Value: FCmpInst::FCMP_ULE)
330 .Case(S: "une", Value: FCmpInst::FCMP_UNE)
331 .Default(Value: FCmpInst::BAD_FCMP_PREDICATE);
332}
333
334FCmpInst::Predicate ConstrainedFPCmpIntrinsic::getPredicate() const {
335 return getFPPredicateFromMD(Op: getArgOperand(i: 2));
336}
337
338unsigned ConstrainedFPIntrinsic::getNonMetadataArgCount() const {
339 // All constrained fp intrinsics have "fpexcept" metadata.
340 unsigned NumArgs = arg_size() - 1;
341
342 // Some intrinsics have "round" metadata.
343 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: getIntrinsicID()))
344 NumArgs -= 1;
345
346 // Compare intrinsics take their predicate as metadata.
347 if (isa<ConstrainedFPCmpIntrinsic>(Val: this))
348 NumArgs -= 1;
349
350 return NumArgs;
351}
352
353bool ConstrainedFPIntrinsic::classof(const IntrinsicInst *I) {
354 return Intrinsic::isConstrainedFPIntrinsic(QID: I->getIntrinsicID());
355}
356
357ElementCount VPIntrinsic::getStaticVectorLength() const {
358 auto GetVectorLengthOfType = [](const Type *T) -> ElementCount {
359 const auto *VT = cast<VectorType>(Val: T);
360 auto ElemCount = VT->getElementCount();
361 return ElemCount;
362 };
363
364 Value *VPMask = getMaskParam();
365 if (!VPMask) {
366 assert((getIntrinsicID() == Intrinsic::vp_merge) &&
367 "Unexpected VP intrinsic without mask operand");
368 return GetVectorLengthOfType(getType());
369 }
370 return GetVectorLengthOfType(VPMask->getType());
371}
372
373Value *VPIntrinsic::getMaskParam() const {
374 if (auto MaskPos = getMaskParamPos(IntrinsicID: getIntrinsicID()))
375 return getArgOperand(i: *MaskPos);
376 return nullptr;
377}
378
379void VPIntrinsic::setMaskParam(Value *NewMask) {
380 auto MaskPos = getMaskParamPos(IntrinsicID: getIntrinsicID());
381 setArgOperand(i: *MaskPos, v: NewMask);
382}
383
384Value *VPIntrinsic::getVectorLengthParam() const {
385 if (auto EVLPos = getVectorLengthParamPos(IntrinsicID: getIntrinsicID()))
386 return getArgOperand(i: *EVLPos);
387 return nullptr;
388}
389
390void VPIntrinsic::setVectorLengthParam(Value *NewEVL) {
391 auto EVLPos = getVectorLengthParamPos(IntrinsicID: getIntrinsicID());
392 setArgOperand(i: *EVLPos, v: NewEVL);
393}
394
395std::optional<unsigned>
396VPIntrinsic::getMaskParamPos(Intrinsic::ID IntrinsicID) {
397 switch (IntrinsicID) {
398 default:
399 return std::nullopt;
400
401#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
402 case Intrinsic::VPID: \
403 return MASKPOS;
404#include "llvm/IR/VPIntrinsics.def"
405 }
406}
407
408std::optional<unsigned>
409VPIntrinsic::getVectorLengthParamPos(Intrinsic::ID IntrinsicID) {
410 switch (IntrinsicID) {
411 default:
412 return std::nullopt;
413
414#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
415 case Intrinsic::VPID: \
416 return VLENPOS;
417#include "llvm/IR/VPIntrinsics.def"
418 }
419}
420
421/// \return the alignment of the pointer used by this load/store/gather or
422/// scatter.
423MaybeAlign VPIntrinsic::getPointerAlignment() const {
424 std::optional<unsigned> PtrParamOpt =
425 getMemoryPointerParamPos(getIntrinsicID());
426 assert(PtrParamOpt && "no pointer argument!");
427 return getParamAlign(ArgNo: *PtrParamOpt);
428}
429
430/// \return The pointer operand of this load,store, gather or scatter.
431Value *VPIntrinsic::getMemoryPointerParam() const {
432 if (auto PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID()))
433 return getArgOperand(i: *PtrParamOpt);
434 return nullptr;
435}
436
437std::optional<unsigned>
438VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) {
439 switch (VPID) {
440 default:
441 return std::nullopt;
442 case Intrinsic::vp_store:
443 case Intrinsic::vp_scatter:
444 case Intrinsic::experimental_vp_strided_store:
445 return 1;
446 case Intrinsic::vp_load:
447 case Intrinsic::vp_load_ff:
448 case Intrinsic::vp_gather:
449 case Intrinsic::experimental_vp_strided_load:
450 return 0;
451 }
452}
453
454/// \return The data (payload) operand of this store or scatter.
455Value *VPIntrinsic::getMemoryDataParam() const {
456 auto DataParamOpt = getMemoryDataParamPos(getIntrinsicID());
457 if (!DataParamOpt)
458 return nullptr;
459 return getArgOperand(i: *DataParamOpt);
460}
461
462std::optional<unsigned> VPIntrinsic::getMemoryDataParamPos(Intrinsic::ID VPID) {
463 switch (VPID) {
464 default:
465 return std::nullopt;
466 case Intrinsic::vp_store:
467 case Intrinsic::vp_scatter:
468 case Intrinsic::experimental_vp_strided_store:
469 return 0;
470 }
471}
472
473constexpr bool isVPIntrinsic(Intrinsic::ID ID) {
474 switch (ID) {
475 default:
476 break;
477#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
478 case Intrinsic::VPID: \
479 return true;
480#include "llvm/IR/VPIntrinsics.def"
481 }
482 return false;
483}
484
485bool VPIntrinsic::isVPIntrinsic(Intrinsic::ID ID) {
486 return ::isVPIntrinsic(ID);
487}
488
489// Equivalent non-predicated opcode
490constexpr static std::optional<unsigned>
491getFunctionalOpcodeForVP(Intrinsic::ID ID) {
492 switch (ID) {
493 default:
494 break;
495#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
496#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) return Instruction::OPC;
497#define END_REGISTER_VP_INTRINSIC(VPID) break;
498#include "llvm/IR/VPIntrinsics.def"
499 }
500 return std::nullopt;
501}
502
503std::optional<unsigned>
504VPIntrinsic::getFunctionalOpcodeForVP(Intrinsic::ID ID) {
505 return ::getFunctionalOpcodeForVP(ID);
506}
507
508// Equivalent non-predicated intrinsic ID
509constexpr static std::optional<Intrinsic::ID>
510getFunctionalIntrinsicIDForVP(Intrinsic::ID ID) {
511 switch (ID) {
512 default:
513 break;
514#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
515#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) return Intrinsic::INTRIN;
516#define END_REGISTER_VP_INTRINSIC(VPID) break;
517#include "llvm/IR/VPIntrinsics.def"
518 }
519 return std::nullopt;
520}
521
522std::optional<Intrinsic::ID>
523VPIntrinsic::getFunctionalIntrinsicIDForVP(Intrinsic::ID ID) {
524 return ::getFunctionalIntrinsicIDForVP(ID);
525}
526
527constexpr static bool doesVPHaveNoFunctionalEquivalent(Intrinsic::ID ID) {
528 switch (ID) {
529 default:
530 break;
531#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
532#define VP_PROPERTY_NO_FUNCTIONAL return true;
533#define END_REGISTER_VP_INTRINSIC(VPID) break;
534#include "llvm/IR/VPIntrinsics.def"
535 }
536 return false;
537}
538
539// All VP intrinsics should have an equivalent non-VP opcode or intrinsic
540// defined, or be marked that they don't have one.
541#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) \
542 static_assert(doesVPHaveNoFunctionalEquivalent(Intrinsic::VPID) || \
543 getFunctionalOpcodeForVP(Intrinsic::VPID) || \
544 getFunctionalIntrinsicIDForVP(Intrinsic::VPID));
545#include "llvm/IR/VPIntrinsics.def"
546
547bool VPIntrinsic::canIgnoreVectorLengthParam() const {
548 using namespace PatternMatch;
549
550 ElementCount EC = getStaticVectorLength();
551
552 // No vlen param - no lanes masked-off by it.
553 auto *VLParam = getVectorLengthParam();
554 if (!VLParam)
555 return true;
556
557 // Note that the VP intrinsic causes undefined behavior if the Explicit Vector
558 // Length parameter is strictly greater-than the number of vector elements of
559 // the operation. This function returns true when this is detected statically
560 // in the IR.
561
562 // Check whether "W == vscale * EC.getKnownMinValue()"
563 if (EC.isScalable()) {
564 // Compare vscale patterns
565 uint64_t VScaleFactor;
566 if (match(V: VLParam, P: m_Mul(L: m_VScale(), R: m_ConstantInt(V&: VScaleFactor))))
567 return VScaleFactor >= EC.getKnownMinValue();
568 return (EC.getKnownMinValue() == 1) && match(V: VLParam, P: m_VScale());
569 }
570
571 // standard SIMD operation
572 const auto *VLConst = dyn_cast<ConstantInt>(Val: VLParam);
573 if (!VLConst)
574 return false;
575
576 uint64_t VLNum = VLConst->getZExtValue();
577 if (VLNum >= EC.getKnownMinValue())
578 return true;
579
580 return false;
581}
582
583Function *VPIntrinsic::getOrInsertDeclarationForParams(
584 Module *M, Intrinsic::ID VPID, Type *ReturnType, ArrayRef<Value *> Params) {
585 assert(isVPIntrinsic(VPID) && "not a VP intrinsic");
586 Function *VPFunc;
587 switch (VPID) {
588 default: {
589 Type *OverloadTy = Params[0]->getType();
590 if (VPReductionIntrinsic::isVPReduction(ID: VPID))
591 OverloadTy =
592 Params[*VPReductionIntrinsic::getVectorParamPos(ID: VPID)]->getType();
593
594 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, OverloadTys: OverloadTy);
595 break;
596 }
597 case Intrinsic::vp_cttz_elts:
598 VPFunc = Intrinsic::getOrInsertDeclaration(
599 M, id: VPID, OverloadTys: {ReturnType, Params[0]->getType()});
600 break;
601 case Intrinsic::vp_merge:
602 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, OverloadTys: {Params[1]->getType()});
603 break;
604 case Intrinsic::vp_load:
605 VPFunc = Intrinsic::getOrInsertDeclaration(
606 M, id: VPID, OverloadTys: {ReturnType, Params[0]->getType()});
607 break;
608 case Intrinsic::vp_load_ff:
609 VPFunc = Intrinsic::getOrInsertDeclaration(
610 M, id: VPID, OverloadTys: {ReturnType->getStructElementType(N: 0), Params[0]->getType()});
611 break;
612 case Intrinsic::experimental_vp_strided_load:
613 VPFunc = Intrinsic::getOrInsertDeclaration(
614 M, id: VPID, OverloadTys: {ReturnType, Params[0]->getType(), Params[1]->getType()});
615 break;
616 case Intrinsic::vp_gather:
617 VPFunc = Intrinsic::getOrInsertDeclaration(
618 M, id: VPID, OverloadTys: {ReturnType, Params[0]->getType()});
619 break;
620 case Intrinsic::vp_store:
621 VPFunc = Intrinsic::getOrInsertDeclaration(
622 M, id: VPID, OverloadTys: {Params[0]->getType(), Params[1]->getType()});
623 break;
624 case Intrinsic::experimental_vp_strided_store:
625 VPFunc = Intrinsic::getOrInsertDeclaration(
626 M, id: VPID,
627 OverloadTys: {Params[0]->getType(), Params[1]->getType(), Params[2]->getType()});
628 break;
629 case Intrinsic::vp_scatter:
630 VPFunc = Intrinsic::getOrInsertDeclaration(
631 M, id: VPID, OverloadTys: {Params[0]->getType(), Params[1]->getType()});
632 break;
633 }
634 assert(VPFunc && "Could not declare VP intrinsic");
635 return VPFunc;
636}
637
638bool VPReductionIntrinsic::isVPReduction(Intrinsic::ID ID) {
639 switch (ID) {
640 case Intrinsic::vp_reduce_add:
641 case Intrinsic::vp_reduce_mul:
642 case Intrinsic::vp_reduce_and:
643 case Intrinsic::vp_reduce_or:
644 case Intrinsic::vp_reduce_xor:
645 case Intrinsic::vp_reduce_smax:
646 case Intrinsic::vp_reduce_smin:
647 case Intrinsic::vp_reduce_umax:
648 case Intrinsic::vp_reduce_umin:
649 case Intrinsic::vp_reduce_fmax:
650 case Intrinsic::vp_reduce_fmin:
651 case Intrinsic::vp_reduce_fmaximum:
652 case Intrinsic::vp_reduce_fminimum:
653 case Intrinsic::vp_reduce_fadd:
654 case Intrinsic::vp_reduce_fmul:
655 return true;
656 default:
657 return false;
658 }
659}
660
661unsigned VPReductionIntrinsic::getVectorParamPos() const {
662 return *VPReductionIntrinsic::getVectorParamPos(ID: getIntrinsicID());
663}
664
665unsigned VPReductionIntrinsic::getStartParamPos() const {
666 return *VPReductionIntrinsic::getStartParamPos(ID: getIntrinsicID());
667}
668
669std::optional<unsigned>
670VPReductionIntrinsic::getVectorParamPos(Intrinsic::ID ID) {
671 if (isVPReduction(ID))
672 return 1;
673 return std::nullopt;
674}
675
676std::optional<unsigned>
677VPReductionIntrinsic::getStartParamPos(Intrinsic::ID ID) {
678 if (isVPReduction(ID))
679 return 0;
680 return std::nullopt;
681}
682
683Instruction::BinaryOps BinaryOpIntrinsic::getBinaryOp() const {
684 switch (getIntrinsicID()) {
685 case Intrinsic::uadd_with_overflow:
686 case Intrinsic::sadd_with_overflow:
687 case Intrinsic::uadd_sat:
688 case Intrinsic::sadd_sat:
689 return Instruction::Add;
690 case Intrinsic::usub_with_overflow:
691 case Intrinsic::ssub_with_overflow:
692 case Intrinsic::usub_sat:
693 case Intrinsic::ssub_sat:
694 return Instruction::Sub;
695 case Intrinsic::umul_with_overflow:
696 case Intrinsic::smul_with_overflow:
697 return Instruction::Mul;
698 default:
699 llvm_unreachable("Invalid intrinsic");
700 }
701}
702
703bool BinaryOpIntrinsic::isSigned() const {
704 switch (getIntrinsicID()) {
705 case Intrinsic::sadd_with_overflow:
706 case Intrinsic::ssub_with_overflow:
707 case Intrinsic::smul_with_overflow:
708 case Intrinsic::sadd_sat:
709 case Intrinsic::ssub_sat:
710 return true;
711 default:
712 return false;
713 }
714}
715
716unsigned BinaryOpIntrinsic::getNoWrapKind() const {
717 if (isSigned())
718 return OverflowingBinaryOperator::NoSignedWrap;
719 else
720 return OverflowingBinaryOperator::NoUnsignedWrap;
721}
722
723const Value *GCProjectionInst::getStatepoint() const {
724 const Value *Token = getArgOperand(i: 0);
725 if (isa<UndefValue>(Val: Token))
726 return Token;
727
728 // Treat none token as if it was undef here
729 if (isa<ConstantTokenNone>(Val: Token))
730 return UndefValue::get(T: Token->getType());
731
732 // This takes care both of relocates for call statepoints and relocates
733 // on normal path of invoke statepoint.
734 if (!isa<LandingPadInst>(Val: Token))
735 return cast<GCStatepointInst>(Val: Token);
736
737 // This relocate is on exceptional path of an invoke statepoint
738 const BasicBlock *InvokeBB =
739 cast<Instruction>(Val: Token)->getParent()->getUniquePredecessor();
740
741 assert(InvokeBB && "safepoints should have unique landingpads");
742 assert(InvokeBB->getTerminator() &&
743 "safepoint block should be well formed");
744
745 return cast<GCStatepointInst>(Val: InvokeBB->getTerminator());
746}
747
748Value *GCRelocateInst::getBasePtr() const {
749 auto Statepoint = getStatepoint();
750 if (isa<UndefValue>(Val: Statepoint))
751 return UndefValue::get(T: Statepoint->getType());
752 // Handle too few (bundle) arguments to avoid crashes when printing invalid
753 // IR, e.g. in the verifier.
754 auto *GCInst = cast<GCStatepointInst>(Val: Statepoint);
755 if (auto Opt = GCInst->getOperandBundle(ID: LLVMContext::OB_gc_live)) {
756 if (getBasePtrIndex() > Opt->Inputs.size())
757 return nullptr;
758 return *(Opt->Inputs.begin() + getBasePtrIndex());
759 }
760 if (getBasePtrIndex() > GCInst->arg_size())
761 return nullptr;
762 return *(GCInst->arg_begin() + getBasePtrIndex());
763}
764
765Value *GCRelocateInst::getDerivedPtr() const {
766 auto *Statepoint = getStatepoint();
767 if (isa<UndefValue>(Val: Statepoint))
768 return UndefValue::get(T: Statepoint->getType());
769
770 // Handle too few (bundle) arguments to avoid crashes when printing invalid
771 // IR, e.g. in the verifier.
772 auto *GCInst = cast<GCStatepointInst>(Val: Statepoint);
773 if (auto Opt = GCInst->getOperandBundle(ID: LLVMContext::OB_gc_live)) {
774 if (getDerivedPtrIndex() > Opt->Inputs.size())
775 return nullptr;
776 return *(Opt->Inputs.begin() + getDerivedPtrIndex());
777 }
778 if (getDerivedPtrIndex() > GCInst->arg_size())
779 return nullptr;
780 return *(GCInst->arg_begin() + getDerivedPtrIndex());
781}
782
783ConvergenceControlInst *ConvergenceControlInst::CreateAnchor(BasicBlock &BB) {
784 Module *M = BB.getModule();
785 Function *Fn = Intrinsic::getOrInsertDeclaration(
786 M, id: llvm::Intrinsic::experimental_convergence_anchor);
787 auto *Call = CallInst::Create(Func: Fn, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
788 return cast<ConvergenceControlInst>(Val: Call);
789}
790
791ConvergenceControlInst *ConvergenceControlInst::CreateEntry(BasicBlock &BB) {
792 Module *M = BB.getModule();
793 Function *Fn = Intrinsic::getOrInsertDeclaration(
794 M, id: llvm::Intrinsic::experimental_convergence_entry);
795 auto *Call = CallInst::Create(Func: Fn, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
796 return cast<ConvergenceControlInst>(Val: Call);
797}
798
799ConvergenceControlInst *
800ConvergenceControlInst::CreateLoop(BasicBlock &BB,
801 ConvergenceControlInst *ParentToken) {
802 Module *M = BB.getModule();
803 Function *Fn = Intrinsic::getOrInsertDeclaration(
804 M, id: llvm::Intrinsic::experimental_convergence_loop);
805 llvm::Value *BundleArgs[] = {ParentToken};
806 llvm::OperandBundleDef OB("convergencectrl", BundleArgs);
807 auto *Call = CallInst::Create(Func: Fn, Args: {}, Bundles: {OB}, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
808 return cast<ConvergenceControlInst>(Val: Call);
809}
810