1//===-- IntrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements methods that make it really easy to deal with intrinsic
10// functions.
11//
12// All intrinsic function calls are instances of the call instruction, so these
13// are all subclasses of the CallInst class. Note that none of these classes
14// has state or virtual methods, which is an important part of this gross/neat
15// hack working.
16//
17// In some cases, arguments to intrinsics need to be generic and are defined as
18// type pointer to empty struct { }*. To access the real item of interest the
19// cast instruction needs to be stripped away.
20//
21//===----------------------------------------------------------------------===//
22
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/Metadata.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/Operator.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/IR/Statepoint.h"
32#include <optional>
33
34using namespace llvm;
35
36bool IntrinsicInst::mayLowerToFunctionCall(Intrinsic::ID IID) {
37 switch (IID) {
38 case Intrinsic::objc_autorelease:
39 case Intrinsic::objc_autoreleasePoolPop:
40 case Intrinsic::objc_autoreleasePoolPush:
41 case Intrinsic::objc_autoreleaseReturnValue:
42 case Intrinsic::objc_claimAutoreleasedReturnValue:
43 case Intrinsic::objc_copyWeak:
44 case Intrinsic::objc_destroyWeak:
45 case Intrinsic::objc_initWeak:
46 case Intrinsic::objc_loadWeak:
47 case Intrinsic::objc_loadWeakRetained:
48 case Intrinsic::objc_moveWeak:
49 case Intrinsic::objc_release:
50 case Intrinsic::objc_retain:
51 case Intrinsic::objc_retainAutorelease:
52 case Intrinsic::objc_retainAutoreleaseReturnValue:
53 case Intrinsic::objc_retainAutoreleasedReturnValue:
54 case Intrinsic::objc_retainBlock:
55 case Intrinsic::objc_storeStrong:
56 case Intrinsic::objc_storeWeak:
57 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
58 case Intrinsic::objc_retainedObject:
59 case Intrinsic::objc_unretainedObject:
60 case Intrinsic::objc_unretainedPointer:
61 case Intrinsic::objc_retain_autorelease:
62 case Intrinsic::objc_sync_enter:
63 case Intrinsic::objc_sync_exit:
64 return true;
65 default:
66 return false;
67 }
68}
69
70//===----------------------------------------------------------------------===//
71/// DbgVariableIntrinsic - This is the common base class for debug info
72/// intrinsics for variables.
73///
74
75iterator_range<location_op_iterator> RawLocationWrapper::location_ops() const {
76 Metadata *MD = getRawLocation();
77 assert(MD && "First operand of DbgVariableIntrinsic should be non-null.");
78 // If operand is ValueAsMetadata, return a range over just that operand.
79 if (auto *VAM = dyn_cast<ValueAsMetadata>(Val: MD)) {
80 return {location_op_iterator(VAM), location_op_iterator(VAM + 1)};
81 }
82 // If operand is DIArgList, return a range over its args.
83 if (auto *AL = dyn_cast<DIArgList>(Val: MD))
84 return {location_op_iterator(AL->args_begin()),
85 location_op_iterator(AL->args_end())};
86 // Operand must be an empty metadata tuple, so return empty iterator.
87 return {location_op_iterator(static_cast<ValueAsMetadata *>(nullptr)),
88 location_op_iterator(static_cast<ValueAsMetadata *>(nullptr))};
89}
90
91iterator_range<location_op_iterator>
92DbgVariableIntrinsic::location_ops() const {
93 return getWrappedLocation().location_ops();
94}
95
96Value *DbgVariableIntrinsic::getVariableLocationOp(unsigned OpIdx) const {
97 return getWrappedLocation().getVariableLocationOp(OpIdx);
98}
99
100Value *RawLocationWrapper::getVariableLocationOp(unsigned OpIdx) const {
101 Metadata *MD = getRawLocation();
102 assert(MD && "First operand of DbgVariableIntrinsic should be non-null.");
103 if (auto *AL = dyn_cast<DIArgList>(Val: MD))
104 return AL->getArgs()[OpIdx]->getValue();
105 if (isa<MDNode>(Val: MD))
106 return nullptr;
107 assert(
108 isa<ValueAsMetadata>(MD) &&
109 "Attempted to get location operand from DbgVariableIntrinsic with none.");
110 auto *V = cast<ValueAsMetadata>(Val: MD);
111 assert(OpIdx == 0 && "Operand Index must be 0 for a debug intrinsic with a "
112 "single location operand.");
113 return V->getValue();
114}
115
116static ValueAsMetadata *getAsMetadata(Value *V) {
117 return isa<MetadataAsValue>(Val: V) ? dyn_cast<ValueAsMetadata>(
118 Val: cast<MetadataAsValue>(Val: V)->getMetadata())
119 : ValueAsMetadata::get(V);
120}
121
122void DbgVariableIntrinsic::replaceVariableLocationOp(Value *OldValue,
123 Value *NewValue,
124 bool AllowEmpty) {
125 // If OldValue is used as the address part of a dbg.assign intrinsic replace
126 // it with NewValue and return true.
127 auto ReplaceDbgAssignAddress = [this, OldValue, NewValue]() -> bool {
128 auto *DAI = dyn_cast<DbgAssignIntrinsic>(Val: this);
129 if (!DAI || OldValue != DAI->getAddress())
130 return false;
131 DAI->setAddress(NewValue);
132 return true;
133 };
134 bool DbgAssignAddrReplaced = ReplaceDbgAssignAddress();
135 (void)DbgAssignAddrReplaced;
136
137 assert(NewValue && "Values must be non-null");
138 auto Locations = location_ops();
139 auto OldIt = find(Range&: Locations, Val: OldValue);
140 if (OldIt == Locations.end()) {
141 if (AllowEmpty || DbgAssignAddrReplaced)
142 return;
143 assert(DbgAssignAddrReplaced &&
144 "OldValue must be dbg.assign addr if unused in DIArgList");
145 return;
146 }
147
148 assert(OldIt != Locations.end() && "OldValue must be a current location");
149 if (!hasArgList()) {
150 Value *NewOperand = isa<MetadataAsValue>(Val: NewValue)
151 ? NewValue
152 : MetadataAsValue::get(
153 Context&: getContext(), MD: ValueAsMetadata::get(V: NewValue));
154 return setArgOperand(i: 0, v: NewOperand);
155 }
156 SmallVector<ValueAsMetadata *, 4> MDs;
157 ValueAsMetadata *NewOperand = getAsMetadata(V: NewValue);
158 for (auto *VMD : Locations)
159 MDs.push_back(Elt: VMD == *OldIt ? NewOperand : getAsMetadata(V: VMD));
160 setArgOperand(
161 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
162}
163void DbgVariableIntrinsic::replaceVariableLocationOp(unsigned OpIdx,
164 Value *NewValue) {
165 assert(OpIdx < getNumVariableLocationOps() && "Invalid Operand Index");
166 if (!hasArgList()) {
167 Value *NewOperand = isa<MetadataAsValue>(Val: NewValue)
168 ? NewValue
169 : MetadataAsValue::get(
170 Context&: getContext(), MD: ValueAsMetadata::get(V: NewValue));
171 return setArgOperand(i: 0, v: NewOperand);
172 }
173 SmallVector<ValueAsMetadata *, 4> MDs;
174 ValueAsMetadata *NewOperand = getAsMetadata(V: NewValue);
175 for (unsigned Idx = 0; Idx < getNumVariableLocationOps(); ++Idx)
176 MDs.push_back(Elt: Idx == OpIdx ? NewOperand
177 : getAsMetadata(V: getVariableLocationOp(OpIdx: Idx)));
178 setArgOperand(
179 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
180}
181
182void DbgVariableIntrinsic::addVariableLocationOps(ArrayRef<Value *> NewValues,
183 DIExpression *NewExpr) {
184 assert(NewExpr->hasAllLocationOps(getNumVariableLocationOps() +
185 NewValues.size()) &&
186 "NewExpr for debug variable intrinsic does not reference every "
187 "location operand.");
188 assert(!is_contained(NewValues, nullptr) && "New values must be non-null");
189 setArgOperand(i: 2, v: MetadataAsValue::get(Context&: getContext(), MD: NewExpr));
190 SmallVector<ValueAsMetadata *, 4> MDs;
191 for (auto *VMD : location_ops())
192 MDs.push_back(Elt: getAsMetadata(V: VMD));
193 for (auto *VMD : NewValues)
194 MDs.push_back(Elt: getAsMetadata(V: VMD));
195 setArgOperand(
196 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
197}
198
199std::optional<uint64_t> DbgVariableIntrinsic::getFragmentSizeInBits() const {
200 if (auto Fragment = getExpression()->getFragmentInfo())
201 return Fragment->SizeInBits;
202 return getVariable()->getSizeInBits();
203}
204
205Value *DbgAssignIntrinsic::getAddress() const {
206 auto *MD = getRawAddress();
207 if (auto *V = dyn_cast<ValueAsMetadata>(Val: MD))
208 return V->getValue();
209
210 // When the value goes to null, it gets replaced by an empty MDNode.
211 assert(!cast<MDNode>(MD)->getNumOperands() && "Expected an empty MDNode");
212 return nullptr;
213}
214
215void DbgAssignIntrinsic::setAssignId(DIAssignID *New) {
216 setOperand(i: OpAssignID, v: MetadataAsValue::get(Context&: getContext(), MD: New));
217}
218
219void DbgAssignIntrinsic::setAddress(Value *V) {
220 setOperand(i: OpAddress,
221 v: MetadataAsValue::get(Context&: getContext(), MD: ValueAsMetadata::get(V)));
222}
223
224void DbgAssignIntrinsic::setKillAddress() {
225 if (isKillAddress())
226 return;
227 setAddress(PoisonValue::get(T: getAddress()->getType()));
228}
229
230bool DbgAssignIntrinsic::isKillAddress() const {
231 Value *Addr = getAddress();
232 return !Addr || isa<UndefValue>(Val: Addr);
233}
234
235void DbgAssignIntrinsic::setValue(Value *V) {
236 setOperand(i: OpValue,
237 v: MetadataAsValue::get(Context&: getContext(), MD: ValueAsMetadata::get(V)));
238}
239
240ConstantInt *InstrProfCntrInstBase::getNumCounters() const {
241 if (InstrProfValueProfileInst::classof(I: this))
242 llvm_unreachable("InstrProfValueProfileInst does not have counters!");
243 return cast<ConstantInt>(Val: getArgOperand(i: 2));
244}
245
246ConstantInt *InstrProfCntrInstBase::getIndex() const {
247 if (InstrProfValueProfileInst::classof(I: this))
248 llvm_unreachable("Please use InstrProfValueProfileInst::getIndex()");
249 return cast<ConstantInt>(Val: getArgOperand(i: 3));
250}
251
252void InstrProfCntrInstBase::setIndex(uint32_t Idx) {
253 assert(isa<InstrProfCntrInstBase>(this));
254 setArgOperand(i: 3, v: ConstantInt::get(Ty: Type::getInt32Ty(C&: getContext()), V: Idx));
255}
256
257Value *InstrProfIncrementInst::getStep() const {
258 if (InstrProfIncrementInstStep::classof(I: this)) {
259 return getArgOperand(i: 4);
260 }
261 const Module *M = getModule();
262 LLVMContext &Context = M->getContext();
263 return ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 1);
264}
265
266Value *InstrProfCallsite::getCallee() const { return getArgOperand(i: 4); }
267
268void InstrProfCallsite::setCallee(Value *Callee) {
269 assert(isa<InstrProfCallsite>(this));
270 setArgOperand(i: 4, v: Callee);
271}
272
273std::optional<RoundingMode> ConstrainedFPIntrinsic::getRoundingMode() const {
274 unsigned NumOperands = arg_size();
275 Metadata *MD = nullptr;
276 auto *MAV = dyn_cast<MetadataAsValue>(Val: getArgOperand(i: NumOperands - 2));
277 if (MAV)
278 MD = MAV->getMetadata();
279 if (!MD || !isa<MDString>(Val: MD))
280 return std::nullopt;
281 return convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
282}
283
284std::optional<fp::ExceptionBehavior>
285ConstrainedFPIntrinsic::getExceptionBehavior() const {
286 unsigned NumOperands = arg_size();
287 Metadata *MD = nullptr;
288 auto *MAV = dyn_cast<MetadataAsValue>(Val: getArgOperand(i: NumOperands - 1));
289 if (MAV)
290 MD = MAV->getMetadata();
291 if (!MD || !isa<MDString>(Val: MD))
292 return std::nullopt;
293 return convertStrToExceptionBehavior(cast<MDString>(Val: MD)->getString());
294}
295
296bool ConstrainedFPIntrinsic::isDefaultFPEnvironment() const {
297 std::optional<fp::ExceptionBehavior> Except = getExceptionBehavior();
298 if (Except) {
299 if (*Except != fp::ebIgnore)
300 return false;
301 }
302
303 std::optional<RoundingMode> Rounding = getRoundingMode();
304 if (Rounding) {
305 if (*Rounding != RoundingMode::NearestTiesToEven)
306 return false;
307 }
308
309 return true;
310}
311
312static FCmpInst::Predicate getFPPredicateFromMD(const Value *Op) {
313 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
314 if (!MD || !isa<MDString>(Val: MD))
315 return FCmpInst::BAD_FCMP_PREDICATE;
316 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
317 .Case(S: "oeq", Value: FCmpInst::FCMP_OEQ)
318 .Case(S: "ogt", Value: FCmpInst::FCMP_OGT)
319 .Case(S: "oge", Value: FCmpInst::FCMP_OGE)
320 .Case(S: "olt", Value: FCmpInst::FCMP_OLT)
321 .Case(S: "ole", Value: FCmpInst::FCMP_OLE)
322 .Case(S: "one", Value: FCmpInst::FCMP_ONE)
323 .Case(S: "ord", Value: FCmpInst::FCMP_ORD)
324 .Case(S: "uno", Value: FCmpInst::FCMP_UNO)
325 .Case(S: "ueq", Value: FCmpInst::FCMP_UEQ)
326 .Case(S: "ugt", Value: FCmpInst::FCMP_UGT)
327 .Case(S: "uge", Value: FCmpInst::FCMP_UGE)
328 .Case(S: "ult", Value: FCmpInst::FCMP_ULT)
329 .Case(S: "ule", Value: FCmpInst::FCMP_ULE)
330 .Case(S: "une", Value: FCmpInst::FCMP_UNE)
331 .Default(Value: FCmpInst::BAD_FCMP_PREDICATE);
332}
333
334FCmpInst::Predicate ConstrainedFPCmpIntrinsic::getPredicate() const {
335 return getFPPredicateFromMD(Op: getArgOperand(i: 2));
336}
337
338unsigned ConstrainedFPIntrinsic::getNonMetadataArgCount() const {
339 // All constrained fp intrinsics have "fpexcept" metadata.
340 unsigned NumArgs = arg_size() - 1;
341
342 // Some intrinsics have "round" metadata.
343 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: getIntrinsicID()))
344 NumArgs -= 1;
345
346 // Compare intrinsics take their predicate as metadata.
347 if (isa<ConstrainedFPCmpIntrinsic>(Val: this))
348 NumArgs -= 1;
349
350 return NumArgs;
351}
352
353bool ConstrainedFPIntrinsic::classof(const IntrinsicInst *I) {
354 return Intrinsic::isConstrainedFPIntrinsic(QID: I->getIntrinsicID());
355}
356
357ElementCount VPIntrinsic::getStaticVectorLength() const {
358 auto GetVectorLengthOfType = [](const Type *T) -> ElementCount {
359 const auto *VT = cast<VectorType>(Val: T);
360 auto ElemCount = VT->getElementCount();
361 return ElemCount;
362 };
363
364 Value *VPMask = getMaskParam();
365 if (!VPMask) {
366 assert((getIntrinsicID() == Intrinsic::vp_merge ||
367 getIntrinsicID() == Intrinsic::vp_select) &&
368 "Unexpected VP intrinsic without mask operand");
369 return GetVectorLengthOfType(getType());
370 }
371 return GetVectorLengthOfType(VPMask->getType());
372}
373
374Value *VPIntrinsic::getMaskParam() const {
375 if (auto MaskPos = getMaskParamPos(IntrinsicID: getIntrinsicID()))
376 return getArgOperand(i: *MaskPos);
377 return nullptr;
378}
379
380void VPIntrinsic::setMaskParam(Value *NewMask) {
381 auto MaskPos = getMaskParamPos(IntrinsicID: getIntrinsicID());
382 setArgOperand(i: *MaskPos, v: NewMask);
383}
384
385Value *VPIntrinsic::getVectorLengthParam() const {
386 if (auto EVLPos = getVectorLengthParamPos(IntrinsicID: getIntrinsicID()))
387 return getArgOperand(i: *EVLPos);
388 return nullptr;
389}
390
391void VPIntrinsic::setVectorLengthParam(Value *NewEVL) {
392 auto EVLPos = getVectorLengthParamPos(IntrinsicID: getIntrinsicID());
393 setArgOperand(i: *EVLPos, v: NewEVL);
394}
395
396std::optional<unsigned>
397VPIntrinsic::getMaskParamPos(Intrinsic::ID IntrinsicID) {
398 switch (IntrinsicID) {
399 default:
400 return std::nullopt;
401
402#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
403 case Intrinsic::VPID: \
404 return MASKPOS;
405#include "llvm/IR/VPIntrinsics.def"
406 }
407}
408
409std::optional<unsigned>
410VPIntrinsic::getVectorLengthParamPos(Intrinsic::ID IntrinsicID) {
411 switch (IntrinsicID) {
412 default:
413 return std::nullopt;
414
415#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
416 case Intrinsic::VPID: \
417 return VLENPOS;
418#include "llvm/IR/VPIntrinsics.def"
419 }
420}
421
422/// \return the alignment of the pointer used by this load/store/gather or
423/// scatter.
424MaybeAlign VPIntrinsic::getPointerAlignment() const {
425 std::optional<unsigned> PtrParamOpt =
426 getMemoryPointerParamPos(getIntrinsicID());
427 assert(PtrParamOpt && "no pointer argument!");
428 return getParamAlign(ArgNo: *PtrParamOpt);
429}
430
431/// \return The pointer operand of this load,store, gather or scatter.
432Value *VPIntrinsic::getMemoryPointerParam() const {
433 if (auto PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID()))
434 return getArgOperand(i: *PtrParamOpt);
435 return nullptr;
436}
437
438std::optional<unsigned>
439VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) {
440 switch (VPID) {
441 default:
442 return std::nullopt;
443 case Intrinsic::vp_store:
444 case Intrinsic::vp_scatter:
445 case Intrinsic::experimental_vp_strided_store:
446 return 1;
447 case Intrinsic::vp_load:
448 case Intrinsic::vp_load_ff:
449 case Intrinsic::vp_gather:
450 case Intrinsic::experimental_vp_strided_load:
451 return 0;
452 }
453}
454
455/// \return The data (payload) operand of this store or scatter.
456Value *VPIntrinsic::getMemoryDataParam() const {
457 auto DataParamOpt = getMemoryDataParamPos(getIntrinsicID());
458 if (!DataParamOpt)
459 return nullptr;
460 return getArgOperand(i: *DataParamOpt);
461}
462
463std::optional<unsigned> VPIntrinsic::getMemoryDataParamPos(Intrinsic::ID VPID) {
464 switch (VPID) {
465 default:
466 return std::nullopt;
467 case Intrinsic::vp_store:
468 case Intrinsic::vp_scatter:
469 case Intrinsic::experimental_vp_strided_store:
470 return 0;
471 }
472}
473
474constexpr bool isVPIntrinsic(Intrinsic::ID ID) {
475 switch (ID) {
476 default:
477 break;
478#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
479 case Intrinsic::VPID: \
480 return true;
481#include "llvm/IR/VPIntrinsics.def"
482 }
483 return false;
484}
485
486bool VPIntrinsic::isVPIntrinsic(Intrinsic::ID ID) {
487 return ::isVPIntrinsic(ID);
488}
489
490// Equivalent non-predicated opcode
491constexpr static std::optional<unsigned>
492getFunctionalOpcodeForVP(Intrinsic::ID ID) {
493 switch (ID) {
494 default:
495 break;
496#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
497#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) return Instruction::OPC;
498#define END_REGISTER_VP_INTRINSIC(VPID) break;
499#include "llvm/IR/VPIntrinsics.def"
500 }
501 return std::nullopt;
502}
503
504std::optional<unsigned>
505VPIntrinsic::getFunctionalOpcodeForVP(Intrinsic::ID ID) {
506 return ::getFunctionalOpcodeForVP(ID);
507}
508
509// Equivalent non-predicated intrinsic ID
510constexpr static std::optional<Intrinsic::ID>
511getFunctionalIntrinsicIDForVP(Intrinsic::ID ID) {
512 switch (ID) {
513 default:
514 break;
515#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
516#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) return Intrinsic::INTRIN;
517#define END_REGISTER_VP_INTRINSIC(VPID) break;
518#include "llvm/IR/VPIntrinsics.def"
519 }
520 return std::nullopt;
521}
522
523std::optional<Intrinsic::ID>
524VPIntrinsic::getFunctionalIntrinsicIDForVP(Intrinsic::ID ID) {
525 return ::getFunctionalIntrinsicIDForVP(ID);
526}
527
528constexpr static bool doesVPHaveNoFunctionalEquivalent(Intrinsic::ID ID) {
529 switch (ID) {
530 default:
531 break;
532#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
533#define VP_PROPERTY_NO_FUNCTIONAL return true;
534#define END_REGISTER_VP_INTRINSIC(VPID) break;
535#include "llvm/IR/VPIntrinsics.def"
536 }
537 return false;
538}
539
540// All VP intrinsics should have an equivalent non-VP opcode or intrinsic
541// defined, or be marked that they don't have one.
542#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) \
543 static_assert(doesVPHaveNoFunctionalEquivalent(Intrinsic::VPID) || \
544 getFunctionalOpcodeForVP(Intrinsic::VPID) || \
545 getFunctionalIntrinsicIDForVP(Intrinsic::VPID));
546#include "llvm/IR/VPIntrinsics.def"
547
548// Equivalent non-predicated constrained intrinsic
549std::optional<Intrinsic::ID>
550VPIntrinsic::getConstrainedIntrinsicIDForVP(Intrinsic::ID ID) {
551 switch (ID) {
552 default:
553 break;
554#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
555#define VP_PROPERTY_CONSTRAINEDFP(CID) return Intrinsic::CID;
556#define END_REGISTER_VP_INTRINSIC(VPID) break;
557#include "llvm/IR/VPIntrinsics.def"
558 }
559 return std::nullopt;
560}
561
562Intrinsic::ID VPIntrinsic::getForOpcode(unsigned IROPC) {
563 switch (IROPC) {
564 default:
565 break;
566
567#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) break;
568#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) case Instruction::OPC:
569#define END_REGISTER_VP_INTRINSIC(VPID) return Intrinsic::VPID;
570#include "llvm/IR/VPIntrinsics.def"
571 }
572 return Intrinsic::not_intrinsic;
573}
574
575constexpr static Intrinsic::ID getForIntrinsic(Intrinsic::ID Id) {
576 if (::isVPIntrinsic(ID: Id))
577 return Id;
578
579 switch (Id) {
580 default:
581 break;
582#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) break;
583#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) case Intrinsic::INTRIN:
584#define END_REGISTER_VP_INTRINSIC(VPID) return Intrinsic::VPID;
585#include "llvm/IR/VPIntrinsics.def"
586 }
587 return Intrinsic::not_intrinsic;
588}
589
590Intrinsic::ID VPIntrinsic::getForIntrinsic(Intrinsic::ID Id) {
591 return ::getForIntrinsic(Id);
592}
593
594bool VPIntrinsic::canIgnoreVectorLengthParam() const {
595 using namespace PatternMatch;
596
597 ElementCount EC = getStaticVectorLength();
598
599 // No vlen param - no lanes masked-off by it.
600 auto *VLParam = getVectorLengthParam();
601 if (!VLParam)
602 return true;
603
604 // Note that the VP intrinsic causes undefined behavior if the Explicit Vector
605 // Length parameter is strictly greater-than the number of vector elements of
606 // the operation. This function returns true when this is detected statically
607 // in the IR.
608
609 // Check whether "W == vscale * EC.getKnownMinValue()"
610 if (EC.isScalable()) {
611 // Compare vscale patterns
612 uint64_t VScaleFactor;
613 if (match(V: VLParam, P: m_Mul(L: m_VScale(), R: m_ConstantInt(V&: VScaleFactor))))
614 return VScaleFactor >= EC.getKnownMinValue();
615 return (EC.getKnownMinValue() == 1) && match(V: VLParam, P: m_VScale());
616 }
617
618 // standard SIMD operation
619 const auto *VLConst = dyn_cast<ConstantInt>(Val: VLParam);
620 if (!VLConst)
621 return false;
622
623 uint64_t VLNum = VLConst->getZExtValue();
624 if (VLNum >= EC.getKnownMinValue())
625 return true;
626
627 return false;
628}
629
630Function *VPIntrinsic::getOrInsertDeclarationForParams(
631 Module *M, Intrinsic::ID VPID, Type *ReturnType, ArrayRef<Value *> Params) {
632 assert(isVPIntrinsic(VPID) && "not a VP intrinsic");
633 Function *VPFunc;
634 switch (VPID) {
635 default: {
636 Type *OverloadTy = Params[0]->getType();
637 if (VPReductionIntrinsic::isVPReduction(ID: VPID))
638 OverloadTy =
639 Params[*VPReductionIntrinsic::getVectorParamPos(ID: VPID)]->getType();
640
641 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, OverloadTys: OverloadTy);
642 break;
643 }
644 case Intrinsic::vp_trunc:
645 case Intrinsic::vp_sext:
646 case Intrinsic::vp_zext:
647 case Intrinsic::vp_fptoui:
648 case Intrinsic::vp_fptosi:
649 case Intrinsic::vp_uitofp:
650 case Intrinsic::vp_sitofp:
651 case Intrinsic::vp_fptrunc:
652 case Intrinsic::vp_fpext:
653 case Intrinsic::vp_ptrtoint:
654 case Intrinsic::vp_inttoptr:
655 case Intrinsic::vp_lrint:
656 case Intrinsic::vp_llrint:
657 case Intrinsic::vp_cttz_elts:
658 VPFunc = Intrinsic::getOrInsertDeclaration(
659 M, id: VPID, OverloadTys: {ReturnType, Params[0]->getType()});
660 break;
661 case Intrinsic::vp_is_fpclass:
662 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, OverloadTys: {Params[0]->getType()});
663 break;
664 case Intrinsic::vp_merge:
665 case Intrinsic::vp_select:
666 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, OverloadTys: {Params[1]->getType()});
667 break;
668 case Intrinsic::vp_load:
669 VPFunc = Intrinsic::getOrInsertDeclaration(
670 M, id: VPID, OverloadTys: {ReturnType, Params[0]->getType()});
671 break;
672 case Intrinsic::vp_load_ff:
673 VPFunc = Intrinsic::getOrInsertDeclaration(
674 M, id: VPID, OverloadTys: {ReturnType->getStructElementType(N: 0), Params[0]->getType()});
675 break;
676 case Intrinsic::experimental_vp_strided_load:
677 VPFunc = Intrinsic::getOrInsertDeclaration(
678 M, id: VPID, OverloadTys: {ReturnType, Params[0]->getType(), Params[1]->getType()});
679 break;
680 case Intrinsic::vp_gather:
681 VPFunc = Intrinsic::getOrInsertDeclaration(
682 M, id: VPID, OverloadTys: {ReturnType, Params[0]->getType()});
683 break;
684 case Intrinsic::vp_store:
685 VPFunc = Intrinsic::getOrInsertDeclaration(
686 M, id: VPID, OverloadTys: {Params[0]->getType(), Params[1]->getType()});
687 break;
688 case Intrinsic::experimental_vp_strided_store:
689 VPFunc = Intrinsic::getOrInsertDeclaration(
690 M, id: VPID,
691 OverloadTys: {Params[0]->getType(), Params[1]->getType(), Params[2]->getType()});
692 break;
693 case Intrinsic::vp_scatter:
694 VPFunc = Intrinsic::getOrInsertDeclaration(
695 M, id: VPID, OverloadTys: {Params[0]->getType(), Params[1]->getType()});
696 break;
697 }
698 assert(VPFunc && "Could not declare VP intrinsic");
699 return VPFunc;
700}
701
702bool VPReductionIntrinsic::isVPReduction(Intrinsic::ID ID) {
703 switch (ID) {
704 case Intrinsic::vp_reduce_add:
705 case Intrinsic::vp_reduce_mul:
706 case Intrinsic::vp_reduce_and:
707 case Intrinsic::vp_reduce_or:
708 case Intrinsic::vp_reduce_xor:
709 case Intrinsic::vp_reduce_smax:
710 case Intrinsic::vp_reduce_smin:
711 case Intrinsic::vp_reduce_umax:
712 case Intrinsic::vp_reduce_umin:
713 case Intrinsic::vp_reduce_fmax:
714 case Intrinsic::vp_reduce_fmin:
715 case Intrinsic::vp_reduce_fmaximum:
716 case Intrinsic::vp_reduce_fminimum:
717 case Intrinsic::vp_reduce_fadd:
718 case Intrinsic::vp_reduce_fmul:
719 return true;
720 default:
721 return false;
722 }
723}
724
725bool VPCastIntrinsic::isVPCast(Intrinsic::ID ID) {
726 // All of the vp.casts correspond to instructions
727 if (std::optional<unsigned> Opc = getFunctionalOpcodeForVP(ID))
728 return Instruction::isCast(Opcode: *Opc);
729 return false;
730}
731
732bool VPCmpIntrinsic::isVPCmp(Intrinsic::ID ID) {
733 switch (ID) {
734 default:
735 return false;
736 case Intrinsic::vp_fcmp:
737 case Intrinsic::vp_icmp:
738 return true;
739 }
740}
741
742bool VPBinOpIntrinsic::isVPBinOp(Intrinsic::ID ID) {
743 switch (ID) {
744 default:
745 break;
746#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
747#define VP_PROPERTY_BINARYOP return true;
748#define END_REGISTER_VP_INTRINSIC(VPID) break;
749#include "llvm/IR/VPIntrinsics.def"
750 }
751 return false;
752}
753
754static ICmpInst::Predicate getIntPredicateFromMD(const Value *Op) {
755 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
756 if (!MD || !isa<MDString>(Val: MD))
757 return ICmpInst::BAD_ICMP_PREDICATE;
758 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
759 .Case(S: "eq", Value: ICmpInst::ICMP_EQ)
760 .Case(S: "ne", Value: ICmpInst::ICMP_NE)
761 .Case(S: "ugt", Value: ICmpInst::ICMP_UGT)
762 .Case(S: "uge", Value: ICmpInst::ICMP_UGE)
763 .Case(S: "ult", Value: ICmpInst::ICMP_ULT)
764 .Case(S: "ule", Value: ICmpInst::ICMP_ULE)
765 .Case(S: "sgt", Value: ICmpInst::ICMP_SGT)
766 .Case(S: "sge", Value: ICmpInst::ICMP_SGE)
767 .Case(S: "slt", Value: ICmpInst::ICMP_SLT)
768 .Case(S: "sle", Value: ICmpInst::ICMP_SLE)
769 .Default(Value: ICmpInst::BAD_ICMP_PREDICATE);
770}
771
772CmpInst::Predicate VPCmpIntrinsic::getPredicate() const {
773 assert(isVPCmp(getIntrinsicID()));
774 return getIntrinsicID() == Intrinsic::vp_fcmp
775 ? getFPPredicateFromMD(Op: getArgOperand(i: 2))
776 : getIntPredicateFromMD(Op: getArgOperand(i: 2));
777}
778
779unsigned VPReductionIntrinsic::getVectorParamPos() const {
780 return *VPReductionIntrinsic::getVectorParamPos(ID: getIntrinsicID());
781}
782
783unsigned VPReductionIntrinsic::getStartParamPos() const {
784 return *VPReductionIntrinsic::getStartParamPos(ID: getIntrinsicID());
785}
786
787std::optional<unsigned>
788VPReductionIntrinsic::getVectorParamPos(Intrinsic::ID ID) {
789 if (isVPReduction(ID))
790 return 1;
791 return std::nullopt;
792}
793
794std::optional<unsigned>
795VPReductionIntrinsic::getStartParamPos(Intrinsic::ID ID) {
796 if (isVPReduction(ID))
797 return 0;
798 return std::nullopt;
799}
800
801Instruction::BinaryOps BinaryOpIntrinsic::getBinaryOp() const {
802 switch (getIntrinsicID()) {
803 case Intrinsic::uadd_with_overflow:
804 case Intrinsic::sadd_with_overflow:
805 case Intrinsic::uadd_sat:
806 case Intrinsic::sadd_sat:
807 return Instruction::Add;
808 case Intrinsic::usub_with_overflow:
809 case Intrinsic::ssub_with_overflow:
810 case Intrinsic::usub_sat:
811 case Intrinsic::ssub_sat:
812 return Instruction::Sub;
813 case Intrinsic::umul_with_overflow:
814 case Intrinsic::smul_with_overflow:
815 return Instruction::Mul;
816 default:
817 llvm_unreachable("Invalid intrinsic");
818 }
819}
820
821bool BinaryOpIntrinsic::isSigned() const {
822 switch (getIntrinsicID()) {
823 case Intrinsic::sadd_with_overflow:
824 case Intrinsic::ssub_with_overflow:
825 case Intrinsic::smul_with_overflow:
826 case Intrinsic::sadd_sat:
827 case Intrinsic::ssub_sat:
828 return true;
829 default:
830 return false;
831 }
832}
833
834unsigned BinaryOpIntrinsic::getNoWrapKind() const {
835 if (isSigned())
836 return OverflowingBinaryOperator::NoSignedWrap;
837 else
838 return OverflowingBinaryOperator::NoUnsignedWrap;
839}
840
841const Value *GCProjectionInst::getStatepoint() const {
842 const Value *Token = getArgOperand(i: 0);
843 if (isa<UndefValue>(Val: Token))
844 return Token;
845
846 // Treat none token as if it was undef here
847 if (isa<ConstantTokenNone>(Val: Token))
848 return UndefValue::get(T: Token->getType());
849
850 // This takes care both of relocates for call statepoints and relocates
851 // on normal path of invoke statepoint.
852 if (!isa<LandingPadInst>(Val: Token))
853 return cast<GCStatepointInst>(Val: Token);
854
855 // This relocate is on exceptional path of an invoke statepoint
856 const BasicBlock *InvokeBB =
857 cast<Instruction>(Val: Token)->getParent()->getUniquePredecessor();
858
859 assert(InvokeBB && "safepoints should have unique landingpads");
860 assert(InvokeBB->getTerminator() &&
861 "safepoint block should be well formed");
862
863 return cast<GCStatepointInst>(Val: InvokeBB->getTerminator());
864}
865
866Value *GCRelocateInst::getBasePtr() const {
867 auto Statepoint = getStatepoint();
868 if (isa<UndefValue>(Val: Statepoint))
869 return UndefValue::get(T: Statepoint->getType());
870
871 auto *GCInst = cast<GCStatepointInst>(Val: Statepoint);
872 if (auto Opt = GCInst->getOperandBundle(ID: LLVMContext::OB_gc_live))
873 return *(Opt->Inputs.begin() + getBasePtrIndex());
874 return *(GCInst->arg_begin() + getBasePtrIndex());
875}
876
877Value *GCRelocateInst::getDerivedPtr() const {
878 auto *Statepoint = getStatepoint();
879 if (isa<UndefValue>(Val: Statepoint))
880 return UndefValue::get(T: Statepoint->getType());
881
882 auto *GCInst = cast<GCStatepointInst>(Val: Statepoint);
883 if (auto Opt = GCInst->getOperandBundle(ID: LLVMContext::OB_gc_live))
884 return *(Opt->Inputs.begin() + getDerivedPtrIndex());
885 return *(GCInst->arg_begin() + getDerivedPtrIndex());
886}
887
888ConvergenceControlInst *ConvergenceControlInst::CreateAnchor(BasicBlock &BB) {
889 Module *M = BB.getModule();
890 Function *Fn = Intrinsic::getOrInsertDeclaration(
891 M, id: llvm::Intrinsic::experimental_convergence_anchor);
892 auto *Call = CallInst::Create(Func: Fn, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
893 return cast<ConvergenceControlInst>(Val: Call);
894}
895
896ConvergenceControlInst *ConvergenceControlInst::CreateEntry(BasicBlock &BB) {
897 Module *M = BB.getModule();
898 Function *Fn = Intrinsic::getOrInsertDeclaration(
899 M, id: llvm::Intrinsic::experimental_convergence_entry);
900 auto *Call = CallInst::Create(Func: Fn, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
901 return cast<ConvergenceControlInst>(Val: Call);
902}
903
904ConvergenceControlInst *
905ConvergenceControlInst::CreateLoop(BasicBlock &BB,
906 ConvergenceControlInst *ParentToken) {
907 Module *M = BB.getModule();
908 Function *Fn = Intrinsic::getOrInsertDeclaration(
909 M, id: llvm::Intrinsic::experimental_convergence_loop);
910 llvm::Value *BundleArgs[] = {ParentToken};
911 llvm::OperandBundleDef OB("convergencectrl", BundleArgs);
912 auto *Call = CallInst::Create(Func: Fn, Args: {}, Bundles: {OB}, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
913 return cast<ConvergenceControlInst>(Val: Call);
914}
915