1//===-- IntrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements methods that make it really easy to deal with intrinsic
10// functions.
11//
12// All intrinsic function calls are instances of the call instruction, so these
13// are all subclasses of the CallInst class. Note that none of these classes
14// has state or virtual methods, which is an important part of this gross/neat
15// hack working.
16//
17// In some cases, arguments to intrinsics need to be generic and are defined as
18// type pointer to empty struct { }*. To access the real item of interest the
19// cast instruction needs to be stripped away.
20//
21//===----------------------------------------------------------------------===//
22
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/Metadata.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/Operator.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/IR/Statepoint.h"
32#include <optional>
33
34using namespace llvm;
35
36bool IntrinsicInst::mayLowerToFunctionCall(Intrinsic::ID IID) {
37 switch (IID) {
38 case Intrinsic::objc_autorelease:
39 case Intrinsic::objc_autoreleasePoolPop:
40 case Intrinsic::objc_autoreleasePoolPush:
41 case Intrinsic::objc_autoreleaseReturnValue:
42 case Intrinsic::objc_copyWeak:
43 case Intrinsic::objc_destroyWeak:
44 case Intrinsic::objc_initWeak:
45 case Intrinsic::objc_loadWeak:
46 case Intrinsic::objc_loadWeakRetained:
47 case Intrinsic::objc_moveWeak:
48 case Intrinsic::objc_release:
49 case Intrinsic::objc_retain:
50 case Intrinsic::objc_retainAutorelease:
51 case Intrinsic::objc_retainAutoreleaseReturnValue:
52 case Intrinsic::objc_retainAutoreleasedReturnValue:
53 case Intrinsic::objc_retainBlock:
54 case Intrinsic::objc_storeStrong:
55 case Intrinsic::objc_storeWeak:
56 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
57 case Intrinsic::objc_retainedObject:
58 case Intrinsic::objc_unretainedObject:
59 case Intrinsic::objc_unretainedPointer:
60 case Intrinsic::objc_retain_autorelease:
61 case Intrinsic::objc_sync_enter:
62 case Intrinsic::objc_sync_exit:
63 return true;
64 default:
65 return false;
66 }
67}
68
69//===----------------------------------------------------------------------===//
70/// DbgVariableIntrinsic - This is the common base class for debug info
71/// intrinsics for variables.
72///
73
74iterator_range<location_op_iterator> RawLocationWrapper::location_ops() const {
75 Metadata *MD = getRawLocation();
76 assert(MD && "First operand of DbgVariableIntrinsic should be non-null.");
77 // If operand is ValueAsMetadata, return a range over just that operand.
78 if (auto *VAM = dyn_cast<ValueAsMetadata>(Val: MD)) {
79 return {location_op_iterator(VAM), location_op_iterator(VAM + 1)};
80 }
81 // If operand is DIArgList, return a range over its args.
82 if (auto *AL = dyn_cast<DIArgList>(Val: MD))
83 return {location_op_iterator(AL->args_begin()),
84 location_op_iterator(AL->args_end())};
85 // Operand must be an empty metadata tuple, so return empty iterator.
86 return {location_op_iterator(static_cast<ValueAsMetadata *>(nullptr)),
87 location_op_iterator(static_cast<ValueAsMetadata *>(nullptr))};
88}
89
90iterator_range<location_op_iterator>
91DbgVariableIntrinsic::location_ops() const {
92 return getWrappedLocation().location_ops();
93}
94
95Value *DbgVariableIntrinsic::getVariableLocationOp(unsigned OpIdx) const {
96 return getWrappedLocation().getVariableLocationOp(OpIdx);
97}
98
99Value *RawLocationWrapper::getVariableLocationOp(unsigned OpIdx) const {
100 Metadata *MD = getRawLocation();
101 assert(MD && "First operand of DbgVariableIntrinsic should be non-null.");
102 if (auto *AL = dyn_cast<DIArgList>(Val: MD))
103 return AL->getArgs()[OpIdx]->getValue();
104 if (isa<MDNode>(Val: MD))
105 return nullptr;
106 assert(
107 isa<ValueAsMetadata>(MD) &&
108 "Attempted to get location operand from DbgVariableIntrinsic with none.");
109 auto *V = cast<ValueAsMetadata>(Val: MD);
110 assert(OpIdx == 0 && "Operand Index must be 0 for a debug intrinsic with a "
111 "single location operand.");
112 return V->getValue();
113}
114
115static ValueAsMetadata *getAsMetadata(Value *V) {
116 return isa<MetadataAsValue>(Val: V) ? dyn_cast<ValueAsMetadata>(
117 Val: cast<MetadataAsValue>(Val: V)->getMetadata())
118 : ValueAsMetadata::get(V);
119}
120
121void DbgVariableIntrinsic::replaceVariableLocationOp(Value *OldValue,
122 Value *NewValue,
123 bool AllowEmpty) {
124 // If OldValue is used as the address part of a dbg.assign intrinsic replace
125 // it with NewValue and return true.
126 auto ReplaceDbgAssignAddress = [this, OldValue, NewValue]() -> bool {
127 auto *DAI = dyn_cast<DbgAssignIntrinsic>(Val: this);
128 if (!DAI || OldValue != DAI->getAddress())
129 return false;
130 DAI->setAddress(NewValue);
131 return true;
132 };
133 bool DbgAssignAddrReplaced = ReplaceDbgAssignAddress();
134 (void)DbgAssignAddrReplaced;
135
136 assert(NewValue && "Values must be non-null");
137 auto Locations = location_ops();
138 auto OldIt = find(Range&: Locations, Val: OldValue);
139 if (OldIt == Locations.end()) {
140 if (AllowEmpty || DbgAssignAddrReplaced)
141 return;
142 assert(DbgAssignAddrReplaced &&
143 "OldValue must be dbg.assign addr if unused in DIArgList");
144 return;
145 }
146
147 assert(OldIt != Locations.end() && "OldValue must be a current location");
148 if (!hasArgList()) {
149 Value *NewOperand = isa<MetadataAsValue>(Val: NewValue)
150 ? NewValue
151 : MetadataAsValue::get(
152 Context&: getContext(), MD: ValueAsMetadata::get(V: NewValue));
153 return setArgOperand(i: 0, v: NewOperand);
154 }
155 SmallVector<ValueAsMetadata *, 4> MDs;
156 ValueAsMetadata *NewOperand = getAsMetadata(V: NewValue);
157 for (auto *VMD : Locations)
158 MDs.push_back(Elt: VMD == *OldIt ? NewOperand : getAsMetadata(V: VMD));
159 setArgOperand(
160 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
161}
162void DbgVariableIntrinsic::replaceVariableLocationOp(unsigned OpIdx,
163 Value *NewValue) {
164 assert(OpIdx < getNumVariableLocationOps() && "Invalid Operand Index");
165 if (!hasArgList()) {
166 Value *NewOperand = isa<MetadataAsValue>(Val: NewValue)
167 ? NewValue
168 : MetadataAsValue::get(
169 Context&: getContext(), MD: ValueAsMetadata::get(V: NewValue));
170 return setArgOperand(i: 0, v: NewOperand);
171 }
172 SmallVector<ValueAsMetadata *, 4> MDs;
173 ValueAsMetadata *NewOperand = getAsMetadata(V: NewValue);
174 for (unsigned Idx = 0; Idx < getNumVariableLocationOps(); ++Idx)
175 MDs.push_back(Elt: Idx == OpIdx ? NewOperand
176 : getAsMetadata(V: getVariableLocationOp(OpIdx: Idx)));
177 setArgOperand(
178 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
179}
180
181void DbgVariableIntrinsic::addVariableLocationOps(ArrayRef<Value *> NewValues,
182 DIExpression *NewExpr) {
183 assert(NewExpr->hasAllLocationOps(getNumVariableLocationOps() +
184 NewValues.size()) &&
185 "NewExpr for debug variable intrinsic does not reference every "
186 "location operand.");
187 assert(!is_contained(NewValues, nullptr) && "New values must be non-null");
188 setArgOperand(i: 2, v: MetadataAsValue::get(Context&: getContext(), MD: NewExpr));
189 SmallVector<ValueAsMetadata *, 4> MDs;
190 for (auto *VMD : location_ops())
191 MDs.push_back(Elt: getAsMetadata(V: VMD));
192 for (auto *VMD : NewValues)
193 MDs.push_back(Elt: getAsMetadata(V: VMD));
194 setArgOperand(
195 i: 0, v: MetadataAsValue::get(Context&: getContext(), MD: DIArgList::get(Context&: getContext(), Args: MDs)));
196}
197
198std::optional<uint64_t> DbgVariableIntrinsic::getFragmentSizeInBits() const {
199 if (auto Fragment = getExpression()->getFragmentInfo())
200 return Fragment->SizeInBits;
201 return getVariable()->getSizeInBits();
202}
203
204Value *DbgAssignIntrinsic::getAddress() const {
205 auto *MD = getRawAddress();
206 if (auto *V = dyn_cast<ValueAsMetadata>(Val: MD))
207 return V->getValue();
208
209 // When the value goes to null, it gets replaced by an empty MDNode.
210 assert(!cast<MDNode>(MD)->getNumOperands() && "Expected an empty MDNode");
211 return nullptr;
212}
213
214void DbgAssignIntrinsic::setAssignId(DIAssignID *New) {
215 setOperand(i: OpAssignID, v: MetadataAsValue::get(Context&: getContext(), MD: New));
216}
217
218void DbgAssignIntrinsic::setAddress(Value *V) {
219 setOperand(i: OpAddress,
220 v: MetadataAsValue::get(Context&: getContext(), MD: ValueAsMetadata::get(V)));
221}
222
223void DbgAssignIntrinsic::setKillAddress() {
224 if (isKillAddress())
225 return;
226 setAddress(PoisonValue::get(T: getAddress()->getType()));
227}
228
229bool DbgAssignIntrinsic::isKillAddress() const {
230 Value *Addr = getAddress();
231 return !Addr || isa<UndefValue>(Val: Addr);
232}
233
234void DbgAssignIntrinsic::setValue(Value *V) {
235 setOperand(i: OpValue,
236 v: MetadataAsValue::get(Context&: getContext(), MD: ValueAsMetadata::get(V)));
237}
238
239ConstantInt *InstrProfCntrInstBase::getNumCounters() const {
240 if (InstrProfValueProfileInst::classof(I: this))
241 llvm_unreachable("InstrProfValueProfileInst does not have counters!");
242 return cast<ConstantInt>(Val: getArgOperand(i: 2));
243}
244
245ConstantInt *InstrProfCntrInstBase::getIndex() const {
246 if (InstrProfValueProfileInst::classof(I: this))
247 llvm_unreachable("Please use InstrProfValueProfileInst::getIndex()");
248 return cast<ConstantInt>(Val: getArgOperand(i: 3));
249}
250
251void InstrProfCntrInstBase::setIndex(uint32_t Idx) {
252 assert(isa<InstrProfCntrInstBase>(this));
253 setArgOperand(i: 3, v: ConstantInt::get(Ty: Type::getInt32Ty(C&: getContext()), V: Idx));
254}
255
256Value *InstrProfIncrementInst::getStep() const {
257 if (InstrProfIncrementInstStep::classof(I: this)) {
258 return getArgOperand(i: 4);
259 }
260 const Module *M = getModule();
261 LLVMContext &Context = M->getContext();
262 return ConstantInt::get(Ty: Type::getInt64Ty(C&: Context), V: 1);
263}
264
265Value *InstrProfCallsite::getCallee() const {
266 if (isa<InstrProfCallsite>(Val: this))
267 return getArgOperand(i: 4);
268 return nullptr;
269}
270
271void InstrProfCallsite::setCallee(Value *Callee) {
272 assert(isa<InstrProfCallsite>(this));
273 setArgOperand(i: 4, v: Callee);
274}
275
276std::optional<RoundingMode> ConstrainedFPIntrinsic::getRoundingMode() const {
277 unsigned NumOperands = arg_size();
278 Metadata *MD = nullptr;
279 auto *MAV = dyn_cast<MetadataAsValue>(Val: getArgOperand(i: NumOperands - 2));
280 if (MAV)
281 MD = MAV->getMetadata();
282 if (!MD || !isa<MDString>(Val: MD))
283 return std::nullopt;
284 return convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
285}
286
287std::optional<fp::ExceptionBehavior>
288ConstrainedFPIntrinsic::getExceptionBehavior() const {
289 unsigned NumOperands = arg_size();
290 Metadata *MD = nullptr;
291 auto *MAV = dyn_cast<MetadataAsValue>(Val: getArgOperand(i: NumOperands - 1));
292 if (MAV)
293 MD = MAV->getMetadata();
294 if (!MD || !isa<MDString>(Val: MD))
295 return std::nullopt;
296 return convertStrToExceptionBehavior(cast<MDString>(Val: MD)->getString());
297}
298
299bool ConstrainedFPIntrinsic::isDefaultFPEnvironment() const {
300 std::optional<fp::ExceptionBehavior> Except = getExceptionBehavior();
301 if (Except) {
302 if (*Except != fp::ebIgnore)
303 return false;
304 }
305
306 std::optional<RoundingMode> Rounding = getRoundingMode();
307 if (Rounding) {
308 if (*Rounding != RoundingMode::NearestTiesToEven)
309 return false;
310 }
311
312 return true;
313}
314
315static FCmpInst::Predicate getFPPredicateFromMD(const Value *Op) {
316 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
317 if (!MD || !isa<MDString>(Val: MD))
318 return FCmpInst::BAD_FCMP_PREDICATE;
319 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
320 .Case(S: "oeq", Value: FCmpInst::FCMP_OEQ)
321 .Case(S: "ogt", Value: FCmpInst::FCMP_OGT)
322 .Case(S: "oge", Value: FCmpInst::FCMP_OGE)
323 .Case(S: "olt", Value: FCmpInst::FCMP_OLT)
324 .Case(S: "ole", Value: FCmpInst::FCMP_OLE)
325 .Case(S: "one", Value: FCmpInst::FCMP_ONE)
326 .Case(S: "ord", Value: FCmpInst::FCMP_ORD)
327 .Case(S: "uno", Value: FCmpInst::FCMP_UNO)
328 .Case(S: "ueq", Value: FCmpInst::FCMP_UEQ)
329 .Case(S: "ugt", Value: FCmpInst::FCMP_UGT)
330 .Case(S: "uge", Value: FCmpInst::FCMP_UGE)
331 .Case(S: "ult", Value: FCmpInst::FCMP_ULT)
332 .Case(S: "ule", Value: FCmpInst::FCMP_ULE)
333 .Case(S: "une", Value: FCmpInst::FCMP_UNE)
334 .Default(Value: FCmpInst::BAD_FCMP_PREDICATE);
335}
336
337FCmpInst::Predicate ConstrainedFPCmpIntrinsic::getPredicate() const {
338 return getFPPredicateFromMD(Op: getArgOperand(i: 2));
339}
340
341unsigned ConstrainedFPIntrinsic::getNonMetadataArgCount() const {
342 // All constrained fp intrinsics have "fpexcept" metadata.
343 unsigned NumArgs = arg_size() - 1;
344
345 // Some intrinsics have "round" metadata.
346 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: getIntrinsicID()))
347 NumArgs -= 1;
348
349 // Compare intrinsics take their predicate as metadata.
350 if (isa<ConstrainedFPCmpIntrinsic>(Val: this))
351 NumArgs -= 1;
352
353 return NumArgs;
354}
355
356bool ConstrainedFPIntrinsic::classof(const IntrinsicInst *I) {
357 return Intrinsic::isConstrainedFPIntrinsic(QID: I->getIntrinsicID());
358}
359
360ElementCount VPIntrinsic::getStaticVectorLength() const {
361 auto GetVectorLengthOfType = [](const Type *T) -> ElementCount {
362 const auto *VT = cast<VectorType>(Val: T);
363 auto ElemCount = VT->getElementCount();
364 return ElemCount;
365 };
366
367 Value *VPMask = getMaskParam();
368 if (!VPMask) {
369 assert((getIntrinsicID() == Intrinsic::vp_merge ||
370 getIntrinsicID() == Intrinsic::vp_select) &&
371 "Unexpected VP intrinsic without mask operand");
372 return GetVectorLengthOfType(getType());
373 }
374 return GetVectorLengthOfType(VPMask->getType());
375}
376
377Value *VPIntrinsic::getMaskParam() const {
378 if (auto MaskPos = getMaskParamPos(IntrinsicID: getIntrinsicID()))
379 return getArgOperand(i: *MaskPos);
380 return nullptr;
381}
382
383void VPIntrinsic::setMaskParam(Value *NewMask) {
384 auto MaskPos = getMaskParamPos(IntrinsicID: getIntrinsicID());
385 setArgOperand(i: *MaskPos, v: NewMask);
386}
387
388Value *VPIntrinsic::getVectorLengthParam() const {
389 if (auto EVLPos = getVectorLengthParamPos(IntrinsicID: getIntrinsicID()))
390 return getArgOperand(i: *EVLPos);
391 return nullptr;
392}
393
394void VPIntrinsic::setVectorLengthParam(Value *NewEVL) {
395 auto EVLPos = getVectorLengthParamPos(IntrinsicID: getIntrinsicID());
396 setArgOperand(i: *EVLPos, v: NewEVL);
397}
398
399std::optional<unsigned>
400VPIntrinsic::getMaskParamPos(Intrinsic::ID IntrinsicID) {
401 switch (IntrinsicID) {
402 default:
403 return std::nullopt;
404
405#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
406 case Intrinsic::VPID: \
407 return MASKPOS;
408#include "llvm/IR/VPIntrinsics.def"
409 }
410}
411
412std::optional<unsigned>
413VPIntrinsic::getVectorLengthParamPos(Intrinsic::ID IntrinsicID) {
414 switch (IntrinsicID) {
415 default:
416 return std::nullopt;
417
418#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
419 case Intrinsic::VPID: \
420 return VLENPOS;
421#include "llvm/IR/VPIntrinsics.def"
422 }
423}
424
425/// \return the alignment of the pointer used by this load/store/gather or
426/// scatter.
427MaybeAlign VPIntrinsic::getPointerAlignment() const {
428 std::optional<unsigned> PtrParamOpt =
429 getMemoryPointerParamPos(getIntrinsicID());
430 assert(PtrParamOpt && "no pointer argument!");
431 return getParamAlign(ArgNo: *PtrParamOpt);
432}
433
434/// \return The pointer operand of this load,store, gather or scatter.
435Value *VPIntrinsic::getMemoryPointerParam() const {
436 if (auto PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID()))
437 return getArgOperand(i: *PtrParamOpt);
438 return nullptr;
439}
440
441std::optional<unsigned>
442VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) {
443 switch (VPID) {
444 default:
445 return std::nullopt;
446 case Intrinsic::vp_store:
447 case Intrinsic::vp_scatter:
448 case Intrinsic::experimental_vp_strided_store:
449 return 1;
450 case Intrinsic::vp_load:
451 case Intrinsic::vp_gather:
452 case Intrinsic::experimental_vp_strided_load:
453 return 0;
454 }
455}
456
457/// \return The data (payload) operand of this store or scatter.
458Value *VPIntrinsic::getMemoryDataParam() const {
459 auto DataParamOpt = getMemoryDataParamPos(getIntrinsicID());
460 if (!DataParamOpt)
461 return nullptr;
462 return getArgOperand(i: *DataParamOpt);
463}
464
465std::optional<unsigned> VPIntrinsic::getMemoryDataParamPos(Intrinsic::ID VPID) {
466 switch (VPID) {
467 default:
468 return std::nullopt;
469 case Intrinsic::vp_store:
470 case Intrinsic::vp_scatter:
471 case Intrinsic::experimental_vp_strided_store:
472 return 0;
473 }
474}
475
476constexpr bool isVPIntrinsic(Intrinsic::ID ID) {
477 switch (ID) {
478 default:
479 break;
480#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
481 case Intrinsic::VPID: \
482 return true;
483#include "llvm/IR/VPIntrinsics.def"
484 }
485 return false;
486}
487
488bool VPIntrinsic::isVPIntrinsic(Intrinsic::ID ID) {
489 return ::isVPIntrinsic(ID);
490}
491
492// Equivalent non-predicated opcode
493constexpr static std::optional<unsigned>
494getFunctionalOpcodeForVP(Intrinsic::ID ID) {
495 switch (ID) {
496 default:
497 break;
498#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
499#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) return Instruction::OPC;
500#define END_REGISTER_VP_INTRINSIC(VPID) break;
501#include "llvm/IR/VPIntrinsics.def"
502 }
503 return std::nullopt;
504}
505
506std::optional<unsigned>
507VPIntrinsic::getFunctionalOpcodeForVP(Intrinsic::ID ID) {
508 return ::getFunctionalOpcodeForVP(ID);
509}
510
511// Equivalent non-predicated intrinsic ID
512constexpr static std::optional<Intrinsic::ID>
513getFunctionalIntrinsicIDForVP(Intrinsic::ID ID) {
514 switch (ID) {
515 default:
516 break;
517#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
518#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) return Intrinsic::INTRIN;
519#define END_REGISTER_VP_INTRINSIC(VPID) break;
520#include "llvm/IR/VPIntrinsics.def"
521 }
522 return std::nullopt;
523}
524
525std::optional<Intrinsic::ID>
526VPIntrinsic::getFunctionalIntrinsicIDForVP(Intrinsic::ID ID) {
527 return ::getFunctionalIntrinsicIDForVP(ID);
528}
529
530constexpr static bool doesVPHaveNoFunctionalEquivalent(Intrinsic::ID ID) {
531 switch (ID) {
532 default:
533 break;
534#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
535#define VP_PROPERTY_NO_FUNCTIONAL return true;
536#define END_REGISTER_VP_INTRINSIC(VPID) break;
537#include "llvm/IR/VPIntrinsics.def"
538 }
539 return false;
540}
541
542// All VP intrinsics should have an equivalent non-VP opcode or intrinsic
543// defined, or be marked that they don't have one.
544#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) \
545 static_assert(doesVPHaveNoFunctionalEquivalent(Intrinsic::VPID) || \
546 getFunctionalOpcodeForVP(Intrinsic::VPID) || \
547 getFunctionalIntrinsicIDForVP(Intrinsic::VPID));
548#include "llvm/IR/VPIntrinsics.def"
549
550// Equivalent non-predicated constrained intrinsic
551std::optional<Intrinsic::ID>
552VPIntrinsic::getConstrainedIntrinsicIDForVP(Intrinsic::ID ID) {
553 switch (ID) {
554 default:
555 break;
556#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
557#define VP_PROPERTY_CONSTRAINEDFP(CID) return Intrinsic::CID;
558#define END_REGISTER_VP_INTRINSIC(VPID) break;
559#include "llvm/IR/VPIntrinsics.def"
560 }
561 return std::nullopt;
562}
563
564Intrinsic::ID VPIntrinsic::getForOpcode(unsigned IROPC) {
565 switch (IROPC) {
566 default:
567 break;
568
569#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) break;
570#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) case Instruction::OPC:
571#define END_REGISTER_VP_INTRINSIC(VPID) return Intrinsic::VPID;
572#include "llvm/IR/VPIntrinsics.def"
573 }
574 return Intrinsic::not_intrinsic;
575}
576
577constexpr static Intrinsic::ID getForIntrinsic(Intrinsic::ID Id) {
578 if (::isVPIntrinsic(ID: Id))
579 return Id;
580
581 switch (Id) {
582 default:
583 break;
584#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) break;
585#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) case Intrinsic::INTRIN:
586#define END_REGISTER_VP_INTRINSIC(VPID) return Intrinsic::VPID;
587#include "llvm/IR/VPIntrinsics.def"
588 }
589 return Intrinsic::not_intrinsic;
590}
591
592Intrinsic::ID VPIntrinsic::getForIntrinsic(Intrinsic::ID Id) {
593 return ::getForIntrinsic(Id);
594}
595
596bool VPIntrinsic::canIgnoreVectorLengthParam() const {
597 using namespace PatternMatch;
598
599 ElementCount EC = getStaticVectorLength();
600
601 // No vlen param - no lanes masked-off by it.
602 auto *VLParam = getVectorLengthParam();
603 if (!VLParam)
604 return true;
605
606 // Note that the VP intrinsic causes undefined behavior if the Explicit Vector
607 // Length parameter is strictly greater-than the number of vector elements of
608 // the operation. This function returns true when this is detected statically
609 // in the IR.
610
611 // Check whether "W == vscale * EC.getKnownMinValue()"
612 if (EC.isScalable()) {
613 // Compare vscale patterns
614 uint64_t VScaleFactor;
615 if (match(V: VLParam, P: m_Mul(L: m_VScale(), R: m_ConstantInt(V&: VScaleFactor))))
616 return VScaleFactor >= EC.getKnownMinValue();
617 return (EC.getKnownMinValue() == 1) && match(V: VLParam, P: m_VScale());
618 }
619
620 // standard SIMD operation
621 const auto *VLConst = dyn_cast<ConstantInt>(Val: VLParam);
622 if (!VLConst)
623 return false;
624
625 uint64_t VLNum = VLConst->getZExtValue();
626 if (VLNum >= EC.getKnownMinValue())
627 return true;
628
629 return false;
630}
631
632Function *VPIntrinsic::getOrInsertDeclarationForParams(
633 Module *M, Intrinsic::ID VPID, Type *ReturnType, ArrayRef<Value *> Params) {
634 assert(isVPIntrinsic(VPID) && "not a VP intrinsic");
635 Function *VPFunc;
636 switch (VPID) {
637 default: {
638 Type *OverloadTy = Params[0]->getType();
639 if (VPReductionIntrinsic::isVPReduction(ID: VPID))
640 OverloadTy =
641 Params[*VPReductionIntrinsic::getVectorParamPos(ID: VPID)]->getType();
642
643 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, Tys: OverloadTy);
644 break;
645 }
646 case Intrinsic::vp_trunc:
647 case Intrinsic::vp_sext:
648 case Intrinsic::vp_zext:
649 case Intrinsic::vp_fptoui:
650 case Intrinsic::vp_fptosi:
651 case Intrinsic::vp_uitofp:
652 case Intrinsic::vp_sitofp:
653 case Intrinsic::vp_fptrunc:
654 case Intrinsic::vp_fpext:
655 case Intrinsic::vp_ptrtoint:
656 case Intrinsic::vp_inttoptr:
657 case Intrinsic::vp_lrint:
658 case Intrinsic::vp_llrint:
659 case Intrinsic::vp_cttz_elts:
660 VPFunc = Intrinsic::getOrInsertDeclaration(
661 M, id: VPID, Tys: {ReturnType, Params[0]->getType()});
662 break;
663 case Intrinsic::vp_is_fpclass:
664 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, Tys: {Params[0]->getType()});
665 break;
666 case Intrinsic::vp_merge:
667 case Intrinsic::vp_select:
668 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, Tys: {Params[1]->getType()});
669 break;
670 case Intrinsic::vp_load:
671 VPFunc = Intrinsic::getOrInsertDeclaration(
672 M, id: VPID, Tys: {ReturnType, Params[0]->getType()});
673 break;
674 case Intrinsic::experimental_vp_strided_load:
675 VPFunc = Intrinsic::getOrInsertDeclaration(
676 M, id: VPID, Tys: {ReturnType, Params[0]->getType(), Params[1]->getType()});
677 break;
678 case Intrinsic::vp_gather:
679 VPFunc = Intrinsic::getOrInsertDeclaration(
680 M, id: VPID, Tys: {ReturnType, Params[0]->getType()});
681 break;
682 case Intrinsic::vp_store:
683 VPFunc = Intrinsic::getOrInsertDeclaration(
684 M, id: VPID, Tys: {Params[0]->getType(), Params[1]->getType()});
685 break;
686 case Intrinsic::experimental_vp_strided_store:
687 VPFunc = Intrinsic::getOrInsertDeclaration(
688 M, id: VPID,
689 Tys: {Params[0]->getType(), Params[1]->getType(), Params[2]->getType()});
690 break;
691 case Intrinsic::vp_scatter:
692 VPFunc = Intrinsic::getOrInsertDeclaration(
693 M, id: VPID, Tys: {Params[0]->getType(), Params[1]->getType()});
694 break;
695 case Intrinsic::experimental_vp_splat:
696 VPFunc = Intrinsic::getOrInsertDeclaration(M, id: VPID, Tys: ReturnType);
697 break;
698 }
699 assert(VPFunc && "Could not declare VP intrinsic");
700 return VPFunc;
701}
702
703bool VPReductionIntrinsic::isVPReduction(Intrinsic::ID ID) {
704 switch (ID) {
705 case Intrinsic::vp_reduce_add:
706 case Intrinsic::vp_reduce_mul:
707 case Intrinsic::vp_reduce_and:
708 case Intrinsic::vp_reduce_or:
709 case Intrinsic::vp_reduce_xor:
710 case Intrinsic::vp_reduce_smax:
711 case Intrinsic::vp_reduce_smin:
712 case Intrinsic::vp_reduce_umax:
713 case Intrinsic::vp_reduce_umin:
714 case Intrinsic::vp_reduce_fmax:
715 case Intrinsic::vp_reduce_fmin:
716 case Intrinsic::vp_reduce_fmaximum:
717 case Intrinsic::vp_reduce_fminimum:
718 case Intrinsic::vp_reduce_fadd:
719 case Intrinsic::vp_reduce_fmul:
720 return true;
721 default:
722 return false;
723 }
724}
725
726bool VPCastIntrinsic::isVPCast(Intrinsic::ID ID) {
727 // All of the vp.casts correspond to instructions
728 if (std::optional<unsigned> Opc = getFunctionalOpcodeForVP(ID))
729 return Instruction::isCast(Opcode: *Opc);
730 return false;
731}
732
733bool VPCmpIntrinsic::isVPCmp(Intrinsic::ID ID) {
734 switch (ID) {
735 default:
736 return false;
737 case Intrinsic::vp_fcmp:
738 case Intrinsic::vp_icmp:
739 return true;
740 }
741}
742
743bool VPBinOpIntrinsic::isVPBinOp(Intrinsic::ID ID) {
744 switch (ID) {
745 default:
746 break;
747#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
748#define VP_PROPERTY_BINARYOP return true;
749#define END_REGISTER_VP_INTRINSIC(VPID) break;
750#include "llvm/IR/VPIntrinsics.def"
751 }
752 return false;
753}
754
755static ICmpInst::Predicate getIntPredicateFromMD(const Value *Op) {
756 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
757 if (!MD || !isa<MDString>(Val: MD))
758 return ICmpInst::BAD_ICMP_PREDICATE;
759 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
760 .Case(S: "eq", Value: ICmpInst::ICMP_EQ)
761 .Case(S: "ne", Value: ICmpInst::ICMP_NE)
762 .Case(S: "ugt", Value: ICmpInst::ICMP_UGT)
763 .Case(S: "uge", Value: ICmpInst::ICMP_UGE)
764 .Case(S: "ult", Value: ICmpInst::ICMP_ULT)
765 .Case(S: "ule", Value: ICmpInst::ICMP_ULE)
766 .Case(S: "sgt", Value: ICmpInst::ICMP_SGT)
767 .Case(S: "sge", Value: ICmpInst::ICMP_SGE)
768 .Case(S: "slt", Value: ICmpInst::ICMP_SLT)
769 .Case(S: "sle", Value: ICmpInst::ICMP_SLE)
770 .Default(Value: ICmpInst::BAD_ICMP_PREDICATE);
771}
772
773CmpInst::Predicate VPCmpIntrinsic::getPredicate() const {
774 assert(isVPCmp(getIntrinsicID()));
775 return getIntrinsicID() == Intrinsic::vp_fcmp
776 ? getFPPredicateFromMD(Op: getArgOperand(i: 2))
777 : getIntPredicateFromMD(Op: getArgOperand(i: 2));
778}
779
780unsigned VPReductionIntrinsic::getVectorParamPos() const {
781 return *VPReductionIntrinsic::getVectorParamPos(ID: getIntrinsicID());
782}
783
784unsigned VPReductionIntrinsic::getStartParamPos() const {
785 return *VPReductionIntrinsic::getStartParamPos(ID: getIntrinsicID());
786}
787
788std::optional<unsigned>
789VPReductionIntrinsic::getVectorParamPos(Intrinsic::ID ID) {
790 if (isVPReduction(ID))
791 return 1;
792 return std::nullopt;
793}
794
795std::optional<unsigned>
796VPReductionIntrinsic::getStartParamPos(Intrinsic::ID ID) {
797 if (isVPReduction(ID))
798 return 0;
799 return std::nullopt;
800}
801
802Instruction::BinaryOps BinaryOpIntrinsic::getBinaryOp() const {
803 switch (getIntrinsicID()) {
804 case Intrinsic::uadd_with_overflow:
805 case Intrinsic::sadd_with_overflow:
806 case Intrinsic::uadd_sat:
807 case Intrinsic::sadd_sat:
808 return Instruction::Add;
809 case Intrinsic::usub_with_overflow:
810 case Intrinsic::ssub_with_overflow:
811 case Intrinsic::usub_sat:
812 case Intrinsic::ssub_sat:
813 return Instruction::Sub;
814 case Intrinsic::umul_with_overflow:
815 case Intrinsic::smul_with_overflow:
816 return Instruction::Mul;
817 default:
818 llvm_unreachable("Invalid intrinsic");
819 }
820}
821
822bool BinaryOpIntrinsic::isSigned() const {
823 switch (getIntrinsicID()) {
824 case Intrinsic::sadd_with_overflow:
825 case Intrinsic::ssub_with_overflow:
826 case Intrinsic::smul_with_overflow:
827 case Intrinsic::sadd_sat:
828 case Intrinsic::ssub_sat:
829 return true;
830 default:
831 return false;
832 }
833}
834
835unsigned BinaryOpIntrinsic::getNoWrapKind() const {
836 if (isSigned())
837 return OverflowingBinaryOperator::NoSignedWrap;
838 else
839 return OverflowingBinaryOperator::NoUnsignedWrap;
840}
841
842const Value *GCProjectionInst::getStatepoint() const {
843 const Value *Token = getArgOperand(i: 0);
844 if (isa<UndefValue>(Val: Token))
845 return Token;
846
847 // Treat none token as if it was undef here
848 if (isa<ConstantTokenNone>(Val: Token))
849 return UndefValue::get(T: Token->getType());
850
851 // This takes care both of relocates for call statepoints and relocates
852 // on normal path of invoke statepoint.
853 if (!isa<LandingPadInst>(Val: Token))
854 return cast<GCStatepointInst>(Val: Token);
855
856 // This relocate is on exceptional path of an invoke statepoint
857 const BasicBlock *InvokeBB =
858 cast<Instruction>(Val: Token)->getParent()->getUniquePredecessor();
859
860 assert(InvokeBB && "safepoints should have unique landingpads");
861 assert(InvokeBB->getTerminator() &&
862 "safepoint block should be well formed");
863
864 return cast<GCStatepointInst>(Val: InvokeBB->getTerminator());
865}
866
867Value *GCRelocateInst::getBasePtr() const {
868 auto Statepoint = getStatepoint();
869 if (isa<UndefValue>(Val: Statepoint))
870 return UndefValue::get(T: Statepoint->getType());
871
872 auto *GCInst = cast<GCStatepointInst>(Val: Statepoint);
873 if (auto Opt = GCInst->getOperandBundle(ID: LLVMContext::OB_gc_live))
874 return *(Opt->Inputs.begin() + getBasePtrIndex());
875 return *(GCInst->arg_begin() + getBasePtrIndex());
876}
877
878Value *GCRelocateInst::getDerivedPtr() const {
879 auto *Statepoint = getStatepoint();
880 if (isa<UndefValue>(Val: Statepoint))
881 return UndefValue::get(T: Statepoint->getType());
882
883 auto *GCInst = cast<GCStatepointInst>(Val: Statepoint);
884 if (auto Opt = GCInst->getOperandBundle(ID: LLVMContext::OB_gc_live))
885 return *(Opt->Inputs.begin() + getDerivedPtrIndex());
886 return *(GCInst->arg_begin() + getDerivedPtrIndex());
887}
888
889ConvergenceControlInst *ConvergenceControlInst::CreateAnchor(BasicBlock &BB) {
890 Module *M = BB.getModule();
891 Function *Fn = Intrinsic::getOrInsertDeclaration(
892 M, id: llvm::Intrinsic::experimental_convergence_anchor);
893 auto *Call = CallInst::Create(Func: Fn, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
894 return cast<ConvergenceControlInst>(Val: Call);
895}
896
897ConvergenceControlInst *ConvergenceControlInst::CreateEntry(BasicBlock &BB) {
898 Module *M = BB.getModule();
899 Function *Fn = Intrinsic::getOrInsertDeclaration(
900 M, id: llvm::Intrinsic::experimental_convergence_entry);
901 auto *Call = CallInst::Create(Func: Fn, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
902 return cast<ConvergenceControlInst>(Val: Call);
903}
904
905ConvergenceControlInst *
906ConvergenceControlInst::CreateLoop(BasicBlock &BB,
907 ConvergenceControlInst *ParentToken) {
908 Module *M = BB.getModule();
909 Function *Fn = Intrinsic::getOrInsertDeclaration(
910 M, id: llvm::Intrinsic::experimental_convergence_loop);
911 llvm::Value *BundleArgs[] = {ParentToken};
912 llvm::OperandBundleDef OB("convergencectrl", BundleArgs);
913 auto *Call = CallInst::Create(Func: Fn, Args: {}, Bundles: {OB}, NameStr: "", InsertBefore: BB.getFirstInsertionPt());
914 return cast<ConvergenceControlInst>(Val: Call);
915}
916