1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallVectorExtras.h"
17#include "llvm/IR/Constant.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/DerivedTypes.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GlobalValue.h"
22#include "llvm/IR/GlobalVariable.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/NoFolder.h"
28#include "llvm/IR/Operator.h"
29#include "llvm/IR/ProfDataUtils.h"
30#include "llvm/IR/Statepoint.h"
31#include "llvm/IR/Type.h"
32#include "llvm/IR/Value.h"
33#include "llvm/Support/Casting.h"
34#include <cassert>
35#include <cstdint>
36#include <optional>
37#include <vector>
38
39using namespace llvm;
40
41/// CreateGlobalString - Make a new global variable with an initializer that
42/// has array of i8 type filled in with the nul terminated string value
43/// specified. If Name is specified, it is the name of the global variable
44/// created.
45GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
46 const Twine &Name,
47 unsigned AddressSpace,
48 Module *M, bool AddNull) {
49 Constant *StrConstant = ConstantDataArray::getString(Context, Initializer: Str, AddNull);
50 if (!M)
51 M = BB->getParent()->getParent();
52 auto *GV = new GlobalVariable(
53 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
54 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
55 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
56 GV->setAlignment(M->getDataLayout().getPrefTypeAlign(Ty: getInt8Ty()));
57 return GV;
58}
59
60Type *IRBuilderBase::getCurrentFunctionReturnType() const {
61 assert(BB && BB->getParent() && "No current function!");
62 return BB->getParent()->getReturnType();
63}
64
65DebugLoc IRBuilderBase::getCurrentDebugLocation() const { return StoredDL; }
66void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
67 // We prefer to set our current debug location if any has been set, but if
68 // our debug location is empty and I has a valid location, we shouldn't
69 // overwrite it.
70 I->setDebugLoc(StoredDL.orElse(Other: I->getDebugLoc()));
71}
72
73Value *IRBuilderBase::CreateAggregateCast(Value *V, Type *DestTy) {
74 Type *SrcTy = V->getType();
75 if (SrcTy == DestTy)
76 return V;
77
78 if (SrcTy->isAggregateType()) {
79 unsigned NumElements;
80 if (SrcTy->isStructTy()) {
81 assert(DestTy->isStructTy() && "Expected StructType");
82 assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements() &&
83 "Expected StructTypes with equal number of elements");
84 NumElements = SrcTy->getStructNumElements();
85 } else {
86 assert(SrcTy->isArrayTy() && DestTy->isArrayTy() && "Expected ArrayType");
87 assert(SrcTy->getArrayNumElements() == DestTy->getArrayNumElements() &&
88 "Expected ArrayTypes with equal number of elements");
89 NumElements = SrcTy->getArrayNumElements();
90 }
91
92 Value *Result = PoisonValue::get(T: DestTy);
93 for (unsigned I = 0; I < NumElements; ++I) {
94 Type *ElementTy = SrcTy->isStructTy() ? DestTy->getStructElementType(N: I)
95 : DestTy->getArrayElementType();
96 Value *Element =
97 CreateAggregateCast(V: CreateExtractValue(Agg: V, Idxs: ArrayRef(I)), DestTy: ElementTy);
98
99 Result = CreateInsertValue(Agg: Result, Val: Element, Idxs: ArrayRef(I));
100 }
101 return Result;
102 }
103
104 return CreateBitOrPointerCast(V, DestTy);
105}
106
107Value *IRBuilderBase::CreateBitPreservingCastChain(const DataLayout &DL,
108 Value *V, Type *NewTy) {
109 Type *OldTy = V->getType();
110
111 if (OldTy == NewTy)
112 return V;
113
114 assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
115 "Integer types must be the exact same to convert.");
116
117 // A variant of bitcast that supports a mixture of fixed and scalable types
118 // that are know to have the same size.
119 auto CreateBitCastLike = [this](Value *In, Type *Ty) -> Value * {
120 Type *InTy = In->getType();
121 if (InTy == Ty)
122 return In;
123
124 if (isa<FixedVectorType>(Val: InTy) && isa<ScalableVectorType>(Val: Ty)) {
125 // For vscale_range(2) expand <4 x i32> to <vscale x 4 x i16> -->
126 // <4 x i32> to <vscale x 2 x i32> to <vscale x 4 x i16>
127 auto *VTy = VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: Ty), EltTy: InTy);
128 return CreateBitCast(
129 V: CreateInsertVector(DstType: VTy, SrcVec: PoisonValue::get(T: VTy), SubVec: In, Idx: getInt64(C: 0)), DestTy: Ty);
130 }
131
132 if (isa<ScalableVectorType>(Val: InTy) && isa<FixedVectorType>(Val: Ty)) {
133 // For vscale_range(2) expand <vscale x 4 x i16> to <4 x i32> -->
134 // <vscale x 4 x i16> to <vscale x 2 x i32> to <4 x i32>
135 auto *VTy = VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: InTy), EltTy: Ty);
136 return CreateExtractVector(DstType: Ty, SrcVec: CreateBitCast(V: In, DestTy: VTy), Idx: getInt64(C: 0));
137 }
138
139 return CreateBitCast(V: In, DestTy: Ty);
140 };
141
142 // See if we need inttoptr for this type pair. May require additional bitcast.
143 if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
144 // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
145 // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
146 // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*>
147 // Directly handle i64 to i8*
148 return CreateIntToPtr(V: CreateBitCastLike(V, DL.getIntPtrType(NewTy)), DestTy: NewTy);
149 }
150
151 // See if we need ptrtoint for this type pair. May require additional bitcast.
152 if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) {
153 // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128
154 // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
155 // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32>
156 // Expand i8* to i64 --> i8* to i64 to i64
157 return CreateBitCastLike(CreatePtrToInt(V, DestTy: DL.getIntPtrType(OldTy)), NewTy);
158 }
159
160 if (OldTy->isPtrOrPtrVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
161 unsigned OldAS = OldTy->getPointerAddressSpace();
162 unsigned NewAS = NewTy->getPointerAddressSpace();
163 // To convert pointers with different address spaces (they are already
164 // checked convertible, i.e. they have the same pointer size), so far we
165 // cannot use `bitcast` (which has restrict on the same address space) or
166 // `addrspacecast` (which is not always no-op casting). Instead, use a pair
167 // of no-op `ptrtoint`/`inttoptr` casts through an integer with the same bit
168 // size.
169 if (OldAS != NewAS) {
170 return CreateIntToPtr(
171 V: CreateBitCastLike(CreatePtrToInt(V, DestTy: DL.getIntPtrType(OldTy)),
172 DL.getIntPtrType(NewTy)),
173 DestTy: NewTy);
174 }
175 }
176
177 return CreateBitCastLike(V, NewTy);
178}
179
180CallInst *
181IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
182 const Twine &Name, FMFSource FMFSource,
183 ArrayRef<OperandBundleDef> OpBundles) {
184 CallInst *CI = CreateCall(Callee, Args: Ops, OpBundles, Name);
185 if (isa<FPMathOperator>(Val: CI))
186 CI->setFastMathFlags(FMFSource.get(Default: FMF));
187 return CI;
188}
189
190static Value *CreateVScaleMultiple(IRBuilderBase &B, Type *Ty, uint64_t Scale) {
191 Value *VScale = B.CreateVScale(Ty);
192 if (Scale == 1)
193 return VScale;
194
195 return B.CreateNUWMul(LHS: VScale, RHS: ConstantInt::get(Ty, V: Scale));
196}
197
198Value *IRBuilderBase::CreateElementCount(Type *Ty, ElementCount EC) {
199 if (EC.isFixed() || EC.isZero())
200 return ConstantInt::get(Ty, V: EC.getKnownMinValue());
201
202 return CreateVScaleMultiple(B&: *this, Ty, Scale: EC.getKnownMinValue());
203}
204
205Value *IRBuilderBase::CreateTypeSize(Type *Ty, TypeSize Size) {
206 if (Size.isFixed() || Size.isZero())
207 return ConstantInt::get(Ty, V: Size.getKnownMinValue());
208
209 return CreateVScaleMultiple(B&: *this, Ty, Scale: Size.getKnownMinValue());
210}
211
212Value *IRBuilderBase::CreateAllocationSize(Type *DestTy, AllocaInst *AI) {
213 const DataLayout &DL = BB->getDataLayout();
214 TypeSize ElemSize = DL.getTypeAllocSize(Ty: AI->getAllocatedType());
215 Value *Size = CreateTypeSize(Ty: DestTy, Size: ElemSize);
216 if (AI->isArrayAllocation())
217 Size = CreateMul(LHS: CreateZExtOrTrunc(V: AI->getArraySize(), DestTy), RHS: Size);
218 return Size;
219}
220
221Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
222 Type *STy = DstType->getScalarType();
223 if (isa<ScalableVectorType>(Val: DstType)) {
224 Type *StepVecType = DstType;
225 // TODO: We expect this special case (element type < 8 bits) to be
226 // temporary - once the intrinsic properly supports < 8 bits this code
227 // can be removed.
228 if (STy->getScalarSizeInBits() < 8)
229 StepVecType =
230 VectorType::get(ElementType: getInt8Ty(), Other: cast<ScalableVectorType>(Val: DstType));
231 Value *Res = CreateIntrinsic(ID: Intrinsic::stepvector, OverloadTypes: {StepVecType}, Args: {},
232 FMFSource: nullptr, Name);
233 if (StepVecType != DstType)
234 Res = CreateTrunc(V: Res, DestTy: DstType);
235 return Res;
236 }
237
238 unsigned NumEls = cast<FixedVectorType>(Val: DstType)->getNumElements();
239
240 // Create a vector of consecutive numbers from zero to VF.
241 // It's okay if the values wrap around.
242 SmallVector<Constant *, 8> Indices;
243 for (unsigned i = 0; i < NumEls; ++i)
244 Indices.push_back(
245 Elt: ConstantInt::get(Ty: STy, V: i, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
246
247 // Add the consecutive indices to the vector value.
248 return ConstantVector::get(V: Indices);
249}
250
251CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
252 MaybeAlign Align, bool isVolatile,
253 const AAMDNodes &AAInfo) {
254 Value *Ops[] = {Ptr, Val, Size, getInt1(V: isVolatile)};
255 Type *Tys[] = {Ptr->getType(), Size->getType()};
256
257 auto *CI = cast<MemSetInst>(
258 Val: CreateIntrinsicWithoutFolding(ID: Intrinsic::memset, OverloadTypes: Tys, Args: Ops));
259
260 if (Align)
261 CI->setDestAlignment(*Align);
262 CI->setAAMetadata(AAInfo);
263 return CI;
264}
265
266CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
267 Value *Val, Value *Size,
268 bool IsVolatile,
269 const AAMDNodes &AAInfo) {
270 Value *Ops[] = {Dst, Val, Size, getInt1(V: IsVolatile)};
271 Type *Tys[] = {Dst->getType(), Size->getType()};
272
273 auto *CI = cast<MemSetInst>(
274 Val: CreateIntrinsicWithoutFolding(ID: Intrinsic::memset_inline, OverloadTypes: Tys, Args: Ops));
275
276 if (DstAlign)
277 CI->setDestAlignment(*DstAlign);
278 CI->setAAMetadata(AAInfo);
279 return CI;
280}
281
282CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
283 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
284 const AAMDNodes &AAInfo) {
285
286 Value *Ops[] = {Ptr, Val, Size, getInt32(C: ElementSize)};
287 Type *Tys[] = {Ptr->getType(), Size->getType()};
288
289 auto *CI = cast<AnyMemSetInst>(Val: CreateIntrinsicWithoutFolding(
290 ID: Intrinsic::memset_element_unordered_atomic, OverloadTypes: Tys, Args: Ops));
291 CI->setDestAlignment(Alignment);
292 CI->setAAMetadata(AAInfo);
293 return CI;
294}
295
296CallInst *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
297 MaybeAlign DstAlign, Value *Src,
298 MaybeAlign SrcAlign, Value *Size,
299 bool isVolatile,
300 const AAMDNodes &AAInfo) {
301 assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||
302 IntrID == Intrinsic::memmove) &&
303 "Unexpected intrinsic ID");
304 Value *Ops[] = {Dst, Src, Size, getInt1(V: isVolatile)};
305 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
306
307 auto *MCI =
308 cast<MemTransferInst>(Val: CreateIntrinsicWithoutFolding(ID: IntrID, OverloadTypes: Tys, Args: Ops));
309
310 if (DstAlign)
311 MCI->setDestAlignment(*DstAlign);
312 if (SrcAlign)
313 MCI->setSourceAlignment(*SrcAlign);
314 MCI->setAAMetadata(AAInfo);
315 return MCI;
316}
317
318CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
319 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
320 uint32_t ElementSize, const AAMDNodes &AAInfo) {
321 assert(DstAlign >= ElementSize &&
322 "Pointer alignment must be at least element size");
323 assert(SrcAlign >= ElementSize &&
324 "Pointer alignment must be at least element size");
325 Value *Ops[] = {Dst, Src, Size, getInt32(C: ElementSize)};
326 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
327
328 auto *AMCI = cast<AnyMemCpyInst>(Val: CreateIntrinsicWithoutFolding(
329 ID: Intrinsic::memcpy_element_unordered_atomic, OverloadTypes: Tys, Args: Ops));
330
331 // Set the alignment of the pointer args.
332 AMCI->setDestAlignment(DstAlign);
333 AMCI->setSourceAlignment(SrcAlign);
334 AMCI->setAAMetadata(AAInfo);
335 return AMCI;
336}
337
338/// isConstantOne - Return true only if val is constant int 1
339static bool isConstantOne(const Value *Val) {
340 assert(Val && "isConstantOne does not work with nullptr Val");
341 const ConstantInt *CVal = dyn_cast<ConstantInt>(Val);
342 return CVal && CVal->isOne();
343}
344
345CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,
346 Value *AllocSize, Value *ArraySize,
347 ArrayRef<OperandBundleDef> OpB,
348 Function *MallocF, const Twine &Name) {
349 // malloc(type) becomes:
350 // i8* malloc(typeSize)
351 // malloc(type, arraySize) becomes:
352 // i8* malloc(typeSize*arraySize)
353 if (!ArraySize)
354 ArraySize = ConstantInt::get(Ty: IntPtrTy, V: 1);
355 else if (ArraySize->getType() != IntPtrTy)
356 ArraySize = CreateIntCast(V: ArraySize, DestTy: IntPtrTy, isSigned: false);
357
358 if (!isConstantOne(Val: ArraySize)) {
359 if (isConstantOne(Val: AllocSize)) {
360 AllocSize = ArraySize; // Operand * 1 = Operand
361 } else {
362 // Multiply type size by the array size...
363 AllocSize = CreateMul(LHS: ArraySize, RHS: AllocSize, Name: "mallocsize");
364 }
365 }
366
367 assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");
368 // Create the call to Malloc.
369 Module *M = BB->getParent()->getParent();
370 Type *BPTy = PointerType::getUnqual(C&: Context);
371 FunctionCallee MallocFunc = MallocF;
372 if (!MallocFunc)
373 // prototype malloc as "void *malloc(size_t)"
374 MallocFunc = M->getOrInsertFunction(Name: "malloc", RetTy: BPTy, Args: IntPtrTy);
375 CallInst *MCall = CreateCall(Callee: MallocFunc, Args: AllocSize, OpBundles: OpB, Name);
376
377 MCall->setTailCall();
378 if (Function *F = dyn_cast<Function>(Val: MallocFunc.getCallee())) {
379 MCall->setCallingConv(F->getCallingConv());
380 F->setReturnDoesNotAlias();
381 }
382
383 assert(!MCall->getType()->isVoidTy() && "Malloc has void return type");
384
385 return MCall;
386}
387
388CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,
389 Value *AllocSize, Value *ArraySize,
390 Function *MallocF, const Twine &Name) {
391
392 return CreateMalloc(IntPtrTy, AllocTy, AllocSize, ArraySize, OpB: {}, MallocF,
393 Name);
394}
395
396/// CreateFree - Generate the IR for a call to the builtin free function.
397CallInst *IRBuilderBase::CreateFree(Value *Source,
398 ArrayRef<OperandBundleDef> Bundles) {
399 assert(Source->getType()->isPointerTy() &&
400 "Can not free something of nonpointer type!");
401
402 Module *M = BB->getParent()->getParent();
403
404 Type *VoidTy = Type::getVoidTy(C&: M->getContext());
405 Type *VoidPtrTy = PointerType::getUnqual(C&: M->getContext());
406 // prototype free as "void free(void*)"
407 FunctionCallee FreeFunc = M->getOrInsertFunction(Name: "free", RetTy: VoidTy, Args: VoidPtrTy);
408 CallInst *Result = CreateCall(Callee: FreeFunc, Args: Source, OpBundles: Bundles, Name: "");
409 Result->setTailCall();
410 if (Function *F = dyn_cast<Function>(Val: FreeFunc.getCallee()))
411 Result->setCallingConv(F->getCallingConv());
412
413 return Result;
414}
415
416CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
417 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
418 uint32_t ElementSize, const AAMDNodes &AAInfo) {
419 assert(DstAlign >= ElementSize &&
420 "Pointer alignment must be at least element size");
421 assert(SrcAlign >= ElementSize &&
422 "Pointer alignment must be at least element size");
423 Value *Ops[] = {Dst, Src, Size, getInt32(C: ElementSize)};
424 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
425
426 CallInst *CI = CreateIntrinsicWithoutFolding(
427 ID: Intrinsic::memmove_element_unordered_atomic, OverloadTypes: Tys, Args: Ops);
428
429 // Set the alignment of the pointer args.
430 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: DstAlign));
431 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: SrcAlign));
432 CI->setAAMetadata(AAInfo);
433 return CI;
434}
435
436Value *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
437 Value *Ops[] = {Src};
438 Type *Tys[] = { Src->getType() };
439 return CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops);
440}
441
442Value *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
443 Value *Ops[] = {Acc, Src};
444 return CreateIntrinsic(ID: Intrinsic::vector_reduce_fadd, OverloadTypes: {Src->getType()}, Args: Ops);
445}
446
447Value *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
448 Value *Ops[] = {Acc, Src};
449 return CreateIntrinsic(ID: Intrinsic::vector_reduce_fmul, OverloadTypes: {Src->getType()}, Args: Ops);
450}
451
452Value *IRBuilderBase::CreateAddReduce(Value *Src) {
453 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_add, Src);
454}
455
456Value *IRBuilderBase::CreateMulReduce(Value *Src) {
457 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_mul, Src);
458}
459
460Value *IRBuilderBase::CreateAndReduce(Value *Src) {
461 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_and, Src);
462}
463
464Value *IRBuilderBase::CreateOrReduce(Value *Src) {
465 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_or, Src);
466}
467
468Value *IRBuilderBase::CreateXorReduce(Value *Src) {
469 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_xor, Src);
470}
471
472Value *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
473 auto ID =
474 IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
475 return getReductionIntrinsic(ID, Src);
476}
477
478Value *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
479 auto ID =
480 IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
481 return getReductionIntrinsic(ID, Src);
482}
483
484Value *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
485 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmax, Src);
486}
487
488Value *IRBuilderBase::CreateFPMinReduce(Value *Src) {
489 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmin, Src);
490}
491
492Value *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
493 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmaximum, Src);
494}
495
496Value *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
497 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fminimum, Src);
498}
499
500CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
501 assert(isa<PointerType>(Ptr->getType()) &&
502 "lifetime.start only applies to pointers.");
503 return CreateIntrinsicWithoutFolding(ID: Intrinsic::lifetime_start,
504 OverloadTypes: {Ptr->getType()}, Args: {Ptr});
505}
506
507CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
508 assert(isa<PointerType>(Ptr->getType()) &&
509 "lifetime.end only applies to pointers.");
510 return CreateIntrinsicWithoutFolding(ID: Intrinsic::lifetime_end,
511 OverloadTypes: {Ptr->getType()}, Args: {Ptr});
512}
513
514CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
515
516 assert(isa<PointerType>(Ptr->getType()) &&
517 "invariant.start only applies to pointers.");
518 if (!Size)
519 Size = getInt64(C: -1);
520 else
521 assert(Size->getType() == getInt64Ty() &&
522 "invariant.start requires the size to be an i64");
523
524 Value *Ops[] = {Size, Ptr};
525 // Fill in the single overloaded type: memory object type.
526 Type *ObjectPtr[1] = {Ptr->getType()};
527 return CreateIntrinsicWithoutFolding(ID: Intrinsic::invariant_start, OverloadTypes: ObjectPtr,
528 Args: Ops);
529}
530
531static MaybeAlign getAlign(Value *Ptr) {
532 if (auto *V = dyn_cast<GlobalVariable>(Val: Ptr))
533 return V->getAlign();
534 if (auto *A = dyn_cast<GlobalAlias>(Val: Ptr))
535 return getAlign(Ptr: A->getAliaseeObject());
536 return {};
537}
538
539CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
540 assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&
541 "threadlocal_address only applies to thread local variables.");
542 CallInst *CI = CreateIntrinsicWithoutFolding(
543 ID: llvm::Intrinsic::threadlocal_address, OverloadTypes: {Ptr->getType()}, Args: {Ptr});
544 if (MaybeAlign A = getAlign(Ptr)) {
545 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *A));
546 CI->addRetAttr(Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *A));
547 }
548 return CI;
549}
550
551CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
552 assert(Cond->getType() == getInt1Ty() &&
553 "an assumption condition must be of type i1");
554 return CreateIntrinsicWithoutFolding(ID: Intrinsic::assume, /*OverloadTypes=*/{},
555 Args: {Cond});
556}
557
558CallInst *
559IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
560 Value *Args[] = {ConstantInt::getTrue(Context&: getContext())};
561 return CreateIntrinsicWithoutFolding(
562 ID: Intrinsic::assume, /*OverloadTypes=*/{}, Args,
563 /*FMFSource=*/nullptr, /*Name=*/"", OpBundles);
564}
565
566Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
567 return CreateIntrinsicWithoutFolding(
568 ID: Intrinsic::experimental_noalias_scope_decl, OverloadTypes: {}, Args: {Scope});
569}
570
571/// Create a call to a Masked Load intrinsic.
572/// \p Ty - vector type to load
573/// \p Ptr - base pointer for the load
574/// \p Alignment - alignment of the source location
575/// \p Mask - vector of booleans which indicates what vector lanes should
576/// be accessed in memory
577/// \p PassThru - pass-through value that is used to fill the masked-off lanes
578/// of the result
579/// \p Name - name of the result variable
580CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
581 Value *Mask, Value *PassThru,
582 const Twine &Name) {
583 auto *PtrTy = cast<PointerType>(Val: Ptr->getType());
584 assert(Ty->isVectorTy() && "Type should be vector");
585 assert(Mask && "Mask should not be all-ones (null)");
586 if (!PassThru)
587 PassThru = PoisonValue::get(T: Ty);
588 Type *OverloadedTypes[] = { Ty, PtrTy };
589 Value *Ops[] = {Ptr, Mask, PassThru};
590 CallInst *CI =
591 CreateMaskedIntrinsic(Id: Intrinsic::masked_load, Ops, OverloadedTypes, Name);
592 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
593 return CI;
594}
595
596/// Create a call to a Masked Store intrinsic.
597/// \p Val - data to be stored,
598/// \p Ptr - base pointer for the store
599/// \p Alignment - alignment of the destination location
600/// \p Mask - vector of booleans which indicates what vector lanes should
601/// be accessed in memory
602CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
603 Align Alignment, Value *Mask) {
604 auto *PtrTy = cast<PointerType>(Val: Ptr->getType());
605 Type *DataTy = Val->getType();
606 assert(DataTy->isVectorTy() && "Val should be a vector");
607 assert(Mask && "Mask should not be all-ones (null)");
608 Type *OverloadedTypes[] = { DataTy, PtrTy };
609 Value *Ops[] = {Val, Ptr, Mask};
610 CallInst *CI =
611 CreateMaskedIntrinsic(Id: Intrinsic::masked_store, Ops, OverloadedTypes);
612 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
613 return CI;
614}
615
616/// Create a call to a Masked intrinsic, with given intrinsic Id,
617/// an array of operands - Ops, and an array of overloaded types -
618/// OverloadedTypes.
619CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
620 ArrayRef<Value *> Ops,
621 ArrayRef<Type *> OverloadedTypes,
622 const Twine &Name) {
623 return CreateIntrinsicWithoutFolding(ID: Id, OverloadTypes: OverloadedTypes, Args: Ops, FMFSource: {}, Name);
624}
625
626/// Create a call to a Masked Gather intrinsic.
627/// \p Ty - vector type to gather
628/// \p Ptrs - vector of pointers for loading
629/// \p Align - alignment for one element
630/// \p Mask - vector of booleans which indicates what vector lanes should
631/// be accessed in memory
632/// \p PassThru - pass-through value that is used to fill the masked-off lanes
633/// of the result
634/// \p Name - name of the result variable
635CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
636 Align Alignment, Value *Mask,
637 Value *PassThru,
638 const Twine &Name) {
639 auto *VecTy = cast<VectorType>(Val: Ty);
640 ElementCount NumElts = VecTy->getElementCount();
641 auto *PtrsTy = cast<VectorType>(Val: Ptrs->getType());
642 assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
643
644 if (!Mask)
645 Mask = getAllOnesMask(NumElts);
646
647 if (!PassThru)
648 PassThru = PoisonValue::get(T: Ty);
649
650 Type *OverloadedTypes[] = {Ty, PtrsTy};
651 Value *Ops[] = {Ptrs, Mask, PassThru};
652
653 // We specify only one type when we create this intrinsic. Types of other
654 // arguments are derived from this type.
655 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_gather, Ops,
656 OverloadedTypes, Name);
657 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
658 return CI;
659}
660
661/// Create a call to a Masked Scatter intrinsic.
662/// \p Data - data to be stored,
663/// \p Ptrs - the vector of pointers, where the \p Data elements should be
664/// stored
665/// \p Align - alignment for one element
666/// \p Mask - vector of booleans which indicates what vector lanes should
667/// be accessed in memory
668CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
669 Align Alignment, Value *Mask) {
670 auto *PtrsTy = cast<VectorType>(Val: Ptrs->getType());
671 auto *DataTy = cast<VectorType>(Val: Data->getType());
672 ElementCount NumElts = PtrsTy->getElementCount();
673
674 if (!Mask)
675 Mask = getAllOnesMask(NumElts);
676
677 Type *OverloadedTypes[] = {DataTy, PtrsTy};
678 Value *Ops[] = {Data, Ptrs, Mask};
679
680 // We specify only one type when we create this intrinsic. Types of other
681 // arguments are derived from this type.
682 CallInst *CI =
683 CreateMaskedIntrinsic(Id: Intrinsic::masked_scatter, Ops, OverloadedTypes);
684 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
685 return CI;
686}
687
688/// Create a call to Masked Expand Load intrinsic
689/// \p Ty - vector type to load
690/// \p Ptr - base pointer for the load
691/// \p Align - alignment of \p Ptr
692/// \p Mask - vector of booleans which indicates what vector lanes should
693/// be accessed in memory
694/// \p PassThru - pass-through value that is used to fill the masked-off lanes
695/// of the result
696/// \p Name - name of the result variable
697CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
698 MaybeAlign Align, Value *Mask,
699 Value *PassThru,
700 const Twine &Name) {
701 assert(Ty->isVectorTy() && "Type should be vector");
702 assert(Mask && "Mask should not be all-ones (null)");
703 if (!PassThru)
704 PassThru = PoisonValue::get(T: Ty);
705 Type *PtrTy = Ptr->getType();
706 Type *OverloadedTypes[] = {Ty, PtrTy};
707 Value *Ops[] = {Ptr, Mask, PassThru};
708 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_expandload, Ops,
709 OverloadedTypes, Name);
710 if (Align)
711 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *Align));
712 return CI;
713}
714
715/// Create a call to Masked Compress Store intrinsic
716/// \p Val - data to be stored,
717/// \p Ptr - base pointer for the store
718/// \p Align - alignment of \p Ptr
719/// \p Mask - vector of booleans which indicates what vector lanes should
720/// be accessed in memory
721CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
722 MaybeAlign Align,
723 Value *Mask) {
724 Type *DataTy = Val->getType();
725 assert(DataTy->isVectorTy() && "Val should be a vector");
726 assert(Mask && "Mask should not be all-ones (null)");
727 Type *PtrTy = Ptr->getType();
728 Type *OverloadedTypes[] = {DataTy, PtrTy};
729 Value *Ops[] = {Val, Ptr, Mask};
730 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_compressstore, Ops,
731 OverloadedTypes);
732 if (Align)
733 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *Align));
734 return CI;
735}
736
737template <typename T0>
738static std::vector<Value *>
739getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
740 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
741 std::vector<Value *> Args;
742 Args.push_back(x: B.getInt64(C: ID));
743 Args.push_back(x: B.getInt32(C: NumPatchBytes));
744 Args.push_back(x: ActualCallee);
745 Args.push_back(B.getInt32(C: CallArgs.size()));
746 Args.push_back(x: B.getInt32(C: Flags));
747 llvm::append_range(Args, CallArgs);
748 // GC Transition and Deopt args are now always handled via operand bundle.
749 // They will be removed from the signature of gc.statepoint shortly.
750 Args.push_back(x: B.getInt32(C: 0));
751 Args.push_back(x: B.getInt32(C: 0));
752 // GC args are now encoded in the gc-live operand bundle
753 return Args;
754}
755
756template<typename T1, typename T2, typename T3>
757static std::vector<OperandBundleDef>
758getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
759 std::optional<ArrayRef<T2>> DeoptArgs,
760 ArrayRef<T3> GCArgs) {
761 std::vector<OperandBundleDef> Rval;
762 if (DeoptArgs)
763 Rval.emplace_back(args: "deopt", args: SmallVector<Value *, 16>(*DeoptArgs));
764 if (TransitionArgs)
765 Rval.emplace_back(args: "gc-transition",
766 args: SmallVector<Value *, 16>(*TransitionArgs));
767 if (GCArgs.size())
768 Rval.emplace_back(args: "gc-live", args: SmallVector<Value *, 16>(GCArgs));
769 return Rval;
770}
771
772template <typename T0, typename T1, typename T2, typename T3>
773static CallInst *CreateGCStatepointCallCommon(
774 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
775 FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
776 std::optional<ArrayRef<T1>> TransitionArgs,
777 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
778 const Twine &Name) {
779 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
780 // Fill in the one generic type'd argument (the function is also vararg)
781 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
782 M, id: Intrinsic::experimental_gc_statepoint,
783 OverloadTys: {ActualCallee.getCallee()->getType()});
784
785 std::vector<Value *> Args = getStatepointArgs(
786 *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
787
788 CallInst *CI = Builder->CreateCall(
789 FnStatepoint, Args,
790 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
791 CI->addParamAttr(ArgNo: 2,
792 Attr: Attribute::get(Context&: Builder->getContext(), Kind: Attribute::ElementType,
793 Ty: ActualCallee.getFunctionType()));
794 return CI;
795}
796
797CallInst *IRBuilderBase::CreateGCStatepointCall(
798 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
799 ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
800 ArrayRef<Value *> GCArgs, const Twine &Name) {
801 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
802 Builder: this, ID, NumPatchBytes, ActualCallee, Flags: uint32_t(StatepointFlags::None),
803 CallArgs, TransitionArgs: std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
804}
805
806CallInst *IRBuilderBase::CreateGCStatepointCall(
807 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
808 uint32_t Flags, ArrayRef<Value *> CallArgs,
809 std::optional<ArrayRef<Use>> TransitionArgs,
810 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
811 const Twine &Name) {
812 return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
813 Builder: this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
814 DeoptArgs, GCArgs, Name);
815}
816
817CallInst *IRBuilderBase::CreateGCStatepointCall(
818 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
819 ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
820 ArrayRef<Value *> GCArgs, const Twine &Name) {
821 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
822 Builder: this, ID, NumPatchBytes, ActualCallee, Flags: uint32_t(StatepointFlags::None),
823 CallArgs, TransitionArgs: std::nullopt, DeoptArgs, GCArgs, Name);
824}
825
826template <typename T0, typename T1, typename T2, typename T3>
827static InvokeInst *CreateGCStatepointInvokeCommon(
828 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
829 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
830 BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
831 std::optional<ArrayRef<T1>> TransitionArgs,
832 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
833 const Twine &Name) {
834 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
835 // Fill in the one generic type'd argument (the function is also vararg)
836 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
837 M, id: Intrinsic::experimental_gc_statepoint,
838 OverloadTys: {ActualInvokee.getCallee()->getType()});
839
840 std::vector<Value *> Args =
841 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
842 Flags, InvokeArgs);
843
844 InvokeInst *II = Builder->CreateInvoke(
845 FnStatepoint, NormalDest, UnwindDest, Args,
846 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
847 II->addParamAttr(ArgNo: 2,
848 Attr: Attribute::get(Context&: Builder->getContext(), Kind: Attribute::ElementType,
849 Ty: ActualInvokee.getFunctionType()));
850 return II;
851}
852
853InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
854 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
855 BasicBlock *NormalDest, BasicBlock *UnwindDest,
856 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
857 ArrayRef<Value *> GCArgs, const Twine &Name) {
858 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
859 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
860 Flags: uint32_t(StatepointFlags::None), InvokeArgs,
861 TransitionArgs: std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
862}
863
864InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
865 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
866 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
867 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
868 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
869 const Twine &Name) {
870 return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
871 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
872 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
873}
874
875InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
876 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
877 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
878 std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
879 const Twine &Name) {
880 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
881 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
882 Flags: uint32_t(StatepointFlags::None), InvokeArgs, TransitionArgs: std::nullopt, DeoptArgs,
883 GCArgs, Name);
884}
885
886CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
887 Type *ResultType, const Twine &Name) {
888 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
889 Type *Types[] = {ResultType};
890
891 Value *Args[] = {Statepoint};
892 return CreateIntrinsicWithoutFolding(ID, OverloadTypes: Types, Args, FMFSource: {}, Name);
893}
894
895CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
896 int BaseOffset, int DerivedOffset,
897 Type *ResultType, const Twine &Name) {
898 Type *Types[] = {ResultType};
899
900 Value *Args[] = {Statepoint, getInt32(C: BaseOffset), getInt32(C: DerivedOffset)};
901 return CreateIntrinsicWithoutFolding(ID: Intrinsic::experimental_gc_relocate,
902 OverloadTypes: Types, Args, FMFSource: {}, Name);
903}
904
905CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
906 const Twine &Name) {
907 Type *PtrTy = DerivedPtr->getType();
908 return CreateIntrinsicWithoutFolding(
909 ID: Intrinsic::experimental_gc_get_pointer_base, OverloadTypes: {PtrTy, PtrTy}, Args: {DerivedPtr},
910 FMFSource: {}, Name);
911}
912
913CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
914 const Twine &Name) {
915 Type *PtrTy = DerivedPtr->getType();
916 return CreateIntrinsicWithoutFolding(
917 ID: Intrinsic::experimental_gc_get_pointer_offset, OverloadTypes: {PtrTy}, Args: {DerivedPtr}, FMFSource: {},
918 Name);
919}
920
921Value *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op,
922 FMFSource FMFSource,
923 const Twine &Name) {
924 Module *M = BB->getModule();
925 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: Op->getType());
926 if (Value *V =
927 Folder.FoldIntrinsic(ID, Ops: Op, Ty: Fn->getReturnType(), FMF: FMFSource.get(Default: FMF),
928 CtxF: GetInsertBlock()->getParent()))
929 return V;
930 return createCallHelper(Callee: Fn, Ops: Op, Name, FMFSource);
931}
932
933Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
934 Value *RHS, FMFSource FMFSource,
935 const Twine &Name) {
936 Module *M = BB->getModule();
937 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: {LHS->getType()});
938 if (Value *V = Folder.FoldIntrinsic(ID, Ops: {LHS, RHS}, Ty: Fn->getReturnType(),
939 FMF: FMFSource.get(Default: FMF),
940 CtxF: GetInsertBlock()->getParent()))
941 return V;
942 return createCallHelper(Callee: Fn, Ops: {LHS, RHS}, Name, FMFSource);
943}
944
945CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(
946 Intrinsic::ID ID, ArrayRef<Type *> OverloadTypes, ArrayRef<Value *> Args,
947 FMFSource FMFSource, const Twine &Name,
948 ArrayRef<OperandBundleDef> OpBundles) {
949 Module *M = BB->getModule();
950 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: OverloadTypes);
951 return createCallHelper(Callee: Fn, Ops: Args, Name, FMFSource, OpBundles);
952}
953
954CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(Type *RetTy,
955 Intrinsic::ID ID,
956 ArrayRef<Value *> Args,
957 FMFSource FMFSource,
958 const Twine &Name) {
959 Module *M = BB->getModule();
960 SmallVector<Type *> ArgTys = llvm::map_to_vector(C&: Args, F: &Value::getType);
961 Function *Fn = Intrinsic::getOrInsertDeclaration(M, IID: ID, RetTy, ArgTys);
962 return createCallHelper(Callee: Fn, Ops: Args, Name, FMFSource);
963}
964
965Value *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
966 ArrayRef<Type *> OverloadTypes,
967 ArrayRef<Value *> Args,
968 FMFSource FMFSource, const Twine &Name,
969 ArrayRef<OperandBundleDef> OpBundles,
970 function_ref<void(CallInst *)> SetFn) {
971 Type *RetTy = Intrinsic::getType(Context, id: ID, OverloadTys: OverloadTypes)->getReturnType();
972 if (Value *V = Folder.FoldIntrinsic(ID, Ops: Args, Ty: RetTy, FMF: FMFSource.get(Default: FMF),
973 CtxF: GetInsertBlock()->getParent()))
974 return V;
975 CallInst *CI = CreateIntrinsicWithoutFolding(ID, OverloadTypes, Args,
976 FMFSource, Name, OpBundles);
977 SetFn(CI);
978 return CI;
979}
980
981Value *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
982 ArrayRef<Value *> Args,
983 FMFSource FMFSource, const Twine &Name,
984 function_ref<void(CallInst *)> SetFn) {
985 if (Value *V = Folder.FoldIntrinsic(ID, Ops: Args, Ty: RetTy, FMF: FMFSource.get(Default: FMF),
986 CtxF: GetInsertBlock()->getParent()))
987 return V;
988 CallInst *CI =
989 CreateIntrinsicWithoutFolding(RetTy, ID, Args, FMFSource, Name);
990 SetFn(CI);
991 return CI;
992}
993
994CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
995 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
996 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
997 std::optional<fp::ExceptionBehavior> Except) {
998 Value *RoundingV = getConstrainedFPRounding(Rounding);
999 Value *ExceptV = getConstrainedFPExcept(Except);
1000
1001 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1002 CallInst *C = CreateIntrinsicWithoutFolding(
1003 ID, OverloadTypes: {L->getType()}, Args: {L, R, RoundingV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1004 setConstrainedFPCallAttr(C);
1005 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1006 return C;
1007}
1008
1009CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
1010 Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
1011 FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag,
1012 std::optional<RoundingMode> Rounding,
1013 std::optional<fp::ExceptionBehavior> Except) {
1014 Value *RoundingV = getConstrainedFPRounding(Rounding);
1015 Value *ExceptV = getConstrainedFPExcept(Except);
1016
1017 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1018
1019 llvm::SmallVector<Value *, 5> ExtArgs(Args);
1020 ExtArgs.push_back(Elt: RoundingV);
1021 ExtArgs.push_back(Elt: ExceptV);
1022 CallInst *C =
1023 CreateIntrinsicWithoutFolding(ID, OverloadTypes: Types, Args: ExtArgs, FMFSource: nullptr, Name, OpBundles: {});
1024 setConstrainedFPCallAttr(C);
1025 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1026 return C;
1027}
1028
1029CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
1030 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
1031 const Twine &Name, MDNode *FPMathTag,
1032 std::optional<fp::ExceptionBehavior> Except) {
1033 Value *ExceptV = getConstrainedFPExcept(Except);
1034
1035 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1036 CallInst *C = CreateIntrinsicWithoutFolding(
1037 ID, OverloadTypes: {L->getType()}, Args: {L, R, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1038 setConstrainedFPCallAttr(C);
1039 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1040 return C;
1041}
1042
1043Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1044 const Twine &Name, MDNode *FPMathTag) {
1045 if (Instruction::isBinaryOp(Opcode: Opc)) {
1046 assert(Ops.size() == 2 && "Invalid number of operands!");
1047 return CreateBinOp(Opc: static_cast<Instruction::BinaryOps>(Opc),
1048 LHS: Ops[0], RHS: Ops[1], Name, FPMathTag);
1049 }
1050 if (Instruction::isUnaryOp(Opcode: Opc)) {
1051 assert(Ops.size() == 1 && "Invalid number of operands!");
1052 return CreateUnOp(Opc: static_cast<Instruction::UnaryOps>(Opc),
1053 V: Ops[0], Name, FPMathTag);
1054 }
1055 llvm_unreachable("Unexpected opcode!");
1056}
1057
1058CallInst *IRBuilderBase::CreateConstrainedFPCast(
1059 Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource,
1060 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
1061 std::optional<fp::ExceptionBehavior> Except) {
1062 Value *ExceptV = getConstrainedFPExcept(Except);
1063
1064 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1065
1066 CallInst *C;
1067 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: ID)) {
1068 Value *RoundingV = getConstrainedFPRounding(Rounding);
1069 C = CreateIntrinsicWithoutFolding(
1070 ID, OverloadTypes: {DestTy, V->getType()}, Args: {V, RoundingV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1071 } else
1072 C = CreateIntrinsicWithoutFolding(ID, OverloadTypes: {DestTy, V->getType()}, Args: {V, ExceptV},
1073 FMFSource: nullptr, Name, OpBundles: {});
1074 setConstrainedFPCallAttr(C);
1075
1076 if (isa<FPMathOperator>(Val: C))
1077 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1078 return C;
1079}
1080
1081Value *IRBuilderBase::CreateFCmpHelper(CmpInst::Predicate P, Value *LHS,
1082 Value *RHS, const Twine &Name,
1083 MDNode *FPMathTag, FMFSource FMFSource,
1084 bool IsSignaling) {
1085 if (IsFPConstrained) {
1086 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
1087 : Intrinsic::experimental_constrained_fcmp;
1088 return CreateConstrainedFPCmp(ID, P, L: LHS, R: RHS, Name);
1089 }
1090
1091 if (auto *V = Folder.FoldCmp(P, LHS, RHS))
1092 return V;
1093 return Insert(
1094 I: setFPAttrs(I: new FCmpInst(P, LHS, RHS), FPMD: FPMathTag, FMF: FMFSource.get(Default: FMF)),
1095 Name);
1096}
1097
1098CallInst *IRBuilderBase::CreateConstrainedFPCmp(
1099 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
1100 const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
1101 Value *PredicateV = getConstrainedFPPredicate(Predicate: P);
1102 Value *ExceptV = getConstrainedFPExcept(Except);
1103
1104 CallInst *C = CreateIntrinsicWithoutFolding(
1105 ID, OverloadTypes: {L->getType()}, Args: {L, R, PredicateV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1106 setConstrainedFPCallAttr(C);
1107 return C;
1108}
1109
1110CallInst *IRBuilderBase::CreateConstrainedFPCall(
1111 Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
1112 std::optional<RoundingMode> Rounding,
1113 std::optional<fp::ExceptionBehavior> Except) {
1114 llvm::SmallVector<Value *, 6> UseArgs(Args);
1115
1116 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: Callee->getIntrinsicID()))
1117 UseArgs.push_back(Elt: getConstrainedFPRounding(Rounding));
1118 UseArgs.push_back(Elt: getConstrainedFPExcept(Except));
1119
1120 CallInst *C = CreateCall(Callee, Args: UseArgs, Name);
1121 setConstrainedFPCallAttr(C);
1122 return C;
1123}
1124
1125Value *IRBuilderBase::CreateSelectWithUnknownProfile(Value *C, Value *True,
1126 Value *False,
1127 StringRef PassName,
1128 const Twine &Name) {
1129 Value *Ret = CreateSelectFMF(C, True, False, FMFSource: {}, Name);
1130 if (auto *SI = dyn_cast<SelectInst>(Val: Ret)) {
1131 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, PassName);
1132 }
1133 return Ret;
1134}
1135
1136Value *IRBuilderBase::CreateSelectFMFWithUnknownProfile(Value *C, Value *True,
1137 Value *False,
1138 FMFSource FMFSource,
1139 StringRef PassName,
1140 const Twine &Name) {
1141 Value *Ret = CreateSelectFMF(C, True, False, FMFSource, Name);
1142 if (auto *SI = dyn_cast<SelectInst>(Val: Ret))
1143 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, PassName);
1144 return Ret;
1145}
1146
1147Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
1148 const Twine &Name, Instruction *MDFrom) {
1149 return CreateSelectFMF(C, True, False, FMFSource: {}, Name, MDFrom);
1150}
1151
1152Value *IRBuilderBase::CreateSelectFMF(Value *C, Value *True, Value *False,
1153 FMFSource FMFSource, const Twine &Name,
1154 Instruction *MDFrom) {
1155 if (auto *V = Folder.FoldSelect(C, True, False, FMF: FMFSource.get(Default: FMF)))
1156 return V;
1157
1158 SelectInst *Sel = SelectInst::Create(C, S1: True, S2: False);
1159 if (MDFrom) {
1160 MDNode *Prof = MDFrom->getMetadata(KindID: LLVMContext::MD_prof);
1161 MDNode *Unpred = MDFrom->getMetadata(KindID: LLVMContext::MD_unpredictable);
1162 Sel = addBranchMetadata(I: Sel, Weights: Prof, Unpredictable: Unpred);
1163 }
1164 if (isa<FPMathOperator>(Val: Sel))
1165 setFPAttrs(I: Sel, /*MDNode=*/FPMD: nullptr, FMF: FMFSource.get(Default: FMF));
1166 return Insert(I: Sel, Name);
1167}
1168
1169Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name,
1170 bool IsNUW) {
1171 assert(LHS->getType() == RHS->getType() &&
1172 "Pointer subtraction operand types must match!");
1173 Value *LHSAddr = CreatePtrToAddr(V: LHS);
1174 Value *RHSAddr = CreatePtrToAddr(V: RHS);
1175 return CreateSub(LHS: LHSAddr, RHS: RHSAddr, Name, HasNUW: IsNUW);
1176}
1177Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
1178 const Twine &Name) {
1179 const DataLayout &DL = BB->getDataLayout();
1180 TypeSize ElemSize = DL.getTypeAllocSize(Ty: ElemTy);
1181 if (ElemSize == TypeSize::getFixed(ExactSize: 1))
1182 return CreatePtrDiff(LHS, RHS, Name);
1183
1184 Value *Diff = CreatePtrDiff(LHS, RHS);
1185 return CreateExactSDiv(LHS: Diff, RHS: CreateTypeSize(Ty: Diff->getType(), Size: ElemSize), Name);
1186}
1187
1188Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
1189 assert(isa<PointerType>(Ptr->getType()) &&
1190 "launder.invariant.group only applies to pointers.");
1191 auto *PtrType = Ptr->getType();
1192 Module *M = BB->getParent()->getParent();
1193 Function *FnLaunderInvariantGroup = Intrinsic::getOrInsertDeclaration(
1194 M, id: Intrinsic::launder_invariant_group, OverloadTys: {PtrType});
1195
1196 assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&
1197 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1198 PtrType &&
1199 "LaunderInvariantGroup should take and return the same type");
1200
1201 return CreateCall(Callee: FnLaunderInvariantGroup, Args: {Ptr});
1202}
1203
1204Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
1205 assert(isa<PointerType>(Ptr->getType()) &&
1206 "strip.invariant.group only applies to pointers.");
1207
1208 auto *PtrType = Ptr->getType();
1209 Module *M = BB->getParent()->getParent();
1210 Function *FnStripInvariantGroup = Intrinsic::getOrInsertDeclaration(
1211 M, id: Intrinsic::strip_invariant_group, OverloadTys: {PtrType});
1212
1213 assert(FnStripInvariantGroup->getReturnType() == PtrType &&
1214 FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1215 PtrType &&
1216 "StripInvariantGroup should take and return the same type");
1217
1218 return CreateCall(Callee: FnStripInvariantGroup, Args: {Ptr});
1219}
1220
1221Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1222 auto *Ty = cast<VectorType>(Val: V->getType());
1223 if (isa<ScalableVectorType>(Val: Ty)) {
1224 Module *M = BB->getParent()->getParent();
1225 Function *F =
1226 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::vector_reverse, OverloadTys: Ty);
1227 return Insert(I: CallInst::Create(Func: F, Args: V), Name);
1228 }
1229 // Keep the original behaviour for fixed vector
1230 SmallVector<int, 8> ShuffleMask;
1231 int NumElts = Ty->getElementCount().getKnownMinValue();
1232 for (int i = 0; i < NumElts; ++i)
1233 ShuffleMask.push_back(Elt: NumElts - i - 1);
1234 return CreateShuffleVector(V, Mask: ShuffleMask, Name);
1235}
1236
1237static SmallVector<int, 8> getSpliceMask(int64_t Imm, unsigned NumElts) {
1238 unsigned Idx = (NumElts + Imm) % NumElts;
1239 SmallVector<int, 8> Mask;
1240 for (unsigned I = 0; I < NumElts; ++I)
1241 Mask.push_back(Elt: Idx + I);
1242 return Mask;
1243}
1244
1245Value *IRBuilderBase::CreateVectorSpliceLeft(Value *V1, Value *V2,
1246 Value *Offset, const Twine &Name) {
1247 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1248 assert(V1->getType() == V2->getType() &&
1249 "Splice expects matching operand types!");
1250
1251 // Emit a shufflevector for fixed vectors with a constant offset
1252 if (auto *COffset = dyn_cast<ConstantInt>(Val: Offset))
1253 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType()))
1254 return CreateShuffleVector(
1255 V1, V2,
1256 Mask: getSpliceMask(Imm: COffset->getZExtValue(), NumElts: FVTy->getNumElements()));
1257
1258 return CreateIntrinsic(ID: Intrinsic::vector_splice_left, OverloadTypes: V1->getType(),
1259 Args: {V1, V2, Offset}, FMFSource: {}, Name);
1260}
1261
1262Value *IRBuilderBase::CreateVectorSpliceRight(Value *V1, Value *V2,
1263 Value *Offset,
1264 const Twine &Name) {
1265 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1266 assert(V1->getType() == V2->getType() &&
1267 "Splice expects matching operand types!");
1268
1269 // Emit a shufflevector for fixed vectors with a constant offset
1270 if (auto *COffset = dyn_cast<ConstantInt>(Val: Offset))
1271 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType()))
1272 return CreateShuffleVector(
1273 V1, V2,
1274 Mask: getSpliceMask(Imm: -COffset->getZExtValue(), NumElts: FVTy->getNumElements()));
1275
1276 return CreateIntrinsic(ID: Intrinsic::vector_splice_right, OverloadTypes: V1->getType(),
1277 Args: {V1, V2, Offset}, FMFSource: {}, Name);
1278}
1279
1280Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1281 const Twine &Name) {
1282 auto EC = ElementCount::getFixed(MinVal: NumElts);
1283 return CreateVectorSplat(EC, V, Name);
1284}
1285
1286Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1287 const Twine &Name) {
1288 assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1289
1290 // First insert it into a poison vector so we can shuffle it.
1291 Value *Poison = PoisonValue::get(T: VectorType::get(ElementType: V->getType(), EC));
1292 V = CreateInsertElement(Vec: Poison, NewElt: V, Idx: getInt64(C: 0), Name: Name + ".splatinsert");
1293
1294 // Shuffle the value across the desired number of elements.
1295 SmallVector<int, 16> Zeros;
1296 Zeros.resize(N: EC.getKnownMinValue());
1297 return CreateShuffleVector(V, Mask: Zeros, Name: Name + ".splat");
1298}
1299
1300Value *IRBuilderBase::CreateVectorInterleave(ArrayRef<Value *> Ops,
1301 const Twine &Name) {
1302 assert(Ops.size() >= 2 && Ops.size() <= 8 &&
1303 "Unexpected number of operands to interleave");
1304
1305 // Make sure all operands are the same type.
1306 assert(isa<VectorType>(Ops[0]->getType()) && "Unexpected type");
1307
1308#ifndef NDEBUG
1309 for (unsigned I = 1; I < Ops.size(); I++) {
1310 assert(Ops[I]->getType() == Ops[0]->getType() &&
1311 "Vector interleave expects matching operand types!");
1312 }
1313#endif
1314
1315 unsigned IID = Intrinsic::getInterleaveIntrinsicID(Factor: Ops.size());
1316 auto *SubvecTy = cast<VectorType>(Val: Ops[0]->getType());
1317 Type *DestTy = VectorType::get(ElementType: SubvecTy->getElementType(),
1318 EC: SubvecTy->getElementCount() * Ops.size());
1319 return CreateIntrinsic(ID: IID, OverloadTypes: {DestTy}, Args: Ops, FMFSource: {}, Name);
1320}
1321
1322Value *IRBuilderBase::CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
1323 unsigned Dimension,
1324 unsigned LastIndex,
1325 MDNode *DbgInfo) {
1326 auto *BaseType = Base->getType();
1327 assert(isa<PointerType>(BaseType) &&
1328 "Invalid Base ptr type for preserve.array.access.index.");
1329
1330 Value *LastIndexV = getInt32(C: LastIndex);
1331 Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 0);
1332 SmallVector<Value *, 4> IdxList(Dimension, Zero);
1333 IdxList.push_back(Elt: LastIndexV);
1334
1335 Type *ResultType = GetElementPtrInst::getGEPReturnType(Ptr: Base, IdxList);
1336
1337 Value *DimV = getInt32(C: Dimension);
1338 CallInst *Fn = CreateIntrinsicWithoutFolding(
1339 ID: Intrinsic::preserve_array_access_index, OverloadTypes: {ResultType, BaseType},
1340 Args: {Base, DimV, LastIndexV});
1341 Fn->addParamAttr(
1342 ArgNo: 0, Attr: Attribute::get(Context&: Fn->getContext(), Kind: Attribute::ElementType, Ty: ElTy));
1343 if (DbgInfo)
1344 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1345
1346 return Fn;
1347}
1348
1349Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1350 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1351 assert(isa<PointerType>(Base->getType()) &&
1352 "Invalid Base ptr type for preserve.union.access.index.");
1353 auto *BaseType = Base->getType();
1354
1355 Value *DIIndex = getInt32(C: FieldIndex);
1356 CallInst *Fn =
1357 CreateIntrinsicWithoutFolding(ID: Intrinsic::preserve_union_access_index,
1358 OverloadTypes: {BaseType, BaseType}, Args: {Base, DIIndex});
1359 if (DbgInfo)
1360 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1361
1362 return Fn;
1363}
1364
1365Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1366 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1367 MDNode *DbgInfo) {
1368 auto *BaseType = Base->getType();
1369 assert(isa<PointerType>(BaseType) &&
1370 "Invalid Base ptr type for preserve.struct.access.index.");
1371
1372 Value *GEPIndex = getInt32(C: Index);
1373 Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 0);
1374 Type *ResultType =
1375 GetElementPtrInst::getGEPReturnType(Ptr: Base, IdxList: {Zero, GEPIndex});
1376
1377 Value *DIIndex = getInt32(C: FieldIndex);
1378 CallInst *Fn = CreateIntrinsicWithoutFolding(
1379 ID: Intrinsic::preserve_struct_access_index, OverloadTypes: {ResultType, BaseType},
1380 Args: {Base, GEPIndex, DIIndex});
1381 Fn->addParamAttr(
1382 ArgNo: 0, Attr: Attribute::get(Context&: Fn->getContext(), Kind: Attribute::ElementType, Ty: ElTy));
1383 if (DbgInfo)
1384 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1385
1386 return Fn;
1387}
1388
1389Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
1390 ConstantInt *TestV = getInt32(C: Test);
1391 return CreateIntrinsic(ID: Intrinsic::is_fpclass, OverloadTypes: {FPNum->getType()},
1392 Args: {FPNum, TestV});
1393}
1394
1395CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1396 Value *PtrValue,
1397 Value *AlignValue,
1398 Value *OffsetValue) {
1399 SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1400 if (OffsetValue)
1401 Vals.push_back(Elt: OffsetValue);
1402 OperandBundleDefT<Value *> AlignOpB("align", Vals);
1403 return CreateAssumption(OpBundles: {AlignOpB});
1404}
1405
1406CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1407 Value *PtrValue,
1408 uint64_t Alignment,
1409 Value *OffsetValue) {
1410 assert(isa<PointerType>(PtrValue->getType()) &&
1411 "trying to create an alignment assumption on a non-pointer?");
1412 assert(Alignment != 0 && "Invalid Alignment");
1413 Value *AlignValue = ConstantInt::get(Ty: getInt64Ty(), V: Alignment);
1414 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1415}
1416
1417CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1418 Value *PtrValue,
1419 Value *Alignment,
1420 Value *OffsetValue) {
1421 assert(isa<PointerType>(PtrValue->getType()) &&
1422 "trying to create an alignment assumption on a non-pointer?");
1423 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue: Alignment, OffsetValue);
1424}
1425
1426CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
1427 Value *SizeValue) {
1428 assert(isa<PointerType>(PtrValue->getType()) &&
1429 "trying to create a deferenceable assumption on a non-pointer?");
1430 SmallVector<Value *, 4> Vals({PtrValue, SizeValue});
1431 OperandBundleDefT<Value *> DereferenceableOpB("dereferenceable", Vals);
1432 return CreateAssumption(OpBundles: {DereferenceableOpB});
1433}
1434
1435CallInst *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
1436 assert(isa<PointerType>(PtrValue->getType()) &&
1437 "trying to create a nonnull assumption on a non-pointer?");
1438 return CreateAssumption(OpBundles: OperandBundleDef("nonnull", PtrValue));
1439}
1440
1441IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default;
1442IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default;
1443IRBuilderFolder::~IRBuilderFolder() = default;
1444void ConstantFolder::anchor() {}
1445void NoFolder::anchor() {}
1446