1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallVectorExtras.h"
17#include "llvm/IR/Constant.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/DerivedTypes.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GlobalValue.h"
22#include "llvm/IR/GlobalVariable.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/NoFolder.h"
28#include "llvm/IR/Operator.h"
29#include "llvm/IR/ProfDataUtils.h"
30#include "llvm/IR/Statepoint.h"
31#include "llvm/IR/Type.h"
32#include "llvm/IR/Value.h"
33#include "llvm/Support/Casting.h"
34#include <cassert>
35#include <cstdint>
36#include <optional>
37#include <vector>
38
39using namespace llvm;
40
41/// CreateGlobalString - Make a new global variable with an initializer that
42/// has array of i8 type filled in with the nul terminated string value
43/// specified. If Name is specified, it is the name of the global variable
44/// created.
45GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
46 const Twine &Name,
47 unsigned AddressSpace,
48 Module *M, bool AddNull) {
49 Constant *StrConstant = ConstantDataArray::getString(Context, Initializer: Str, AddNull);
50 if (!M)
51 M = BB->getParent()->getParent();
52 auto *GV = new GlobalVariable(
53 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
54 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
55 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
56 GV->setAlignment(M->getDataLayout().getPrefTypeAlign(Ty: getInt8Ty()));
57 return GV;
58}
59
60Type *IRBuilderBase::getCurrentFunctionReturnType() const {
61 assert(BB && BB->getParent() && "No current function!");
62 return BB->getParent()->getReturnType();
63}
64
65DebugLoc IRBuilderBase::getCurrentDebugLocation() const { return StoredDL; }
66void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
67 // We prefer to set our current debug location if any has been set, but if
68 // our debug location is empty and I has a valid location, we shouldn't
69 // overwrite it.
70 I->setDebugLoc(StoredDL.orElse(Other: I->getDebugLoc()));
71}
72
73Value *IRBuilderBase::CreateAggregateCast(Value *V, Type *DestTy) {
74 Type *SrcTy = V->getType();
75 if (SrcTy == DestTy)
76 return V;
77
78 if (SrcTy->isAggregateType()) {
79 unsigned NumElements;
80 if (SrcTy->isStructTy()) {
81 assert(DestTy->isStructTy() && "Expected StructType");
82 assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements() &&
83 "Expected StructTypes with equal number of elements");
84 NumElements = SrcTy->getStructNumElements();
85 } else {
86 assert(SrcTy->isArrayTy() && DestTy->isArrayTy() && "Expected ArrayType");
87 assert(SrcTy->getArrayNumElements() == DestTy->getArrayNumElements() &&
88 "Expected ArrayTypes with equal number of elements");
89 NumElements = SrcTy->getArrayNumElements();
90 }
91
92 Value *Result = PoisonValue::get(T: DestTy);
93 for (unsigned I = 0; I < NumElements; ++I) {
94 Type *ElementTy = SrcTy->isStructTy() ? DestTy->getStructElementType(N: I)
95 : DestTy->getArrayElementType();
96 Value *Element =
97 CreateAggregateCast(V: CreateExtractValue(Agg: V, Idxs: ArrayRef(I)), DestTy: ElementTy);
98
99 Result = CreateInsertValue(Agg: Result, Val: Element, Idxs: ArrayRef(I));
100 }
101 return Result;
102 }
103
104 return CreateBitOrPointerCast(V, DestTy);
105}
106
107Value *IRBuilderBase::CreateBitPreservingCastChain(const DataLayout &DL,
108 Value *V, Type *NewTy) {
109 Type *OldTy = V->getType();
110
111 if (OldTy == NewTy)
112 return V;
113
114 assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
115 "Integer types must be the exact same to convert.");
116
117 // A variant of bitcast that supports a mixture of fixed and scalable types
118 // that are know to have the same size.
119 auto CreateBitCastLike = [this](Value *In, Type *Ty) -> Value * {
120 Type *InTy = In->getType();
121 if (InTy == Ty)
122 return In;
123
124 if (isa<FixedVectorType>(Val: InTy) && isa<ScalableVectorType>(Val: Ty)) {
125 // For vscale_range(2) expand <4 x i32> to <vscale x 4 x i16> -->
126 // <4 x i32> to <vscale x 2 x i32> to <vscale x 4 x i16>
127 auto *VTy = VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: Ty), EltTy: InTy);
128 return CreateBitCast(
129 V: CreateInsertVector(DstType: VTy, SrcVec: PoisonValue::get(T: VTy), SubVec: In, Idx: getInt64(C: 0)), DestTy: Ty);
130 }
131
132 if (isa<ScalableVectorType>(Val: InTy) && isa<FixedVectorType>(Val: Ty)) {
133 // For vscale_range(2) expand <vscale x 4 x i16> to <4 x i32> -->
134 // <vscale x 4 x i16> to <vscale x 2 x i32> to <4 x i32>
135 auto *VTy = VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: InTy), EltTy: Ty);
136 return CreateExtractVector(DstType: Ty, SrcVec: CreateBitCast(V: In, DestTy: VTy), Idx: getInt64(C: 0));
137 }
138
139 return CreateBitCast(V: In, DestTy: Ty);
140 };
141
142 // See if we need inttoptr for this type pair. May require additional bitcast.
143 if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
144 // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
145 // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
146 // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*>
147 // Directly handle i64 to i8*
148 return CreateIntToPtr(V: CreateBitCastLike(V, DL.getIntPtrType(NewTy)), DestTy: NewTy);
149 }
150
151 // See if we need ptrtoint for this type pair. May require additional bitcast.
152 if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) {
153 // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128
154 // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
155 // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32>
156 // Expand i8* to i64 --> i8* to i64 to i64
157 return CreateBitCastLike(CreatePtrToInt(V, DestTy: DL.getIntPtrType(OldTy)), NewTy);
158 }
159
160 if (OldTy->isPtrOrPtrVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
161 unsigned OldAS = OldTy->getPointerAddressSpace();
162 unsigned NewAS = NewTy->getPointerAddressSpace();
163 // To convert pointers with different address spaces (they are already
164 // checked convertible, i.e. they have the same pointer size), so far we
165 // cannot use `bitcast` (which has restrict on the same address space) or
166 // `addrspacecast` (which is not always no-op casting). Instead, use a pair
167 // of no-op `ptrtoint`/`inttoptr` casts through an integer with the same bit
168 // size.
169 if (OldAS != NewAS) {
170 return CreateIntToPtr(
171 V: CreateBitCastLike(CreatePtrToInt(V, DestTy: DL.getIntPtrType(OldTy)),
172 DL.getIntPtrType(NewTy)),
173 DestTy: NewTy);
174 }
175 }
176
177 return CreateBitCastLike(V, NewTy);
178}
179
180CallInst *
181IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
182 const Twine &Name, FMFSource FMFSource,
183 ArrayRef<OperandBundleDef> OpBundles) {
184 CallInst *CI = CreateCall(Callee, Args: Ops, OpBundles, Name);
185 if (isa<FPMathOperator>(Val: CI))
186 CI->setFastMathFlags(FMFSource.get(Default: FMF));
187 return CI;
188}
189
190static Value *CreateVScaleMultiple(IRBuilderBase &B, Type *Ty, uint64_t Scale) {
191 Value *VScale = B.CreateVScale(Ty);
192 if (Scale == 1)
193 return VScale;
194
195 return B.CreateNUWMul(LHS: VScale, RHS: ConstantInt::get(Ty, V: Scale));
196}
197
198Value *IRBuilderBase::CreateElementCount(Type *Ty, ElementCount EC) {
199 if (EC.isFixed() || EC.isZero())
200 return ConstantInt::get(Ty, V: EC.getKnownMinValue());
201
202 return CreateVScaleMultiple(B&: *this, Ty, Scale: EC.getKnownMinValue());
203}
204
205Value *IRBuilderBase::CreateTypeSize(Type *Ty, TypeSize Size) {
206 if (Size.isFixed() || Size.isZero())
207 return ConstantInt::get(Ty, V: Size.getKnownMinValue());
208
209 return CreateVScaleMultiple(B&: *this, Ty, Scale: Size.getKnownMinValue());
210}
211
212Value *IRBuilderBase::CreateAllocationSize(Type *DestTy, AllocaInst *AI) {
213 const DataLayout &DL = BB->getDataLayout();
214 TypeSize ElemSize = AI->getAllocationBaseSize(DL);
215 Value *Size = CreateTypeSize(Ty: DestTy, Size: ElemSize);
216 if (AI->isArrayAllocation())
217 Size = CreateMul(LHS: CreateZExtOrTrunc(V: AI->getArraySize(), DestTy), RHS: Size);
218 return Size;
219}
220
221Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
222 Type *STy = DstType->getScalarType();
223 if (isa<ScalableVectorType>(Val: DstType)) {
224 Type *StepVecType = DstType;
225 // TODO: We expect this special case (element type < 8 bits) to be
226 // temporary - once the intrinsic properly supports < 8 bits this code
227 // can be removed.
228 if (STy->getScalarSizeInBits() < 8)
229 StepVecType =
230 VectorType::get(ElementType: getInt8Ty(), Other: cast<ScalableVectorType>(Val: DstType));
231 Value *Res = CreateIntrinsic(ID: Intrinsic::stepvector, OverloadTypes: {StepVecType}, Args: {},
232 FMFSource: nullptr, Name);
233 if (StepVecType != DstType)
234 Res = CreateTrunc(V: Res, DestTy: DstType);
235 return Res;
236 }
237
238 unsigned NumEls = cast<FixedVectorType>(Val: DstType)->getNumElements();
239
240 // Create a vector of consecutive numbers from zero to VF.
241 // It's okay if the values wrap around.
242 SmallVector<Constant *, 8> Indices;
243 for (unsigned i = 0; i < NumEls; ++i)
244 Indices.push_back(
245 Elt: ConstantInt::get(Ty: STy, V: i, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
246
247 // Add the consecutive indices to the vector value.
248 return ConstantVector::get(V: Indices);
249}
250
251CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
252 MaybeAlign Align, bool isVolatile,
253 const AAMDNodes &AAInfo) {
254 Value *Ops[] = {Ptr, Val, Size, getInt1(V: isVolatile)};
255 Type *Tys[] = {Ptr->getType(), Size->getType()};
256
257 auto *CI = cast<MemSetInst>(
258 Val: CreateIntrinsicWithoutFolding(ID: Intrinsic::memset, OverloadTypes: Tys, Args: Ops));
259
260 if (Align)
261 CI->setDestAlignment(*Align);
262 CI->setAAMetadata(AAInfo);
263 return CI;
264}
265
266CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
267 Value *Val, Value *Size,
268 bool IsVolatile,
269 const AAMDNodes &AAInfo) {
270 Value *Ops[] = {Dst, Val, Size, getInt1(V: IsVolatile)};
271 Type *Tys[] = {Dst->getType(), Size->getType()};
272
273 auto *CI = cast<MemSetInst>(
274 Val: CreateIntrinsicWithoutFolding(ID: Intrinsic::memset_inline, OverloadTypes: Tys, Args: Ops));
275
276 if (DstAlign)
277 CI->setDestAlignment(*DstAlign);
278 CI->setAAMetadata(AAInfo);
279 return CI;
280}
281
282CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
283 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
284 const AAMDNodes &AAInfo) {
285
286 Value *Ops[] = {Ptr, Val, Size, getInt32(C: ElementSize)};
287 Type *Tys[] = {Ptr->getType(), Size->getType()};
288
289 auto *CI = cast<AnyMemSetInst>(Val: CreateIntrinsicWithoutFolding(
290 ID: Intrinsic::memset_element_unordered_atomic, OverloadTypes: Tys, Args: Ops));
291 CI->setDestAlignment(Alignment);
292 CI->setAAMetadata(AAInfo);
293 return CI;
294}
295
296CallInst *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
297 MaybeAlign DstAlign, Value *Src,
298 MaybeAlign SrcAlign, Value *Size,
299 bool isVolatile,
300 const AAMDNodes &AAInfo) {
301 assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||
302 IntrID == Intrinsic::memmove) &&
303 "Unexpected intrinsic ID");
304 Value *Ops[] = {Dst, Src, Size, getInt1(V: isVolatile)};
305 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
306
307 auto *MCI =
308 cast<MemTransferInst>(Val: CreateIntrinsicWithoutFolding(ID: IntrID, OverloadTypes: Tys, Args: Ops));
309
310 if (DstAlign)
311 MCI->setDestAlignment(*DstAlign);
312 if (SrcAlign)
313 MCI->setSourceAlignment(*SrcAlign);
314 MCI->setAAMetadata(AAInfo);
315 return MCI;
316}
317
318CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
319 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
320 uint32_t ElementSize, const AAMDNodes &AAInfo) {
321 assert(DstAlign >= ElementSize &&
322 "Pointer alignment must be at least element size");
323 assert(SrcAlign >= ElementSize &&
324 "Pointer alignment must be at least element size");
325 Value *Ops[] = {Dst, Src, Size, getInt32(C: ElementSize)};
326 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
327
328 auto *AMCI = cast<AnyMemCpyInst>(Val: CreateIntrinsicWithoutFolding(
329 ID: Intrinsic::memcpy_element_unordered_atomic, OverloadTypes: Tys, Args: Ops));
330
331 // Set the alignment of the pointer args.
332 AMCI->setDestAlignment(DstAlign);
333 AMCI->setSourceAlignment(SrcAlign);
334 AMCI->setAAMetadata(AAInfo);
335 return AMCI;
336}
337
338/// isConstantOne - Return true only if val is constant int 1
339static bool isConstantOne(const Value *Val) {
340 assert(Val && "isConstantOne does not work with nullptr Val");
341 const ConstantInt *CVal = dyn_cast<ConstantInt>(Val);
342 return CVal && CVal->isOne();
343}
344
345CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,
346 Value *AllocSize, Value *ArraySize,
347 ArrayRef<OperandBundleDef> OpB,
348 Function *MallocF, const Twine &Name) {
349 // malloc(type) becomes:
350 // i8* malloc(typeSize)
351 // malloc(type, arraySize) becomes:
352 // i8* malloc(typeSize*arraySize)
353 if (!ArraySize)
354 ArraySize = ConstantInt::get(Ty: IntPtrTy, V: 1);
355 else if (ArraySize->getType() != IntPtrTy)
356 ArraySize = CreateIntCast(V: ArraySize, DestTy: IntPtrTy, isSigned: false);
357
358 if (!isConstantOne(Val: ArraySize)) {
359 if (isConstantOne(Val: AllocSize)) {
360 AllocSize = ArraySize; // Operand * 1 = Operand
361 } else {
362 // Multiply type size by the array size...
363 AllocSize = CreateMul(LHS: ArraySize, RHS: AllocSize, Name: "mallocsize");
364 }
365 }
366
367 assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");
368 // Create the call to Malloc.
369 Module *M = BB->getParent()->getParent();
370 Type *BPTy = PointerType::getUnqual(C&: Context);
371 FunctionCallee MallocFunc = MallocF;
372 if (!MallocFunc)
373 // prototype malloc as "void *malloc(size_t)"
374 MallocFunc = M->getOrInsertFunction(Name: "malloc", RetTy: BPTy, Args: IntPtrTy);
375 CallInst *MCall = CreateCall(Callee: MallocFunc, Args: AllocSize, OpBundles: OpB, Name);
376
377 MCall->setTailCall();
378 if (Function *F = dyn_cast<Function>(Val: MallocFunc.getCallee())) {
379 MCall->setCallingConv(F->getCallingConv());
380 F->setReturnDoesNotAlias();
381 }
382
383 assert(!MCall->getType()->isVoidTy() && "Malloc has void return type");
384
385 return MCall;
386}
387
388CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,
389 Value *AllocSize, Value *ArraySize,
390 Function *MallocF, const Twine &Name) {
391
392 return CreateMalloc(IntPtrTy, AllocTy, AllocSize, ArraySize, OpB: {}, MallocF,
393 Name);
394}
395
396/// CreateFree - Generate the IR for a call to the builtin free function.
397CallInst *IRBuilderBase::CreateFree(Value *Source,
398 ArrayRef<OperandBundleDef> Bundles) {
399 assert(Source->getType()->isPointerTy() &&
400 "Can not free something of nonpointer type!");
401
402 Module *M = BB->getParent()->getParent();
403
404 Type *VoidTy = Type::getVoidTy(C&: M->getContext());
405 Type *VoidPtrTy = PointerType::getUnqual(C&: M->getContext());
406 // prototype free as "void free(void*)"
407 FunctionCallee FreeFunc = M->getOrInsertFunction(Name: "free", RetTy: VoidTy, Args: VoidPtrTy);
408 CallInst *Result = CreateCall(Callee: FreeFunc, Args: Source, OpBundles: Bundles, Name: "");
409 Result->setTailCall();
410 if (Function *F = dyn_cast<Function>(Val: FreeFunc.getCallee()))
411 Result->setCallingConv(F->getCallingConv());
412
413 return Result;
414}
415
416CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
417 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
418 uint32_t ElementSize, const AAMDNodes &AAInfo) {
419 assert(DstAlign >= ElementSize &&
420 "Pointer alignment must be at least element size");
421 assert(SrcAlign >= ElementSize &&
422 "Pointer alignment must be at least element size");
423 Value *Ops[] = {Dst, Src, Size, getInt32(C: ElementSize)};
424 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
425
426 CallInst *CI = CreateIntrinsicWithoutFolding(
427 ID: Intrinsic::memmove_element_unordered_atomic, OverloadTypes: Tys, Args: Ops);
428
429 // Set the alignment of the pointer args.
430 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: DstAlign));
431 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: SrcAlign));
432 CI->setAAMetadata(AAInfo);
433 return CI;
434}
435
436Value *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
437 Value *Ops[] = {Src};
438 Type *Tys[] = { Src->getType() };
439 return CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops);
440}
441
442Value *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
443 Value *Ops[] = {Acc, Src};
444 return CreateIntrinsic(ID: Intrinsic::vector_reduce_fadd, OverloadTypes: {Src->getType()}, Args: Ops);
445}
446
447Value *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
448 Value *Ops[] = {Acc, Src};
449 return CreateIntrinsic(ID: Intrinsic::vector_reduce_fmul, OverloadTypes: {Src->getType()}, Args: Ops);
450}
451
452Value *IRBuilderBase::CreateAddReduce(Value *Src) {
453 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_add, Src);
454}
455
456Value *IRBuilderBase::CreateMulReduce(Value *Src) {
457 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_mul, Src);
458}
459
460Value *IRBuilderBase::CreateAndReduce(Value *Src) {
461 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_and, Src);
462}
463
464Value *IRBuilderBase::CreateOrReduce(Value *Src) {
465 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_or, Src);
466}
467
468Value *IRBuilderBase::CreateXorReduce(Value *Src) {
469 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_xor, Src);
470}
471
472Value *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
473 auto ID =
474 IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
475 return getReductionIntrinsic(ID, Src);
476}
477
478Value *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
479 auto ID =
480 IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
481 return getReductionIntrinsic(ID, Src);
482}
483
484Value *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
485 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmax, Src);
486}
487
488Value *IRBuilderBase::CreateFPMinReduce(Value *Src) {
489 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmin, Src);
490}
491
492Value *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
493 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmaximum, Src);
494}
495
496Value *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
497 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fminimum, Src);
498}
499
500Value *IRBuilderBase::CreateFPMaximumNumReduce(Value *Src) {
501 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmaximumnum, Src);
502}
503
504Value *IRBuilderBase::CreateFPMinimumNumReduce(Value *Src) {
505 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fminimumnum, Src);
506}
507
508CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
509 assert(isa<PointerType>(Ptr->getType()) &&
510 "lifetime.start only applies to pointers.");
511 return CreateIntrinsicWithoutFolding(ID: Intrinsic::lifetime_start,
512 OverloadTypes: {Ptr->getType()}, Args: {Ptr});
513}
514
515CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
516 assert(isa<PointerType>(Ptr->getType()) &&
517 "lifetime.end only applies to pointers.");
518 return CreateIntrinsicWithoutFolding(ID: Intrinsic::lifetime_end,
519 OverloadTypes: {Ptr->getType()}, Args: {Ptr});
520}
521
522CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
523
524 assert(isa<PointerType>(Ptr->getType()) &&
525 "invariant.start only applies to pointers.");
526 if (!Size)
527 Size = getInt64(C: -1);
528 else
529 assert(Size->getType() == getInt64Ty() &&
530 "invariant.start requires the size to be an i64");
531
532 Value *Ops[] = {Size, Ptr};
533 // Fill in the single overloaded type: memory object type.
534 Type *ObjectPtr[1] = {Ptr->getType()};
535 return CreateIntrinsicWithoutFolding(ID: Intrinsic::invariant_start, OverloadTypes: ObjectPtr,
536 Args: Ops);
537}
538
539static MaybeAlign getAlign(Value *Ptr) {
540 if (auto *V = dyn_cast<GlobalVariable>(Val: Ptr))
541 return V->getAlign();
542 if (auto *A = dyn_cast<GlobalAlias>(Val: Ptr))
543 return getAlign(Ptr: A->getAliaseeObject());
544 return {};
545}
546
547CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
548 assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&
549 "threadlocal_address only applies to thread local variables.");
550 CallInst *CI = CreateIntrinsicWithoutFolding(
551 ID: llvm::Intrinsic::threadlocal_address, OverloadTypes: {Ptr->getType()}, Args: {Ptr});
552 if (MaybeAlign A = getAlign(Ptr)) {
553 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *A));
554 CI->addRetAttr(Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *A));
555 }
556 return CI;
557}
558
559CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
560 assert(Cond->getType() == getInt1Ty() &&
561 "an assumption condition must be of type i1");
562 return CreateIntrinsicWithoutFolding(ID: Intrinsic::assume, /*OverloadTypes=*/{},
563 Args: {Cond});
564}
565
566CallInst *
567IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
568 Value *Args[] = {ConstantInt::getTrue(Context&: getContext())};
569 return CreateIntrinsicWithoutFolding(
570 ID: Intrinsic::assume, /*OverloadTypes=*/{}, Args,
571 /*FMFSource=*/nullptr, /*Name=*/"", OpBundles);
572}
573
574Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
575 return CreateIntrinsicWithoutFolding(
576 ID: Intrinsic::experimental_noalias_scope_decl, OverloadTypes: {}, Args: {Scope});
577}
578
579/// Create a call to a Masked Load intrinsic.
580/// \p Ty - vector type to load
581/// \p Ptr - base pointer for the load
582/// \p Alignment - alignment of the source location
583/// \p Mask - vector of booleans which indicates what vector lanes should
584/// be accessed in memory
585/// \p PassThru - pass-through value that is used to fill the masked-off lanes
586/// of the result
587/// \p Name - name of the result variable
588CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
589 Value *Mask, Value *PassThru,
590 const Twine &Name) {
591 auto *PtrTy = cast<PointerType>(Val: Ptr->getType());
592 assert(Ty->isVectorTy() && "Type should be vector");
593 assert(Mask && "Mask should not be all-ones (null)");
594 if (!PassThru)
595 PassThru = PoisonValue::get(T: Ty);
596 Type *OverloadedTypes[] = { Ty, PtrTy };
597 Value *Ops[] = {Ptr, Mask, PassThru};
598 CallInst *CI =
599 CreateMaskedIntrinsic(Id: Intrinsic::masked_load, Ops, OverloadedTypes, Name);
600 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
601 return CI;
602}
603
604/// Create a call to a Masked Store intrinsic.
605/// \p Val - data to be stored,
606/// \p Ptr - base pointer for the store
607/// \p Alignment - alignment of the destination location
608/// \p Mask - vector of booleans which indicates what vector lanes should
609/// be accessed in memory
610CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
611 Align Alignment, Value *Mask) {
612 auto *PtrTy = cast<PointerType>(Val: Ptr->getType());
613 Type *DataTy = Val->getType();
614 assert(DataTy->isVectorTy() && "Val should be a vector");
615 assert(Mask && "Mask should not be all-ones (null)");
616 Type *OverloadedTypes[] = { DataTy, PtrTy };
617 Value *Ops[] = {Val, Ptr, Mask};
618 CallInst *CI =
619 CreateMaskedIntrinsic(Id: Intrinsic::masked_store, Ops, OverloadedTypes);
620 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
621 return CI;
622}
623
624/// Create a call to a Masked intrinsic, with given intrinsic Id,
625/// an array of operands - Ops, and an array of overloaded types -
626/// OverloadedTypes.
627CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
628 ArrayRef<Value *> Ops,
629 ArrayRef<Type *> OverloadedTypes,
630 const Twine &Name) {
631 return CreateIntrinsicWithoutFolding(ID: Id, OverloadTypes: OverloadedTypes, Args: Ops, FMFSource: {}, Name);
632}
633
634/// Create a call to a Masked Gather intrinsic.
635/// \p Ty - vector type to gather
636/// \p Ptrs - vector of pointers for loading
637/// \p Align - alignment for one element
638/// \p Mask - vector of booleans which indicates what vector lanes should
639/// be accessed in memory
640/// \p PassThru - pass-through value that is used to fill the masked-off lanes
641/// of the result
642/// \p Name - name of the result variable
643CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
644 Align Alignment, Value *Mask,
645 Value *PassThru,
646 const Twine &Name) {
647 auto *VecTy = cast<VectorType>(Val: Ty);
648 ElementCount NumElts = VecTy->getElementCount();
649 auto *PtrsTy = cast<VectorType>(Val: Ptrs->getType());
650 assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
651
652 if (!Mask)
653 Mask = getAllOnesMask(NumElts);
654
655 if (!PassThru)
656 PassThru = PoisonValue::get(T: Ty);
657
658 Type *OverloadedTypes[] = {Ty, PtrsTy};
659 Value *Ops[] = {Ptrs, Mask, PassThru};
660
661 // We specify only one type when we create this intrinsic. Types of other
662 // arguments are derived from this type.
663 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_gather, Ops,
664 OverloadedTypes, Name);
665 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
666 return CI;
667}
668
669/// Create a call to a Masked Scatter intrinsic.
670/// \p Data - data to be stored,
671/// \p Ptrs - the vector of pointers, where the \p Data elements should be
672/// stored
673/// \p Align - alignment for one element
674/// \p Mask - vector of booleans which indicates what vector lanes should
675/// be accessed in memory
676CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
677 Align Alignment, Value *Mask) {
678 auto *PtrsTy = cast<VectorType>(Val: Ptrs->getType());
679 auto *DataTy = cast<VectorType>(Val: Data->getType());
680 ElementCount NumElts = PtrsTy->getElementCount();
681
682 if (!Mask)
683 Mask = getAllOnesMask(NumElts);
684
685 Type *OverloadedTypes[] = {DataTy, PtrsTy};
686 Value *Ops[] = {Data, Ptrs, Mask};
687
688 // We specify only one type when we create this intrinsic. Types of other
689 // arguments are derived from this type.
690 CallInst *CI =
691 CreateMaskedIntrinsic(Id: Intrinsic::masked_scatter, Ops, OverloadedTypes);
692 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
693 return CI;
694}
695
696/// Create a call to Masked Expand Load intrinsic
697/// \p Ty - vector type to load
698/// \p Ptr - base pointer for the load
699/// \p Align - alignment of \p Ptr
700/// \p Mask - vector of booleans which indicates what vector lanes should
701/// be accessed in memory
702/// \p PassThru - pass-through value that is used to fill the masked-off lanes
703/// of the result
704/// \p Name - name of the result variable
705CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
706 MaybeAlign Align, Value *Mask,
707 Value *PassThru,
708 const Twine &Name) {
709 assert(Ty->isVectorTy() && "Type should be vector");
710 assert(Mask && "Mask should not be all-ones (null)");
711 if (!PassThru)
712 PassThru = PoisonValue::get(T: Ty);
713 Type *PtrTy = Ptr->getType();
714 Type *OverloadedTypes[] = {Ty, PtrTy};
715 Value *Ops[] = {Ptr, Mask, PassThru};
716 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_expandload, Ops,
717 OverloadedTypes, Name);
718 if (Align)
719 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *Align));
720 return CI;
721}
722
723/// Create a call to Masked Compress Store intrinsic
724/// \p Val - data to be stored,
725/// \p Ptr - base pointer for the store
726/// \p Align - alignment of \p Ptr
727/// \p Mask - vector of booleans which indicates what vector lanes should
728/// be accessed in memory
729CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
730 MaybeAlign Align,
731 Value *Mask) {
732 Type *DataTy = Val->getType();
733 assert(DataTy->isVectorTy() && "Val should be a vector");
734 assert(Mask && "Mask should not be all-ones (null)");
735 Type *PtrTy = Ptr->getType();
736 Type *OverloadedTypes[] = {DataTy, PtrTy};
737 Value *Ops[] = {Val, Ptr, Mask};
738 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_compressstore, Ops,
739 OverloadedTypes);
740 if (Align)
741 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *Align));
742 return CI;
743}
744
745template <typename T0>
746static std::vector<Value *>
747getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
748 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
749 std::vector<Value *> Args;
750 Args.push_back(x: B.getInt64(C: ID));
751 Args.push_back(x: B.getInt32(C: NumPatchBytes));
752 Args.push_back(x: ActualCallee);
753 Args.push_back(B.getInt32(C: CallArgs.size()));
754 Args.push_back(x: B.getInt32(C: Flags));
755 llvm::append_range(Args, CallArgs);
756 // GC Transition and Deopt args are now always handled via operand bundle.
757 // They will be removed from the signature of gc.statepoint shortly.
758 Args.push_back(x: B.getInt32(C: 0));
759 Args.push_back(x: B.getInt32(C: 0));
760 // GC args are now encoded in the gc-live operand bundle
761 return Args;
762}
763
764template<typename T1, typename T2, typename T3>
765static std::vector<OperandBundleDef>
766getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
767 std::optional<ArrayRef<T2>> DeoptArgs,
768 ArrayRef<T3> GCArgs) {
769 std::vector<OperandBundleDef> Rval;
770 if (DeoptArgs)
771 Rval.emplace_back(args: "deopt", args: SmallVector<Value *, 16>(*DeoptArgs));
772 if (TransitionArgs)
773 Rval.emplace_back(args: "gc-transition",
774 args: SmallVector<Value *, 16>(*TransitionArgs));
775 if (GCArgs.size())
776 Rval.emplace_back(args: "gc-live", args: SmallVector<Value *, 16>(GCArgs));
777 return Rval;
778}
779
780template <typename T0, typename T1, typename T2, typename T3>
781static CallInst *CreateGCStatepointCallCommon(
782 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
783 FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
784 std::optional<ArrayRef<T1>> TransitionArgs,
785 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
786 const Twine &Name) {
787 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
788 // Fill in the one generic type'd argument (the function is also vararg)
789 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
790 M, id: Intrinsic::experimental_gc_statepoint,
791 OverloadTys: {ActualCallee.getCallee()->getType()});
792
793 std::vector<Value *> Args = getStatepointArgs(
794 *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
795
796 CallInst *CI = Builder->CreateCall(
797 FnStatepoint, Args,
798 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
799 CI->addParamAttr(ArgNo: 2,
800 Attr: Attribute::get(Context&: Builder->getContext(), Kind: Attribute::ElementType,
801 Ty: ActualCallee.getFunctionType()));
802 return CI;
803}
804
805CallInst *IRBuilderBase::CreateGCStatepointCall(
806 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
807 ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
808 ArrayRef<Value *> GCArgs, const Twine &Name) {
809 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
810 Builder: this, ID, NumPatchBytes, ActualCallee, Flags: uint32_t(StatepointFlags::None),
811 CallArgs, TransitionArgs: std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
812}
813
814CallInst *IRBuilderBase::CreateGCStatepointCall(
815 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
816 uint32_t Flags, ArrayRef<Value *> CallArgs,
817 std::optional<ArrayRef<Use>> TransitionArgs,
818 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
819 const Twine &Name) {
820 return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
821 Builder: this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
822 DeoptArgs, GCArgs, Name);
823}
824
825CallInst *IRBuilderBase::CreateGCStatepointCall(
826 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
827 ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
828 ArrayRef<Value *> GCArgs, const Twine &Name) {
829 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
830 Builder: this, ID, NumPatchBytes, ActualCallee, Flags: uint32_t(StatepointFlags::None),
831 CallArgs, TransitionArgs: std::nullopt, DeoptArgs, GCArgs, Name);
832}
833
834template <typename T0, typename T1, typename T2, typename T3>
835static InvokeInst *CreateGCStatepointInvokeCommon(
836 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
837 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
838 BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
839 std::optional<ArrayRef<T1>> TransitionArgs,
840 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
841 const Twine &Name) {
842 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
843 // Fill in the one generic type'd argument (the function is also vararg)
844 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
845 M, id: Intrinsic::experimental_gc_statepoint,
846 OverloadTys: {ActualInvokee.getCallee()->getType()});
847
848 std::vector<Value *> Args =
849 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
850 Flags, InvokeArgs);
851
852 InvokeInst *II = Builder->CreateInvoke(
853 FnStatepoint, NormalDest, UnwindDest, Args,
854 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
855 II->addParamAttr(ArgNo: 2,
856 Attr: Attribute::get(Context&: Builder->getContext(), Kind: Attribute::ElementType,
857 Ty: ActualInvokee.getFunctionType()));
858 return II;
859}
860
861InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
862 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
863 BasicBlock *NormalDest, BasicBlock *UnwindDest,
864 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
865 ArrayRef<Value *> GCArgs, const Twine &Name) {
866 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
867 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
868 Flags: uint32_t(StatepointFlags::None), InvokeArgs,
869 TransitionArgs: std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
870}
871
872InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
873 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
874 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
875 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
876 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
877 const Twine &Name) {
878 return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
879 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
880 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
881}
882
883InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
884 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
885 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
886 std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
887 const Twine &Name) {
888 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
889 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
890 Flags: uint32_t(StatepointFlags::None), InvokeArgs, TransitionArgs: std::nullopt, DeoptArgs,
891 GCArgs, Name);
892}
893
894CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
895 Type *ResultType, const Twine &Name) {
896 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
897 Type *Types[] = {ResultType};
898
899 Value *Args[] = {Statepoint};
900 return CreateIntrinsicWithoutFolding(ID, OverloadTypes: Types, Args, FMFSource: {}, Name);
901}
902
903CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
904 int BaseOffset, int DerivedOffset,
905 Type *ResultType, const Twine &Name) {
906 Type *Types[] = {ResultType};
907
908 Value *Args[] = {Statepoint, getInt32(C: BaseOffset), getInt32(C: DerivedOffset)};
909 return CreateIntrinsicWithoutFolding(ID: Intrinsic::experimental_gc_relocate,
910 OverloadTypes: Types, Args, FMFSource: {}, Name);
911}
912
913CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
914 const Twine &Name) {
915 Type *PtrTy = DerivedPtr->getType();
916 return CreateIntrinsicWithoutFolding(
917 ID: Intrinsic::experimental_gc_get_pointer_base, OverloadTypes: PtrTy, Args: DerivedPtr, FMFSource: {}, Name);
918}
919
920CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
921 const Twine &Name) {
922 Type *PtrTy = DerivedPtr->getType();
923 return CreateIntrinsicWithoutFolding(
924 ID: Intrinsic::experimental_gc_get_pointer_offset, OverloadTypes: {PtrTy}, Args: {DerivedPtr}, FMFSource: {},
925 Name);
926}
927
928Value *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op,
929 FMFSource FMFSource,
930 const Twine &Name) {
931 Module *M = BB->getModule();
932 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: Op->getType());
933 if (Value *V =
934 Folder.FoldIntrinsic(ID, Ops: Op, Ty: Fn->getReturnType(), FMF: FMFSource.get(Default: FMF),
935 CtxF: GetInsertBlock()->getParent()))
936 return V;
937 return createCallHelper(Callee: Fn, Ops: Op, Name, FMFSource);
938}
939
940Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
941 Value *RHS, FMFSource FMFSource,
942 const Twine &Name) {
943 Module *M = BB->getModule();
944 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: {LHS->getType()});
945 if (Value *V = Folder.FoldIntrinsic(ID, Ops: {LHS, RHS}, Ty: Fn->getReturnType(),
946 FMF: FMFSource.get(Default: FMF),
947 CtxF: GetInsertBlock()->getParent()))
948 return V;
949 return createCallHelper(Callee: Fn, Ops: {LHS, RHS}, Name, FMFSource);
950}
951
952CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(
953 Intrinsic::ID ID, ArrayRef<Type *> OverloadTypes, ArrayRef<Value *> Args,
954 FMFSource FMFSource, const Twine &Name,
955 ArrayRef<OperandBundleDef> OpBundles) {
956 Module *M = BB->getModule();
957 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: OverloadTypes);
958 return createCallHelper(Callee: Fn, Ops: Args, Name, FMFSource, OpBundles);
959}
960
961CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(Type *RetTy,
962 Intrinsic::ID ID,
963 ArrayRef<Value *> Args,
964 FMFSource FMFSource,
965 const Twine &Name) {
966 Module *M = BB->getModule();
967 SmallVector<Type *> ArgTys = llvm::map_to_vector(C&: Args, F: &Value::getType);
968 Function *Fn = Intrinsic::getOrInsertDeclaration(M, IID: ID, RetTy, ArgTys);
969 return createCallHelper(Callee: Fn, Ops: Args, Name, FMFSource);
970}
971
972Value *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
973 ArrayRef<Type *> OverloadTypes,
974 ArrayRef<Value *> Args,
975 FMFSource FMFSource, const Twine &Name,
976 ArrayRef<OperandBundleDef> OpBundles,
977 function_ref<void(CallInst *)> SetFn) {
978 Type *RetTy = Intrinsic::getType(Context, id: ID, OverloadTys: OverloadTypes)->getReturnType();
979 if (Value *V = Folder.FoldIntrinsic(ID, Ops: Args, Ty: RetTy, FMF: FMFSource.get(Default: FMF),
980 CtxF: GetInsertBlock()->getParent()))
981 return V;
982 CallInst *CI = CreateIntrinsicWithoutFolding(ID, OverloadTypes, Args,
983 FMFSource, Name, OpBundles);
984 SetFn(CI);
985 return CI;
986}
987
988Value *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
989 ArrayRef<Value *> Args,
990 FMFSource FMFSource, const Twine &Name,
991 function_ref<void(CallInst *)> SetFn) {
992 if (Value *V = Folder.FoldIntrinsic(ID, Ops: Args, Ty: RetTy, FMF: FMFSource.get(Default: FMF),
993 CtxF: GetInsertBlock()->getParent()))
994 return V;
995 CallInst *CI =
996 CreateIntrinsicWithoutFolding(RetTy, ID, Args, FMFSource, Name);
997 SetFn(CI);
998 return CI;
999}
1000
1001CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
1002 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
1003 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
1004 std::optional<fp::ExceptionBehavior> Except) {
1005 Value *RoundingV = getConstrainedFPRounding(Rounding);
1006 Value *ExceptV = getConstrainedFPExcept(Except);
1007
1008 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1009 CallInst *C = CreateIntrinsicWithoutFolding(
1010 ID, OverloadTypes: {L->getType()}, Args: {L, R, RoundingV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1011 setConstrainedFPCallAttr(C);
1012 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1013 return C;
1014}
1015
1016CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
1017 Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
1018 FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag,
1019 std::optional<RoundingMode> Rounding,
1020 std::optional<fp::ExceptionBehavior> Except) {
1021 Value *RoundingV = getConstrainedFPRounding(Rounding);
1022 Value *ExceptV = getConstrainedFPExcept(Except);
1023
1024 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1025
1026 llvm::SmallVector<Value *, 5> ExtArgs(Args);
1027 ExtArgs.push_back(Elt: RoundingV);
1028 ExtArgs.push_back(Elt: ExceptV);
1029 CallInst *C =
1030 CreateIntrinsicWithoutFolding(ID, OverloadTypes: Types, Args: ExtArgs, FMFSource: nullptr, Name, OpBundles: {});
1031 setConstrainedFPCallAttr(C);
1032 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1033 return C;
1034}
1035
1036CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
1037 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
1038 const Twine &Name, MDNode *FPMathTag,
1039 std::optional<fp::ExceptionBehavior> Except) {
1040 Value *ExceptV = getConstrainedFPExcept(Except);
1041
1042 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1043 CallInst *C = CreateIntrinsicWithoutFolding(
1044 ID, OverloadTypes: {L->getType()}, Args: {L, R, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1045 setConstrainedFPCallAttr(C);
1046 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1047 return C;
1048}
1049
1050Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1051 const Twine &Name, MDNode *FPMathTag) {
1052 if (Instruction::isBinaryOp(Opcode: Opc)) {
1053 assert(Ops.size() == 2 && "Invalid number of operands!");
1054 return CreateBinOp(Opc: static_cast<Instruction::BinaryOps>(Opc),
1055 LHS: Ops[0], RHS: Ops[1], Name, FPMathTag);
1056 }
1057 if (Instruction::isUnaryOp(Opcode: Opc)) {
1058 assert(Ops.size() == 1 && "Invalid number of operands!");
1059 return CreateUnOp(Opc: static_cast<Instruction::UnaryOps>(Opc),
1060 V: Ops[0], Name, FPMathTag);
1061 }
1062 llvm_unreachable("Unexpected opcode!");
1063}
1064
1065CallInst *IRBuilderBase::CreateConstrainedFPCast(
1066 Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource,
1067 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
1068 std::optional<fp::ExceptionBehavior> Except) {
1069 Value *ExceptV = getConstrainedFPExcept(Except);
1070
1071 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1072
1073 CallInst *C;
1074 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: ID)) {
1075 Value *RoundingV = getConstrainedFPRounding(Rounding);
1076 C = CreateIntrinsicWithoutFolding(
1077 ID, OverloadTypes: {DestTy, V->getType()}, Args: {V, RoundingV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1078 } else
1079 C = CreateIntrinsicWithoutFolding(ID, OverloadTypes: {DestTy, V->getType()}, Args: {V, ExceptV},
1080 FMFSource: nullptr, Name, OpBundles: {});
1081 setConstrainedFPCallAttr(C);
1082
1083 if (isa<FPMathOperator>(Val: C))
1084 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1085 return C;
1086}
1087
1088Value *IRBuilderBase::CreateFCmpHelper(CmpInst::Predicate P, Value *LHS,
1089 Value *RHS, const Twine &Name,
1090 MDNode *FPMathTag, FMFSource FMFSource,
1091 bool IsSignaling) {
1092 if (IsFPConstrained) {
1093 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
1094 : Intrinsic::experimental_constrained_fcmp;
1095 return CreateConstrainedFPCmp(ID, P, L: LHS, R: RHS, Name);
1096 }
1097
1098 if (auto *V = Folder.FoldCmp(P, LHS, RHS))
1099 return V;
1100 return Insert(
1101 I: setFPAttrs(I: new FCmpInst(P, LHS, RHS), FPMD: FPMathTag, FMF: FMFSource.get(Default: FMF)),
1102 Name);
1103}
1104
1105CallInst *IRBuilderBase::CreateConstrainedFPCmp(
1106 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
1107 const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
1108 Value *PredicateV = getConstrainedFPPredicate(Predicate: P);
1109 Value *ExceptV = getConstrainedFPExcept(Except);
1110
1111 CallInst *C = CreateIntrinsicWithoutFolding(
1112 ID, OverloadTypes: {L->getType()}, Args: {L, R, PredicateV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1113 setConstrainedFPCallAttr(C);
1114 return C;
1115}
1116
1117CallInst *IRBuilderBase::CreateConstrainedFPCall(
1118 Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
1119 std::optional<RoundingMode> Rounding,
1120 std::optional<fp::ExceptionBehavior> Except) {
1121 llvm::SmallVector<Value *, 6> UseArgs(Args);
1122
1123 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: Callee->getIntrinsicID()))
1124 UseArgs.push_back(Elt: getConstrainedFPRounding(Rounding));
1125 UseArgs.push_back(Elt: getConstrainedFPExcept(Except));
1126
1127 CallInst *C = CreateCall(Callee, Args: UseArgs, Name);
1128 setConstrainedFPCallAttr(C);
1129 return C;
1130}
1131
1132Value *IRBuilderBase::CreateSelectWithUnknownProfile(Value *C, Value *True,
1133 Value *False,
1134 StringRef PassName,
1135 const Twine &Name) {
1136 Value *Ret = CreateSelectFMF(C, True, False, FMFSource: {}, Name);
1137 if (auto *SI = dyn_cast<SelectInst>(Val: Ret)) {
1138 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, PassName);
1139 }
1140 return Ret;
1141}
1142
1143Value *IRBuilderBase::CreateSelectFMFWithUnknownProfile(Value *C, Value *True,
1144 Value *False,
1145 FMFSource FMFSource,
1146 StringRef PassName,
1147 const Twine &Name) {
1148 Value *Ret = CreateSelectFMF(C, True, False, FMFSource, Name);
1149 if (auto *SI = dyn_cast<SelectInst>(Val: Ret))
1150 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, PassName);
1151 return Ret;
1152}
1153
1154Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
1155 const Twine &Name, Instruction *MDFrom) {
1156 return CreateSelectFMF(C, True, False, FMFSource: {}, Name, MDFrom);
1157}
1158
1159Value *IRBuilderBase::CreateSelectFMF(Value *C, Value *True, Value *False,
1160 FMFSource FMFSource, const Twine &Name,
1161 Instruction *MDFrom) {
1162 if (auto *V = Folder.FoldSelect(C, True, False, FMF: FMFSource.get(Default: FMF)))
1163 return V;
1164
1165 SelectInst *Sel = SelectInst::Create(C, S1: True, S2: False);
1166 if (MDFrom) {
1167 MDNode *Prof = MDFrom->getMetadata(KindID: LLVMContext::MD_prof);
1168 MDNode *Unpred = MDFrom->getMetadata(KindID: LLVMContext::MD_unpredictable);
1169 Sel = addBranchMetadata(I: Sel, Weights: Prof, Unpredictable: Unpred);
1170 }
1171 if (isa<FPMathOperator>(Val: Sel))
1172 setFPAttrs(I: Sel, /*MDNode=*/FPMD: nullptr, FMF: FMFSource.get(Default: FMF));
1173 return Insert(I: Sel, Name);
1174}
1175
1176Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name,
1177 bool IsNUW) {
1178 assert(LHS->getType() == RHS->getType() &&
1179 "Pointer subtraction operand types must match!");
1180 Value *LHSAddr = CreatePtrToAddr(V: LHS);
1181 Value *RHSAddr = CreatePtrToAddr(V: RHS);
1182 return CreateSub(LHS: LHSAddr, RHS: RHSAddr, Name, HasNUW: IsNUW);
1183}
1184Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
1185 const Twine &Name) {
1186 const DataLayout &DL = BB->getDataLayout();
1187 TypeSize ElemSize = DL.getTypeAllocSize(Ty: ElemTy);
1188 if (ElemSize == TypeSize::getFixed(ExactSize: 1))
1189 return CreatePtrDiff(LHS, RHS, Name);
1190
1191 Value *Diff = CreatePtrDiff(LHS, RHS);
1192 return CreateExactSDiv(LHS: Diff, RHS: CreateTypeSize(Ty: Diff->getType(), Size: ElemSize), Name);
1193}
1194
1195Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
1196 assert(isa<PointerType>(Ptr->getType()) &&
1197 "launder.invariant.group only applies to pointers.");
1198 auto *PtrType = Ptr->getType();
1199 Module *M = BB->getParent()->getParent();
1200 Function *FnLaunderInvariantGroup = Intrinsic::getOrInsertDeclaration(
1201 M, id: Intrinsic::launder_invariant_group, OverloadTys: {PtrType});
1202
1203 assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&
1204 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1205 PtrType &&
1206 "LaunderInvariantGroup should take and return the same type");
1207
1208 return CreateCall(Callee: FnLaunderInvariantGroup, Args: {Ptr});
1209}
1210
1211Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
1212 assert(isa<PointerType>(Ptr->getType()) &&
1213 "strip.invariant.group only applies to pointers.");
1214
1215 auto *PtrType = Ptr->getType();
1216 Module *M = BB->getParent()->getParent();
1217 Function *FnStripInvariantGroup = Intrinsic::getOrInsertDeclaration(
1218 M, id: Intrinsic::strip_invariant_group, OverloadTys: {PtrType});
1219
1220 assert(FnStripInvariantGroup->getReturnType() == PtrType &&
1221 FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1222 PtrType &&
1223 "StripInvariantGroup should take and return the same type");
1224
1225 return CreateCall(Callee: FnStripInvariantGroup, Args: {Ptr});
1226}
1227
1228Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1229 auto *Ty = cast<VectorType>(Val: V->getType());
1230 if (isa<ScalableVectorType>(Val: Ty)) {
1231 Module *M = BB->getParent()->getParent();
1232 Function *F =
1233 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::vector_reverse, OverloadTys: Ty);
1234 return Insert(I: CallInst::Create(Func: F, Args: V), Name);
1235 }
1236 // Keep the original behaviour for fixed vector
1237 SmallVector<int, 8> ShuffleMask;
1238 int NumElts = Ty->getElementCount().getKnownMinValue();
1239 for (int i = 0; i < NumElts; ++i)
1240 ShuffleMask.push_back(Elt: NumElts - i - 1);
1241 return CreateShuffleVector(V, Mask: ShuffleMask, Name);
1242}
1243
1244static SmallVector<int, 8> getSpliceMask(int64_t Imm, unsigned NumElts) {
1245 unsigned Idx = (NumElts + Imm) % NumElts;
1246 SmallVector<int, 8> Mask;
1247 for (unsigned I = 0; I < NumElts; ++I)
1248 Mask.push_back(Elt: Idx + I);
1249 return Mask;
1250}
1251
1252Value *IRBuilderBase::CreateVectorSpliceLeft(Value *V1, Value *V2,
1253 Value *Offset, const Twine &Name) {
1254 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1255 assert(V1->getType() == V2->getType() &&
1256 "Splice expects matching operand types!");
1257
1258 // Emit a shufflevector for fixed vectors with a constant offset
1259 if (auto *COffset = dyn_cast<ConstantInt>(Val: Offset))
1260 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType()))
1261 return CreateShuffleVector(
1262 V1, V2,
1263 Mask: getSpliceMask(Imm: COffset->getZExtValue(), NumElts: FVTy->getNumElements()));
1264
1265 return CreateIntrinsic(ID: Intrinsic::vector_splice_left, OverloadTypes: V1->getType(),
1266 Args: {V1, V2, Offset}, FMFSource: {}, Name);
1267}
1268
1269Value *IRBuilderBase::CreateVectorSpliceRight(Value *V1, Value *V2,
1270 Value *Offset,
1271 const Twine &Name) {
1272 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1273 assert(V1->getType() == V2->getType() &&
1274 "Splice expects matching operand types!");
1275
1276 // Emit a shufflevector for fixed vectors with a constant offset
1277 if (auto *COffset = dyn_cast<ConstantInt>(Val: Offset))
1278 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType()))
1279 return CreateShuffleVector(
1280 V1, V2,
1281 Mask: getSpliceMask(Imm: -COffset->getZExtValue(), NumElts: FVTy->getNumElements()));
1282
1283 return CreateIntrinsic(ID: Intrinsic::vector_splice_right, OverloadTypes: V1->getType(),
1284 Args: {V1, V2, Offset}, FMFSource: {}, Name);
1285}
1286
1287Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1288 const Twine &Name) {
1289 auto EC = ElementCount::getFixed(MinVal: NumElts);
1290 return CreateVectorSplat(EC, V, Name);
1291}
1292
1293Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1294 const Twine &Name) {
1295 assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1296
1297 // First insert it into a poison vector so we can shuffle it.
1298 Value *Poison = PoisonValue::get(T: VectorType::get(ElementType: V->getType(), EC));
1299 V = CreateInsertElement(Vec: Poison, NewElt: V, Idx: getInt64(C: 0), Name: Name + ".splatinsert");
1300
1301 // Shuffle the value across the desired number of elements.
1302 SmallVector<int, 16> Zeros;
1303 Zeros.resize(N: EC.getKnownMinValue());
1304 return CreateShuffleVector(V, Mask: Zeros, Name: Name + ".splat");
1305}
1306
1307Value *IRBuilderBase::CreateVectorInterleave(ArrayRef<Value *> Ops,
1308 const Twine &Name) {
1309 assert(Ops.size() >= 2 && Ops.size() <= 8 &&
1310 "Unexpected number of operands to interleave");
1311
1312 // Make sure all operands are the same type.
1313 assert(isa<VectorType>(Ops[0]->getType()) && "Unexpected type");
1314
1315#ifndef NDEBUG
1316 for (unsigned I = 1; I < Ops.size(); I++) {
1317 assert(Ops[I]->getType() == Ops[0]->getType() &&
1318 "Vector interleave expects matching operand types!");
1319 }
1320#endif
1321
1322 unsigned IID = Intrinsic::getInterleaveIntrinsicID(Factor: Ops.size());
1323 auto *SubvecTy = cast<VectorType>(Val: Ops[0]->getType());
1324 Type *DestTy = VectorType::get(ElementType: SubvecTy->getElementType(),
1325 EC: SubvecTy->getElementCount() * Ops.size());
1326 return CreateIntrinsic(ID: IID, OverloadTypes: {DestTy}, Args: Ops, FMFSource: {}, Name);
1327}
1328
1329Value *IRBuilderBase::CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
1330 unsigned Dimension,
1331 unsigned LastIndex,
1332 MDNode *DbgInfo) {
1333 auto *BaseType = Base->getType();
1334 assert(isa<PointerType>(BaseType) &&
1335 "Invalid Base ptr type for preserve.array.access.index.");
1336
1337 Value *LastIndexV = getInt32(C: LastIndex);
1338 Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 0);
1339 SmallVector<Value *, 4> IdxList(Dimension, Zero);
1340 IdxList.push_back(Elt: LastIndexV);
1341
1342 Type *ResultType = GetElementPtrInst::getGEPReturnType(Ptr: Base, IdxList);
1343
1344 Value *DimV = getInt32(C: Dimension);
1345 CallInst *Fn = CreateIntrinsicWithoutFolding(
1346 ID: Intrinsic::preserve_array_access_index, OverloadTypes: {ResultType, BaseType},
1347 Args: {Base, DimV, LastIndexV});
1348 Fn->addParamAttr(
1349 ArgNo: 0, Attr: Attribute::get(Context&: Fn->getContext(), Kind: Attribute::ElementType, Ty: ElTy));
1350 if (DbgInfo)
1351 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1352
1353 return Fn;
1354}
1355
1356Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1357 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1358 assert(isa<PointerType>(Base->getType()) &&
1359 "Invalid Base ptr type for preserve.union.access.index.");
1360 auto *BaseType = Base->getType();
1361
1362 Value *DIIndex = getInt32(C: FieldIndex);
1363 CallInst *Fn =
1364 CreateIntrinsicWithoutFolding(ID: Intrinsic::preserve_union_access_index,
1365 OverloadTypes: {BaseType, BaseType}, Args: {Base, DIIndex});
1366 if (DbgInfo)
1367 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1368
1369 return Fn;
1370}
1371
1372Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1373 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1374 MDNode *DbgInfo) {
1375 auto *BaseType = Base->getType();
1376 assert(isa<PointerType>(BaseType) &&
1377 "Invalid Base ptr type for preserve.struct.access.index.");
1378
1379 Value *GEPIndex = getInt32(C: Index);
1380 Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 0);
1381 Type *ResultType =
1382 GetElementPtrInst::getGEPReturnType(Ptr: Base, IdxList: {Zero, GEPIndex});
1383
1384 Value *DIIndex = getInt32(C: FieldIndex);
1385 CallInst *Fn = CreateIntrinsicWithoutFolding(
1386 ID: Intrinsic::preserve_struct_access_index, OverloadTypes: {ResultType, BaseType},
1387 Args: {Base, GEPIndex, DIIndex});
1388 Fn->addParamAttr(
1389 ArgNo: 0, Attr: Attribute::get(Context&: Fn->getContext(), Kind: Attribute::ElementType, Ty: ElTy));
1390 if (DbgInfo)
1391 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1392
1393 return Fn;
1394}
1395
1396Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
1397 ConstantInt *TestV = getInt32(C: Test);
1398 return CreateIntrinsic(ID: Intrinsic::is_fpclass, OverloadTypes: {FPNum->getType()},
1399 Args: {FPNum, TestV});
1400}
1401
1402CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1403 Value *PtrValue,
1404 Value *AlignValue,
1405 Value *OffsetValue) {
1406 SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1407 if (OffsetValue)
1408 Vals.push_back(Elt: OffsetValue);
1409 OperandBundleDefT<Value *> AlignOpB("align", Vals);
1410 return CreateAssumption(OpBundles: {AlignOpB});
1411}
1412
1413CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1414 Value *PtrValue,
1415 uint64_t Alignment,
1416 Value *OffsetValue) {
1417 assert(isa<PointerType>(PtrValue->getType()) &&
1418 "trying to create an alignment assumption on a non-pointer?");
1419 assert(Alignment != 0 && "Invalid Alignment");
1420 Value *AlignValue = ConstantInt::get(Ty: getInt64Ty(), V: Alignment);
1421 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1422}
1423
1424CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1425 Value *PtrValue,
1426 Value *Alignment,
1427 Value *OffsetValue) {
1428 assert(isa<PointerType>(PtrValue->getType()) &&
1429 "trying to create an alignment assumption on a non-pointer?");
1430 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue: Alignment, OffsetValue);
1431}
1432
1433CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
1434 Value *SizeValue) {
1435 assert(isa<PointerType>(PtrValue->getType()) &&
1436 "trying to create a deferenceable assumption on a non-pointer?");
1437 SmallVector<Value *, 4> Vals({PtrValue, SizeValue});
1438 OperandBundleDefT<Value *> DereferenceableOpB("dereferenceable", Vals);
1439 return CreateAssumption(OpBundles: {DereferenceableOpB});
1440}
1441
1442CallInst *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
1443 assert(isa<PointerType>(PtrValue->getType()) &&
1444 "trying to create a nonnull assumption on a non-pointer?");
1445 return CreateAssumption(OpBundles: OperandBundleDef("nonnull", PtrValue));
1446}
1447
1448IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default;
1449IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default;
1450IRBuilderFolder::~IRBuilderFolder() = default;
1451void ConstantFolder::anchor() {}
1452void NoFolder::anchor() {}
1453