1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallVectorExtras.h"
17#include "llvm/IR/Constant.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/DerivedTypes.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GlobalValue.h"
22#include "llvm/IR/GlobalVariable.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/NoFolder.h"
28#include "llvm/IR/Operator.h"
29#include "llvm/IR/ProfDataUtils.h"
30#include "llvm/IR/Statepoint.h"
31#include "llvm/IR/Type.h"
32#include "llvm/IR/Value.h"
33#include "llvm/Support/Casting.h"
34#include <cassert>
35#include <cstdint>
36#include <optional>
37#include <vector>
38
39using namespace llvm;
40
41/// CreateGlobalString - Make a new global variable with an initializer that
42/// has array of i8 type filled in with the nul terminated string value
43/// specified. If Name is specified, it is the name of the global variable
44/// created.
45GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
46 const Twine &Name,
47 unsigned AddressSpace,
48 Module *M, bool AddNull) {
49 Constant *StrConstant = ConstantDataArray::getString(Context, Initializer: Str, AddNull);
50 if (!M)
51 M = BB->getParent()->getParent();
52 auto *GV = new GlobalVariable(
53 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
54 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
55 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
56 GV->setAlignment(M->getDataLayout().getPrefTypeAlign(Ty: getInt8Ty()));
57 return GV;
58}
59
60Type *IRBuilderBase::getCurrentFunctionReturnType() const {
61 assert(BB && BB->getParent() && "No current function!");
62 return BB->getParent()->getReturnType();
63}
64
65DebugLoc IRBuilderBase::getCurrentDebugLocation() const { return StoredDL; }
66void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
67 // We prefer to set our current debug location if any has been set, but if
68 // our debug location is empty and I has a valid location, we shouldn't
69 // overwrite it.
70 I->setDebugLoc(StoredDL.orElse(Other: I->getDebugLoc()));
71}
72
73Value *IRBuilderBase::CreateAggregateCast(Value *V, Type *DestTy) {
74 Type *SrcTy = V->getType();
75 if (SrcTy == DestTy)
76 return V;
77
78 if (SrcTy->isAggregateType()) {
79 unsigned NumElements;
80 if (SrcTy->isStructTy()) {
81 assert(DestTy->isStructTy() && "Expected StructType");
82 assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements() &&
83 "Expected StructTypes with equal number of elements");
84 NumElements = SrcTy->getStructNumElements();
85 } else {
86 assert(SrcTy->isArrayTy() && DestTy->isArrayTy() && "Expected ArrayType");
87 assert(SrcTy->getArrayNumElements() == DestTy->getArrayNumElements() &&
88 "Expected ArrayTypes with equal number of elements");
89 NumElements = SrcTy->getArrayNumElements();
90 }
91
92 Value *Result = PoisonValue::get(T: DestTy);
93 for (unsigned I = 0; I < NumElements; ++I) {
94 Type *ElementTy = SrcTy->isStructTy() ? DestTy->getStructElementType(N: I)
95 : DestTy->getArrayElementType();
96 Value *Element =
97 CreateAggregateCast(V: CreateExtractValue(Agg: V, Idxs: ArrayRef(I)), DestTy: ElementTy);
98
99 Result = CreateInsertValue(Agg: Result, Val: Element, Idxs: ArrayRef(I));
100 }
101 return Result;
102 }
103
104 return CreateBitOrPointerCast(V, DestTy);
105}
106
107Value *IRBuilderBase::CreateBitPreservingCastChain(const DataLayout &DL,
108 Value *V, Type *NewTy) {
109 Type *OldTy = V->getType();
110
111 if (OldTy == NewTy)
112 return V;
113
114 assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
115 "Integer types must be the exact same to convert.");
116
117 // A variant of bitcast that supports a mixture of fixed and scalable types
118 // that are know to have the same size.
119 auto CreateBitCastLike = [this](Value *In, Type *Ty) -> Value * {
120 Type *InTy = In->getType();
121 if (InTy == Ty)
122 return In;
123
124 if (isa<FixedVectorType>(Val: InTy) && isa<ScalableVectorType>(Val: Ty)) {
125 // For vscale_range(2) expand <4 x i32> to <vscale x 4 x i16> -->
126 // <4 x i32> to <vscale x 2 x i32> to <vscale x 4 x i16>
127 auto *VTy = VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: Ty), EltTy: InTy);
128 return CreateBitCast(
129 V: CreateInsertVector(DstType: VTy, SrcVec: PoisonValue::get(T: VTy), SubVec: In, Idx: getInt64(C: 0)), DestTy: Ty);
130 }
131
132 if (isa<ScalableVectorType>(Val: InTy) && isa<FixedVectorType>(Val: Ty)) {
133 // For vscale_range(2) expand <vscale x 4 x i16> to <4 x i32> -->
134 // <vscale x 4 x i16> to <vscale x 2 x i32> to <4 x i32>
135 auto *VTy = VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: InTy), EltTy: Ty);
136 return CreateExtractVector(DstType: Ty, SrcVec: CreateBitCast(V: In, DestTy: VTy), Idx: getInt64(C: 0));
137 }
138
139 return CreateBitCast(V: In, DestTy: Ty);
140 };
141
142 // See if we need inttoptr for this type pair. May require additional bitcast.
143 bool OldIsIntLike =
144 OldTy->isIntOrIntVectorTy() || OldTy->isByteOrByteVectorTy();
145 if (OldIsIntLike && NewTy->isPtrOrPtrVectorTy()) {
146 // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
147 // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
148 // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*>
149 // Directly handle i64 to i8*
150 return CreateIntToPtr(V: CreateBitCastLike(V, DL.getIntPtrType(NewTy)), DestTy: NewTy);
151 }
152
153 // See if we need ptrtoint for this type pair. May require additional bitcast.
154 bool NewIsIntLike =
155 NewTy->isIntOrIntVectorTy() || NewTy->isByteOrByteVectorTy();
156 if (OldTy->isPtrOrPtrVectorTy() && NewIsIntLike) {
157 // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128
158 // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
159 // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32>
160 // Expand i8* to i64 --> i8* to i64 to i64
161 return CreateBitCastLike(CreatePtrToInt(V, DestTy: DL.getIntPtrType(OldTy)), NewTy);
162 }
163
164 if (OldTy->isPtrOrPtrVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
165 unsigned OldAS = OldTy->getPointerAddressSpace();
166 unsigned NewAS = NewTy->getPointerAddressSpace();
167 // To convert pointers with different address spaces (they are already
168 // checked convertible, i.e. they have the same pointer size), so far we
169 // cannot use `bitcast` (which has restrict on the same address space) or
170 // `addrspacecast` (which is not always no-op casting). Instead, use a pair
171 // of no-op `ptrtoint`/`inttoptr` casts through an integer with the same bit
172 // size.
173 if (OldAS != NewAS) {
174 return CreateIntToPtr(
175 V: CreateBitCastLike(CreatePtrToInt(V, DestTy: DL.getIntPtrType(OldTy)),
176 DL.getIntPtrType(NewTy)),
177 DestTy: NewTy);
178 }
179 }
180
181 return CreateBitCastLike(V, NewTy);
182}
183
184CallInst *
185IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
186 const Twine &Name, FMFSource FMFSource,
187 ArrayRef<OperandBundleDef> OpBundles) {
188 CallInst *CI = CreateCall(Callee, Args: Ops, OpBundles, Name);
189 if (isa<FPMathOperator>(Val: CI))
190 CI->setFastMathFlags(FMFSource.get(Default: FMF));
191 return CI;
192}
193
194static Value *CreateVScaleMultiple(IRBuilderBase &B, Type *Ty, uint64_t Scale) {
195 Value *VScale = B.CreateVScale(Ty);
196 if (Scale == 1)
197 return VScale;
198
199 return B.CreateNUWMul(LHS: VScale, RHS: ConstantInt::get(Ty, V: Scale));
200}
201
202Value *IRBuilderBase::CreateElementCount(Type *Ty, ElementCount EC) {
203 if (EC.isFixed() || EC.isZero())
204 return ConstantInt::get(Ty, V: EC.getKnownMinValue());
205
206 return CreateVScaleMultiple(B&: *this, Ty, Scale: EC.getKnownMinValue());
207}
208
209Value *IRBuilderBase::CreateTypeSize(Type *Ty, TypeSize Size) {
210 if (Size.isFixed() || Size.isZero())
211 return ConstantInt::get(Ty, V: Size.getKnownMinValue());
212
213 return CreateVScaleMultiple(B&: *this, Ty, Scale: Size.getKnownMinValue());
214}
215
216Value *IRBuilderBase::CreateAllocationSize(Type *DestTy, AllocaInst *AI) {
217 const DataLayout &DL = BB->getDataLayout();
218 TypeSize ElemSize = AI->getAllocationBaseSize(DL);
219 Value *Size = CreateTypeSize(Ty: DestTy, Size: ElemSize);
220 if (AI->isArrayAllocation())
221 Size = CreateMul(LHS: CreateZExtOrTrunc(V: AI->getArraySize(), DestTy), RHS: Size);
222 return Size;
223}
224
225Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
226 Type *STy = DstType->getScalarType();
227 if (isa<ScalableVectorType>(Val: DstType)) {
228 Type *StepVecType = DstType;
229 // TODO: We expect this special case (element type < 8 bits) to be
230 // temporary - once the intrinsic properly supports < 8 bits this code
231 // can be removed.
232 if (STy->getScalarSizeInBits() < 8)
233 StepVecType =
234 VectorType::get(ElementType: getInt8Ty(), Other: cast<ScalableVectorType>(Val: DstType));
235 Value *Res = CreateIntrinsic(ID: Intrinsic::stepvector, OverloadTypes: {StepVecType}, Args: {},
236 FMFSource: nullptr, Name);
237 if (StepVecType != DstType)
238 Res = CreateTrunc(V: Res, DestTy: DstType);
239 return Res;
240 }
241
242 unsigned NumEls = cast<FixedVectorType>(Val: DstType)->getNumElements();
243
244 // Create a vector of consecutive numbers from zero to VF.
245 // It's okay if the values wrap around.
246 SmallVector<Constant *, 8> Indices;
247 for (unsigned i = 0; i < NumEls; ++i)
248 Indices.push_back(
249 Elt: ConstantInt::get(Ty: STy, V: i, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
250
251 // Add the consecutive indices to the vector value.
252 return ConstantVector::get(V: Indices);
253}
254
255CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
256 MaybeAlign Align, bool isVolatile,
257 const AAMDNodes &AAInfo) {
258 Value *Ops[] = {Ptr, Val, Size, getInt1(V: isVolatile)};
259 Type *Tys[] = {Ptr->getType(), Size->getType()};
260
261 auto *CI = cast<MemSetInst>(
262 Val: CreateIntrinsicWithoutFolding(ID: Intrinsic::memset, OverloadTypes: Tys, Args: Ops));
263
264 if (Align)
265 CI->setDestAlignment(*Align);
266 CI->setAAMetadata(AAInfo);
267 return CI;
268}
269
270CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
271 Value *Val, Value *Size,
272 bool IsVolatile,
273 const AAMDNodes &AAInfo) {
274 Value *Ops[] = {Dst, Val, Size, getInt1(V: IsVolatile)};
275 Type *Tys[] = {Dst->getType(), Size->getType()};
276
277 auto *CI = cast<MemSetInst>(
278 Val: CreateIntrinsicWithoutFolding(ID: Intrinsic::memset_inline, OverloadTypes: Tys, Args: Ops));
279
280 if (DstAlign)
281 CI->setDestAlignment(*DstAlign);
282 CI->setAAMetadata(AAInfo);
283 return CI;
284}
285
286CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
287 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
288 const AAMDNodes &AAInfo) {
289
290 Value *Ops[] = {Ptr, Val, Size, getInt32(C: ElementSize)};
291 Type *Tys[] = {Ptr->getType(), Size->getType()};
292
293 auto *CI = cast<AnyMemSetInst>(Val: CreateIntrinsicWithoutFolding(
294 ID: Intrinsic::memset_element_unordered_atomic, OverloadTypes: Tys, Args: Ops));
295 CI->setDestAlignment(Alignment);
296 CI->setAAMetadata(AAInfo);
297 return CI;
298}
299
300CallInst *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
301 MaybeAlign DstAlign, Value *Src,
302 MaybeAlign SrcAlign, Value *Size,
303 bool isVolatile,
304 const AAMDNodes &AAInfo) {
305 assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||
306 IntrID == Intrinsic::memmove) &&
307 "Unexpected intrinsic ID");
308 Value *Ops[] = {Dst, Src, Size, getInt1(V: isVolatile)};
309 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
310
311 auto *MCI =
312 cast<MemTransferInst>(Val: CreateIntrinsicWithoutFolding(ID: IntrID, OverloadTypes: Tys, Args: Ops));
313
314 if (DstAlign)
315 MCI->setDestAlignment(*DstAlign);
316 if (SrcAlign)
317 MCI->setSourceAlignment(*SrcAlign);
318 MCI->setAAMetadata(AAInfo);
319 return MCI;
320}
321
322CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
323 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
324 uint32_t ElementSize, const AAMDNodes &AAInfo) {
325 assert(DstAlign >= ElementSize &&
326 "Pointer alignment must be at least element size");
327 assert(SrcAlign >= ElementSize &&
328 "Pointer alignment must be at least element size");
329 Value *Ops[] = {Dst, Src, Size, getInt32(C: ElementSize)};
330 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
331
332 auto *AMCI = cast<AnyMemCpyInst>(Val: CreateIntrinsicWithoutFolding(
333 ID: Intrinsic::memcpy_element_unordered_atomic, OverloadTypes: Tys, Args: Ops));
334
335 // Set the alignment of the pointer args.
336 AMCI->setDestAlignment(DstAlign);
337 AMCI->setSourceAlignment(SrcAlign);
338 AMCI->setAAMetadata(AAInfo);
339 return AMCI;
340}
341
342/// isConstantOne - Return true only if val is constant int 1
343static bool isConstantOne(const Value *Val) {
344 assert(Val && "isConstantOne does not work with nullptr Val");
345 const ConstantInt *CVal = dyn_cast<ConstantInt>(Val);
346 return CVal && CVal->isOne();
347}
348
349CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Value *AllocSize,
350 Value *ArraySize,
351 ArrayRef<OperandBundleDef> OpB,
352 Function *MallocF, const Twine &Name) {
353 // malloc(type) becomes:
354 // i8* malloc(typeSize)
355 // malloc(type, arraySize) becomes:
356 // i8* malloc(typeSize*arraySize)
357 if (!ArraySize)
358 ArraySize = ConstantInt::get(Ty: IntPtrTy, V: 1);
359 else if (ArraySize->getType() != IntPtrTy)
360 ArraySize = CreateIntCast(V: ArraySize, DestTy: IntPtrTy, isSigned: false);
361
362 if (!isConstantOne(Val: ArraySize)) {
363 if (isConstantOne(Val: AllocSize)) {
364 AllocSize = ArraySize; // Operand * 1 = Operand
365 } else {
366 // Multiply type size by the array size...
367 AllocSize = CreateMul(LHS: ArraySize, RHS: AllocSize, Name: "mallocsize");
368 }
369 }
370
371 assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");
372 // Create the call to Malloc.
373 Module *M = BB->getParent()->getParent();
374 Type *BPTy = PointerType::getUnqual(C&: Context);
375 FunctionCallee MallocFunc = MallocF;
376 if (!MallocFunc)
377 // prototype malloc as "void *malloc(size_t)"
378 MallocFunc = M->getOrInsertFunction(Name: "malloc", RetTy: BPTy, Args: IntPtrTy);
379 CallInst *MCall = CreateCall(Callee: MallocFunc, Args: AllocSize, OpBundles: OpB, Name);
380
381 MCall->setTailCall();
382 if (Function *F = dyn_cast<Function>(Val: MallocFunc.getCallee())) {
383 MCall->setCallingConv(F->getCallingConv());
384 F->setReturnDoesNotAlias();
385 }
386
387 assert(!MCall->getType()->isVoidTy() && "Malloc has void return type");
388
389 return MCall;
390}
391
392CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Value *AllocSize,
393 Value *ArraySize, Function *MallocF,
394 const Twine &Name) {
395
396 return CreateMalloc(IntPtrTy, AllocSize, ArraySize, OpB: {}, MallocF, Name);
397}
398
399/// CreateFree - Generate the IR for a call to the builtin free function.
400CallInst *IRBuilderBase::CreateFree(Value *Source,
401 ArrayRef<OperandBundleDef> Bundles) {
402 assert(Source->getType()->isPointerTy() &&
403 "Can not free something of nonpointer type!");
404
405 Module *M = BB->getParent()->getParent();
406
407 Type *VoidTy = Type::getVoidTy(C&: M->getContext());
408 Type *VoidPtrTy = PointerType::getUnqual(C&: M->getContext());
409 // prototype free as "void free(void*)"
410 FunctionCallee FreeFunc = M->getOrInsertFunction(Name: "free", RetTy: VoidTy, Args: VoidPtrTy);
411 CallInst *Result = CreateCall(Callee: FreeFunc, Args: Source, OpBundles: Bundles, Name: "");
412 Result->setTailCall();
413 if (Function *F = dyn_cast<Function>(Val: FreeFunc.getCallee()))
414 Result->setCallingConv(F->getCallingConv());
415
416 return Result;
417}
418
419CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
420 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
421 uint32_t ElementSize, const AAMDNodes &AAInfo) {
422 assert(DstAlign >= ElementSize &&
423 "Pointer alignment must be at least element size");
424 assert(SrcAlign >= ElementSize &&
425 "Pointer alignment must be at least element size");
426 Value *Ops[] = {Dst, Src, Size, getInt32(C: ElementSize)};
427 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
428
429 CallInst *CI = CreateIntrinsicWithoutFolding(
430 ID: Intrinsic::memmove_element_unordered_atomic, OverloadTypes: Tys, Args: Ops);
431
432 // Set the alignment of the pointer args.
433 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: DstAlign));
434 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: SrcAlign));
435 CI->setAAMetadata(AAInfo);
436 return CI;
437}
438
439Value *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
440 Value *Ops[] = {Src};
441 Type *Tys[] = { Src->getType() };
442 return CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops);
443}
444
445Value *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
446 Value *Ops[] = {Acc, Src};
447 return CreateIntrinsic(ID: Intrinsic::vector_reduce_fadd, OverloadTypes: {Src->getType()}, Args: Ops);
448}
449
450Value *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
451 Value *Ops[] = {Acc, Src};
452 return CreateIntrinsic(ID: Intrinsic::vector_reduce_fmul, OverloadTypes: {Src->getType()}, Args: Ops);
453}
454
455Value *IRBuilderBase::CreateAddReduce(Value *Src) {
456 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_add, Src);
457}
458
459Value *IRBuilderBase::CreateMulReduce(Value *Src) {
460 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_mul, Src);
461}
462
463Value *IRBuilderBase::CreateAndReduce(Value *Src) {
464 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_and, Src);
465}
466
467Value *IRBuilderBase::CreateOrReduce(Value *Src) {
468 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_or, Src);
469}
470
471Value *IRBuilderBase::CreateXorReduce(Value *Src) {
472 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_xor, Src);
473}
474
475Value *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
476 auto ID =
477 IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
478 return getReductionIntrinsic(ID, Src);
479}
480
481Value *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
482 auto ID =
483 IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
484 return getReductionIntrinsic(ID, Src);
485}
486
487Value *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
488 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmax, Src);
489}
490
491Value *IRBuilderBase::CreateFPMinReduce(Value *Src) {
492 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmin, Src);
493}
494
495Value *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
496 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmaximum, Src);
497}
498
499Value *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
500 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fminimum, Src);
501}
502
503Value *IRBuilderBase::CreateFPMaximumNumReduce(Value *Src) {
504 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmaximumnum, Src);
505}
506
507Value *IRBuilderBase::CreateFPMinimumNumReduce(Value *Src) {
508 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fminimumnum, Src);
509}
510
511CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
512 assert(isa<PointerType>(Ptr->getType()) &&
513 "lifetime.start only applies to pointers.");
514 return CreateIntrinsicWithoutFolding(ID: Intrinsic::lifetime_start,
515 OverloadTypes: {Ptr->getType()}, Args: {Ptr});
516}
517
518CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
519 assert(isa<PointerType>(Ptr->getType()) &&
520 "lifetime.end only applies to pointers.");
521 return CreateIntrinsicWithoutFolding(ID: Intrinsic::lifetime_end,
522 OverloadTypes: {Ptr->getType()}, Args: {Ptr});
523}
524
525CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
526
527 assert(isa<PointerType>(Ptr->getType()) &&
528 "invariant.start only applies to pointers.");
529 if (!Size)
530 Size = getInt64(C: -1);
531 else
532 assert(Size->getType() == getInt64Ty() &&
533 "invariant.start requires the size to be an i64");
534
535 Value *Ops[] = {Size, Ptr};
536 // Fill in the single overloaded type: memory object type.
537 Type *ObjectPtr[1] = {Ptr->getType()};
538 return CreateIntrinsicWithoutFolding(ID: Intrinsic::invariant_start, OverloadTypes: ObjectPtr,
539 Args: Ops);
540}
541
542static MaybeAlign getAlign(Value *Ptr) {
543 if (auto *V = dyn_cast<GlobalVariable>(Val: Ptr))
544 return V->getAlign();
545 if (auto *A = dyn_cast<GlobalAlias>(Val: Ptr))
546 return getAlign(Ptr: A->getAliaseeObject());
547 return {};
548}
549
550CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
551 assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&
552 "threadlocal_address only applies to thread local variables.");
553 CallInst *CI = CreateIntrinsicWithoutFolding(
554 ID: llvm::Intrinsic::threadlocal_address, OverloadTypes: {Ptr->getType()}, Args: {Ptr});
555 if (MaybeAlign A = getAlign(Ptr)) {
556 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *A));
557 CI->addRetAttr(Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *A));
558 }
559 return CI;
560}
561
562CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
563 assert(Cond->getType() == getInt1Ty() &&
564 "an assumption condition must be of type i1");
565 return CreateIntrinsicWithoutFolding(ID: Intrinsic::assume, /*OverloadTypes=*/{},
566 Args: {Cond});
567}
568
569CallInst *
570IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
571 Value *Args[] = {ConstantInt::getTrue(Context&: getContext())};
572 return CreateIntrinsicWithoutFolding(
573 ID: Intrinsic::assume, /*OverloadTypes=*/{}, Args,
574 /*FMFSource=*/nullptr, /*Name=*/"", OpBundles);
575}
576
577Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
578 return CreateIntrinsicWithoutFolding(
579 ID: Intrinsic::experimental_noalias_scope_decl, OverloadTypes: {}, Args: {Scope});
580}
581
582/// Create a call to a Masked Load intrinsic.
583/// \p Ty - vector type to load
584/// \p Ptr - base pointer for the load
585/// \p Alignment - alignment of the source location
586/// \p Mask - vector of booleans which indicates what vector lanes should
587/// be accessed in memory
588/// \p PassThru - pass-through value that is used to fill the masked-off lanes
589/// of the result
590/// \p Name - name of the result variable
591CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
592 Value *Mask, Value *PassThru,
593 const Twine &Name) {
594 auto *PtrTy = cast<PointerType>(Val: Ptr->getType());
595 assert(Ty->isVectorTy() && "Type should be vector");
596 assert(Mask && "Mask should not be all-ones (null)");
597 if (!PassThru)
598 PassThru = PoisonValue::get(T: Ty);
599 Type *OverloadedTypes[] = { Ty, PtrTy };
600 Value *Ops[] = {Ptr, Mask, PassThru};
601 CallInst *CI =
602 CreateMaskedIntrinsic(Id: Intrinsic::masked_load, Ops, OverloadedTypes, Name);
603 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
604 return CI;
605}
606
607/// Create a call to a Masked Store intrinsic.
608/// \p Val - data to be stored,
609/// \p Ptr - base pointer for the store
610/// \p Alignment - alignment of the destination location
611/// \p Mask - vector of booleans which indicates what vector lanes should
612/// be accessed in memory
613CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
614 Align Alignment, Value *Mask) {
615 auto *PtrTy = cast<PointerType>(Val: Ptr->getType());
616 Type *DataTy = Val->getType();
617 assert(DataTy->isVectorTy() && "Val should be a vector");
618 assert(Mask && "Mask should not be all-ones (null)");
619 Type *OverloadedTypes[] = { DataTy, PtrTy };
620 Value *Ops[] = {Val, Ptr, Mask};
621 CallInst *CI =
622 CreateMaskedIntrinsic(Id: Intrinsic::masked_store, Ops, OverloadedTypes);
623 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
624 return CI;
625}
626
627/// Create a call to a Masked intrinsic, with given intrinsic Id,
628/// an array of operands - Ops, and an array of overloaded types -
629/// OverloadedTypes.
630CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
631 ArrayRef<Value *> Ops,
632 ArrayRef<Type *> OverloadedTypes,
633 const Twine &Name) {
634 return CreateIntrinsicWithoutFolding(ID: Id, OverloadTypes: OverloadedTypes, Args: Ops, FMFSource: {}, Name);
635}
636
637/// Create a call to a Masked Gather intrinsic.
638/// \p Ty - vector type to gather
639/// \p Ptrs - vector of pointers for loading
640/// \p Align - alignment for one element
641/// \p Mask - vector of booleans which indicates what vector lanes should
642/// be accessed in memory
643/// \p PassThru - pass-through value that is used to fill the masked-off lanes
644/// of the result
645/// \p Name - name of the result variable
646CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
647 Align Alignment, Value *Mask,
648 Value *PassThru,
649 const Twine &Name) {
650 auto *VecTy = cast<VectorType>(Val: Ty);
651 ElementCount NumElts = VecTy->getElementCount();
652 auto *PtrsTy = cast<VectorType>(Val: Ptrs->getType());
653 assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
654
655 if (!Mask)
656 Mask = getAllOnesMask(NumElts);
657
658 if (!PassThru)
659 PassThru = PoisonValue::get(T: Ty);
660
661 Type *OverloadedTypes[] = {Ty, PtrsTy};
662 Value *Ops[] = {Ptrs, Mask, PassThru};
663
664 // We specify only one type when we create this intrinsic. Types of other
665 // arguments are derived from this type.
666 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_gather, Ops,
667 OverloadedTypes, Name);
668 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
669 return CI;
670}
671
672/// Create a call to a Masked Scatter intrinsic.
673/// \p Data - data to be stored,
674/// \p Ptrs - the vector of pointers, where the \p Data elements should be
675/// stored
676/// \p Align - alignment for one element
677/// \p Mask - vector of booleans which indicates what vector lanes should
678/// be accessed in memory
679CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
680 Align Alignment, Value *Mask) {
681 auto *PtrsTy = cast<VectorType>(Val: Ptrs->getType());
682 auto *DataTy = cast<VectorType>(Val: Data->getType());
683 ElementCount NumElts = PtrsTy->getElementCount();
684
685 if (!Mask)
686 Mask = getAllOnesMask(NumElts);
687
688 Type *OverloadedTypes[] = {DataTy, PtrsTy};
689 Value *Ops[] = {Data, Ptrs, Mask};
690
691 // We specify only one type when we create this intrinsic. Types of other
692 // arguments are derived from this type.
693 CallInst *CI =
694 CreateMaskedIntrinsic(Id: Intrinsic::masked_scatter, Ops, OverloadedTypes);
695 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
696 return CI;
697}
698
699/// Create a call to Masked Expand Load intrinsic
700/// \p Ty - vector type to load
701/// \p Ptr - base pointer for the load
702/// \p Align - alignment of \p Ptr
703/// \p Mask - vector of booleans which indicates what vector lanes should
704/// be accessed in memory
705/// \p PassThru - pass-through value that is used to fill the masked-off lanes
706/// of the result
707/// \p Name - name of the result variable
708CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
709 MaybeAlign Align, Value *Mask,
710 Value *PassThru,
711 const Twine &Name) {
712 assert(Ty->isVectorTy() && "Type should be vector");
713 assert(Mask && "Mask should not be all-ones (null)");
714 if (!PassThru)
715 PassThru = PoisonValue::get(T: Ty);
716 Type *PtrTy = Ptr->getType();
717 Type *OverloadedTypes[] = {Ty, PtrTy};
718 Value *Ops[] = {Ptr, Mask, PassThru};
719 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_expandload, Ops,
720 OverloadedTypes, Name);
721 if (Align)
722 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *Align));
723 return CI;
724}
725
726/// Create a call to Masked Compress Store intrinsic
727/// \p Val - data to be stored,
728/// \p Ptr - base pointer for the store
729/// \p Align - alignment of \p Ptr
730/// \p Mask - vector of booleans which indicates what vector lanes should
731/// be accessed in memory
732CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
733 MaybeAlign Align,
734 Value *Mask) {
735 Type *DataTy = Val->getType();
736 assert(DataTy->isVectorTy() && "Val should be a vector");
737 assert(Mask && "Mask should not be all-ones (null)");
738 Type *PtrTy = Ptr->getType();
739 Type *OverloadedTypes[] = {DataTy, PtrTy};
740 Value *Ops[] = {Val, Ptr, Mask};
741 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_compressstore, Ops,
742 OverloadedTypes);
743 if (Align)
744 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *Align));
745 return CI;
746}
747
748template <typename T0>
749static std::vector<Value *>
750getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
751 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
752 std::vector<Value *> Args;
753 Args.push_back(x: B.getInt64(C: ID));
754 Args.push_back(x: B.getInt32(C: NumPatchBytes));
755 Args.push_back(x: ActualCallee);
756 Args.push_back(B.getInt32(C: CallArgs.size()));
757 Args.push_back(x: B.getInt32(C: Flags));
758 llvm::append_range(Args, CallArgs);
759 // GC Transition and Deopt args are now always handled via operand bundle.
760 // They will be removed from the signature of gc.statepoint shortly.
761 Args.push_back(x: B.getInt32(C: 0));
762 Args.push_back(x: B.getInt32(C: 0));
763 // GC args are now encoded in the gc-live operand bundle
764 return Args;
765}
766
767template<typename T1, typename T2, typename T3>
768static std::vector<OperandBundleDef>
769getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
770 std::optional<ArrayRef<T2>> DeoptArgs,
771 ArrayRef<T3> GCArgs) {
772 std::vector<OperandBundleDef> Rval;
773 if (DeoptArgs)
774 Rval.emplace_back(args: "deopt", args: SmallVector<Value *, 16>(*DeoptArgs));
775 if (TransitionArgs)
776 Rval.emplace_back(args: "gc-transition",
777 args: SmallVector<Value *, 16>(*TransitionArgs));
778 if (GCArgs.size())
779 Rval.emplace_back(args: "gc-live", args: SmallVector<Value *, 16>(GCArgs));
780 return Rval;
781}
782
783template <typename T0, typename T1, typename T2, typename T3>
784static CallInst *CreateGCStatepointCallCommon(
785 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
786 FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
787 std::optional<ArrayRef<T1>> TransitionArgs,
788 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
789 const Twine &Name) {
790 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
791 // Fill in the one generic type'd argument (the function is also vararg)
792 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
793 M, id: Intrinsic::experimental_gc_statepoint,
794 OverloadTys: {ActualCallee.getCallee()->getType()});
795
796 std::vector<Value *> Args = getStatepointArgs(
797 *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
798
799 CallInst *CI = Builder->CreateCall(
800 FnStatepoint, Args,
801 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
802 CI->addParamAttr(ArgNo: 2,
803 Attr: Attribute::get(Context&: Builder->getContext(), Kind: Attribute::ElementType,
804 Ty: ActualCallee.getFunctionType()));
805 return CI;
806}
807
808CallInst *IRBuilderBase::CreateGCStatepointCall(
809 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
810 ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
811 ArrayRef<Value *> GCArgs, const Twine &Name) {
812 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
813 Builder: this, ID, NumPatchBytes, ActualCallee, Flags: uint32_t(StatepointFlags::None),
814 CallArgs, TransitionArgs: std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
815}
816
817CallInst *IRBuilderBase::CreateGCStatepointCall(
818 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
819 uint32_t Flags, ArrayRef<Value *> CallArgs,
820 std::optional<ArrayRef<Use>> TransitionArgs,
821 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
822 const Twine &Name) {
823 return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
824 Builder: this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
825 DeoptArgs, GCArgs, Name);
826}
827
828CallInst *IRBuilderBase::CreateGCStatepointCall(
829 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
830 ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
831 ArrayRef<Value *> GCArgs, const Twine &Name) {
832 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
833 Builder: this, ID, NumPatchBytes, ActualCallee, Flags: uint32_t(StatepointFlags::None),
834 CallArgs, TransitionArgs: std::nullopt, DeoptArgs, GCArgs, Name);
835}
836
837template <typename T0, typename T1, typename T2, typename T3>
838static InvokeInst *CreateGCStatepointInvokeCommon(
839 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
840 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
841 BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
842 std::optional<ArrayRef<T1>> TransitionArgs,
843 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
844 const Twine &Name) {
845 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
846 // Fill in the one generic type'd argument (the function is also vararg)
847 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
848 M, id: Intrinsic::experimental_gc_statepoint,
849 OverloadTys: {ActualInvokee.getCallee()->getType()});
850
851 std::vector<Value *> Args =
852 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
853 Flags, InvokeArgs);
854
855 InvokeInst *II = Builder->CreateInvoke(
856 FnStatepoint, NormalDest, UnwindDest, Args,
857 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
858 II->addParamAttr(ArgNo: 2,
859 Attr: Attribute::get(Context&: Builder->getContext(), Kind: Attribute::ElementType,
860 Ty: ActualInvokee.getFunctionType()));
861 return II;
862}
863
864InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
865 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
866 BasicBlock *NormalDest, BasicBlock *UnwindDest,
867 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
868 ArrayRef<Value *> GCArgs, const Twine &Name) {
869 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
870 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
871 Flags: uint32_t(StatepointFlags::None), InvokeArgs,
872 TransitionArgs: std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
873}
874
875InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
876 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
877 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
878 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
879 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
880 const Twine &Name) {
881 return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
882 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
883 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
884}
885
886InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
887 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
888 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
889 std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
890 const Twine &Name) {
891 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
892 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
893 Flags: uint32_t(StatepointFlags::None), InvokeArgs, TransitionArgs: std::nullopt, DeoptArgs,
894 GCArgs, Name);
895}
896
897CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
898 Type *ResultType, const Twine &Name) {
899 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
900 Type *Types[] = {ResultType};
901
902 Value *Args[] = {Statepoint};
903 return CreateIntrinsicWithoutFolding(ID, OverloadTypes: Types, Args, FMFSource: {}, Name);
904}
905
906CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
907 int BaseOffset, int DerivedOffset,
908 Type *ResultType, const Twine &Name) {
909 Type *Types[] = {ResultType};
910
911 Value *Args[] = {Statepoint, getInt32(C: BaseOffset), getInt32(C: DerivedOffset)};
912 return CreateIntrinsicWithoutFolding(ID: Intrinsic::experimental_gc_relocate,
913 OverloadTypes: Types, Args, FMFSource: {}, Name);
914}
915
916CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
917 const Twine &Name) {
918 Type *PtrTy = DerivedPtr->getType();
919 return CreateIntrinsicWithoutFolding(
920 ID: Intrinsic::experimental_gc_get_pointer_base, OverloadTypes: PtrTy, Args: DerivedPtr, FMFSource: {}, Name);
921}
922
923CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
924 const Twine &Name) {
925 Type *PtrTy = DerivedPtr->getType();
926 return CreateIntrinsicWithoutFolding(
927 ID: Intrinsic::experimental_gc_get_pointer_offset, OverloadTypes: {PtrTy}, Args: {DerivedPtr}, FMFSource: {},
928 Name);
929}
930
931Value *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op,
932 FMFSource FMFSource,
933 const Twine &Name) {
934 Module *M = BB->getModule();
935 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: Op->getType());
936 if (Value *V =
937 Folder.FoldIntrinsic(ID, Ops: Op, Ty: Fn->getReturnType(), FMF: FMFSource.get(Default: FMF),
938 CtxF: GetInsertBlock()->getParent()))
939 return V;
940 return createCallHelper(Callee: Fn, Ops: Op, Name, FMFSource);
941}
942
943Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
944 Value *RHS, FMFSource FMFSource,
945 const Twine &Name) {
946 Module *M = BB->getModule();
947 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: {LHS->getType()});
948 if (Value *V = Folder.FoldIntrinsic(ID, Ops: {LHS, RHS}, Ty: Fn->getReturnType(),
949 FMF: FMFSource.get(Default: FMF),
950 CtxF: GetInsertBlock()->getParent()))
951 return V;
952 return createCallHelper(Callee: Fn, Ops: {LHS, RHS}, Name, FMFSource);
953}
954
955CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(
956 Intrinsic::ID ID, ArrayRef<Type *> OverloadTypes, ArrayRef<Value *> Args,
957 FMFSource FMFSource, const Twine &Name,
958 ArrayRef<OperandBundleDef> OpBundles) {
959 Module *M = BB->getModule();
960 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: OverloadTypes);
961 return createCallHelper(Callee: Fn, Ops: Args, Name, FMFSource, OpBundles);
962}
963
964CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(Type *RetTy,
965 Intrinsic::ID ID,
966 ArrayRef<Value *> Args,
967 FMFSource FMFSource,
968 const Twine &Name) {
969 Module *M = BB->getModule();
970 SmallVector<Type *> ArgTys = llvm::map_to_vector(C&: Args, F: &Value::getType);
971 Function *Fn = Intrinsic::getOrInsertDeclaration(M, IID: ID, RetTy, ArgTys);
972 return createCallHelper(Callee: Fn, Ops: Args, Name, FMFSource);
973}
974
975Value *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
976 ArrayRef<Type *> OverloadTypes,
977 ArrayRef<Value *> Args,
978 FMFSource FMFSource, const Twine &Name,
979 ArrayRef<OperandBundleDef> OpBundles,
980 function_ref<void(CallInst *)> SetFn) {
981 Type *RetTy = Intrinsic::getType(Context, id: ID, OverloadTys: OverloadTypes)->getReturnType();
982 if (Value *V = Folder.FoldIntrinsic(ID, Ops: Args, Ty: RetTy, FMF: FMFSource.get(Default: FMF),
983 CtxF: GetInsertBlock()->getParent()))
984 return V;
985 CallInst *CI = CreateIntrinsicWithoutFolding(ID, OverloadTypes, Args,
986 FMFSource, Name, OpBundles);
987 SetFn(CI);
988 return CI;
989}
990
991Value *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
992 ArrayRef<Value *> Args,
993 FMFSource FMFSource, const Twine &Name,
994 function_ref<void(CallInst *)> SetFn) {
995 if (Value *V = Folder.FoldIntrinsic(ID, Ops: Args, Ty: RetTy, FMF: FMFSource.get(Default: FMF),
996 CtxF: GetInsertBlock()->getParent()))
997 return V;
998 CallInst *CI =
999 CreateIntrinsicWithoutFolding(RetTy, ID, Args, FMFSource, Name);
1000 SetFn(CI);
1001 return CI;
1002}
1003
1004CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
1005 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
1006 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
1007 std::optional<fp::ExceptionBehavior> Except) {
1008 Value *RoundingV = getConstrainedFPRounding(Rounding);
1009 Value *ExceptV = getConstrainedFPExcept(Except);
1010
1011 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1012 CallInst *C = CreateIntrinsicWithoutFolding(
1013 ID, OverloadTypes: {L->getType()}, Args: {L, R, RoundingV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1014 setConstrainedFPCallAttr(C);
1015 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1016 return C;
1017}
1018
1019CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
1020 Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
1021 FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag,
1022 std::optional<RoundingMode> Rounding,
1023 std::optional<fp::ExceptionBehavior> Except) {
1024 Value *RoundingV = getConstrainedFPRounding(Rounding);
1025 Value *ExceptV = getConstrainedFPExcept(Except);
1026
1027 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1028
1029 llvm::SmallVector<Value *, 5> ExtArgs(Args);
1030 ExtArgs.push_back(Elt: RoundingV);
1031 ExtArgs.push_back(Elt: ExceptV);
1032 CallInst *C =
1033 CreateIntrinsicWithoutFolding(ID, OverloadTypes: Types, Args: ExtArgs, FMFSource: nullptr, Name, OpBundles: {});
1034 setConstrainedFPCallAttr(C);
1035 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1036 return C;
1037}
1038
1039CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
1040 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
1041 const Twine &Name, MDNode *FPMathTag,
1042 std::optional<fp::ExceptionBehavior> Except) {
1043 Value *ExceptV = getConstrainedFPExcept(Except);
1044
1045 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1046 CallInst *C = CreateIntrinsicWithoutFolding(
1047 ID, OverloadTypes: {L->getType()}, Args: {L, R, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1048 setConstrainedFPCallAttr(C);
1049 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1050 return C;
1051}
1052
1053Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1054 const Twine &Name, MDNode *FPMathTag) {
1055 if (Instruction::isBinaryOp(Opcode: Opc)) {
1056 assert(Ops.size() == 2 && "Invalid number of operands!");
1057 return CreateBinOp(Opc: static_cast<Instruction::BinaryOps>(Opc),
1058 LHS: Ops[0], RHS: Ops[1], Name, FPMathTag);
1059 }
1060 if (Instruction::isUnaryOp(Opcode: Opc)) {
1061 assert(Ops.size() == 1 && "Invalid number of operands!");
1062 return CreateUnOp(Opc: static_cast<Instruction::UnaryOps>(Opc),
1063 V: Ops[0], Name, FPMathTag);
1064 }
1065 llvm_unreachable("Unexpected opcode!");
1066}
1067
1068CallInst *IRBuilderBase::CreateConstrainedFPCast(
1069 Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource,
1070 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
1071 std::optional<fp::ExceptionBehavior> Except) {
1072 Value *ExceptV = getConstrainedFPExcept(Except);
1073
1074 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1075
1076 CallInst *C;
1077 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: ID)) {
1078 Value *RoundingV = getConstrainedFPRounding(Rounding);
1079 C = CreateIntrinsicWithoutFolding(
1080 ID, OverloadTypes: {DestTy, V->getType()}, Args: {V, RoundingV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1081 } else
1082 C = CreateIntrinsicWithoutFolding(ID, OverloadTypes: {DestTy, V->getType()}, Args: {V, ExceptV},
1083 FMFSource: nullptr, Name, OpBundles: {});
1084 setConstrainedFPCallAttr(C);
1085
1086 if (isa<FPMathOperator>(Val: C))
1087 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1088 return C;
1089}
1090
1091Value *IRBuilderBase::CreateFCmpHelper(CmpInst::Predicate P, Value *LHS,
1092 Value *RHS, const Twine &Name,
1093 MDNode *FPMathTag, FMFSource FMFSource,
1094 bool IsSignaling) {
1095 if (IsFPConstrained) {
1096 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
1097 : Intrinsic::experimental_constrained_fcmp;
1098 return CreateConstrainedFPCmp(ID, P, L: LHS, R: RHS, Name);
1099 }
1100
1101 if (auto *V = Folder.FoldCmp(P, LHS, RHS))
1102 return V;
1103 return Insert(
1104 I: setFPAttrs(I: new FCmpInst(P, LHS, RHS), FPMD: FPMathTag, FMF: FMFSource.get(Default: FMF)),
1105 Name);
1106}
1107
1108CallInst *IRBuilderBase::CreateConstrainedFPCmp(
1109 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
1110 const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
1111 Value *PredicateV = getConstrainedFPPredicate(Predicate: P);
1112 Value *ExceptV = getConstrainedFPExcept(Except);
1113
1114 CallInst *C = CreateIntrinsicWithoutFolding(
1115 ID, OverloadTypes: {L->getType()}, Args: {L, R, PredicateV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1116 setConstrainedFPCallAttr(C);
1117 return C;
1118}
1119
1120CallInst *IRBuilderBase::CreateConstrainedFPCall(
1121 Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
1122 std::optional<RoundingMode> Rounding,
1123 std::optional<fp::ExceptionBehavior> Except) {
1124 llvm::SmallVector<Value *, 6> UseArgs(Args);
1125
1126 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: Callee->getIntrinsicID()))
1127 UseArgs.push_back(Elt: getConstrainedFPRounding(Rounding));
1128 UseArgs.push_back(Elt: getConstrainedFPExcept(Except));
1129
1130 CallInst *C = CreateCall(Callee, Args: UseArgs, Name);
1131 setConstrainedFPCallAttr(C);
1132 return C;
1133}
1134
1135Value *IRBuilderBase::CreateSelectWithUnknownProfile(Value *C, Value *True,
1136 Value *False,
1137 StringRef PassName,
1138 const Twine &Name) {
1139 Value *Ret = CreateSelectFMF(C, True, False, FMFSource: {}, Name);
1140 if (auto *SI = dyn_cast<SelectInst>(Val: Ret)) {
1141 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, PassName);
1142 }
1143 return Ret;
1144}
1145
1146Value *IRBuilderBase::CreateSelectFMFWithUnknownProfile(Value *C, Value *True,
1147 Value *False,
1148 FMFSource FMFSource,
1149 StringRef PassName,
1150 const Twine &Name) {
1151 Value *Ret = CreateSelectFMF(C, True, False, FMFSource, Name);
1152 if (auto *SI = dyn_cast<SelectInst>(Val: Ret))
1153 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, PassName);
1154 return Ret;
1155}
1156
1157Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
1158 const Twine &Name, Instruction *MDFrom) {
1159 return CreateSelectFMF(C, True, False, FMFSource: {}, Name, MDFrom);
1160}
1161
1162Value *IRBuilderBase::CreateSelectFMF(Value *C, Value *True, Value *False,
1163 FMFSource FMFSource, const Twine &Name,
1164 Instruction *MDFrom) {
1165 if (auto *V = Folder.FoldSelect(C, True, False, FMF: FMFSource.get(Default: FMF)))
1166 return V;
1167
1168 SelectInst *Sel = SelectInst::Create(C, S1: True, S2: False);
1169 if (MDFrom) {
1170 MDNode *Prof = MDFrom->getMetadata(KindID: LLVMContext::MD_prof);
1171 MDNode *Unpred = MDFrom->getMetadata(KindID: LLVMContext::MD_unpredictable);
1172 Sel = addBranchMetadata(I: Sel, Weights: Prof, Unpredictable: Unpred);
1173 }
1174 if (isa<FPMathOperator>(Val: Sel))
1175 setFPAttrs(I: Sel, /*MDNode=*/FPMD: nullptr, FMF: FMFSource.get(Default: FMF));
1176 return Insert(I: Sel, Name);
1177}
1178
1179Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name,
1180 bool IsNUW) {
1181 assert(LHS->getType() == RHS->getType() &&
1182 "Pointer subtraction operand types must match!");
1183 Value *LHSAddr = CreatePtrToAddr(V: LHS);
1184 Value *RHSAddr = CreatePtrToAddr(V: RHS);
1185 return CreateSub(LHS: LHSAddr, RHS: RHSAddr, Name, HasNUW: IsNUW);
1186}
1187Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
1188 const Twine &Name) {
1189 const DataLayout &DL = BB->getDataLayout();
1190 TypeSize ElemSize = DL.getTypeAllocSize(Ty: ElemTy);
1191 if (ElemSize == TypeSize::getFixed(ExactSize: 1))
1192 return CreatePtrDiff(LHS, RHS, Name);
1193
1194 Value *Diff = CreatePtrDiff(LHS, RHS);
1195 return CreateExactSDiv(LHS: Diff, RHS: CreateTypeSize(Ty: Diff->getType(), Size: ElemSize), Name);
1196}
1197
1198Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
1199 assert(isa<PointerType>(Ptr->getType()) &&
1200 "launder.invariant.group only applies to pointers.");
1201 auto *PtrType = Ptr->getType();
1202 Module *M = BB->getParent()->getParent();
1203 Function *FnLaunderInvariantGroup = Intrinsic::getOrInsertDeclaration(
1204 M, id: Intrinsic::launder_invariant_group, OverloadTys: {PtrType});
1205
1206 assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&
1207 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1208 PtrType &&
1209 "LaunderInvariantGroup should take and return the same type");
1210
1211 return CreateCall(Callee: FnLaunderInvariantGroup, Args: {Ptr});
1212}
1213
1214Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
1215 assert(isa<PointerType>(Ptr->getType()) &&
1216 "strip.invariant.group only applies to pointers.");
1217
1218 auto *PtrType = Ptr->getType();
1219 Module *M = BB->getParent()->getParent();
1220 Function *FnStripInvariantGroup = Intrinsic::getOrInsertDeclaration(
1221 M, id: Intrinsic::strip_invariant_group, OverloadTys: {PtrType});
1222
1223 assert(FnStripInvariantGroup->getReturnType() == PtrType &&
1224 FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1225 PtrType &&
1226 "StripInvariantGroup should take and return the same type");
1227
1228 return CreateCall(Callee: FnStripInvariantGroup, Args: {Ptr});
1229}
1230
1231Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1232 auto *Ty = cast<VectorType>(Val: V->getType());
1233 if (isa<ScalableVectorType>(Val: Ty)) {
1234 Module *M = BB->getParent()->getParent();
1235 Function *F =
1236 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::vector_reverse, OverloadTys: Ty);
1237 return Insert(I: CallInst::Create(Func: F, Args: V), Name);
1238 }
1239 // Keep the original behaviour for fixed vector
1240 SmallVector<int, 8> ShuffleMask;
1241 int NumElts = Ty->getElementCount().getKnownMinValue();
1242 for (int i = 0; i < NumElts; ++i)
1243 ShuffleMask.push_back(Elt: NumElts - i - 1);
1244 return CreateShuffleVector(V, Mask: ShuffleMask, Name);
1245}
1246
1247static SmallVector<int, 8> getSpliceMask(int64_t Imm, unsigned NumElts) {
1248 unsigned Idx = (NumElts + Imm) % NumElts;
1249 SmallVector<int, 8> Mask;
1250 for (unsigned I = 0; I < NumElts; ++I)
1251 Mask.push_back(Elt: Idx + I);
1252 return Mask;
1253}
1254
1255Value *IRBuilderBase::CreateVectorSpliceLeft(Value *V1, Value *V2,
1256 Value *Offset, const Twine &Name) {
1257 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1258 assert(V1->getType() == V2->getType() &&
1259 "Splice expects matching operand types!");
1260
1261 // Emit a shufflevector for fixed vectors with a constant offset
1262 if (auto *COffset = dyn_cast<ConstantInt>(Val: Offset))
1263 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType()))
1264 return CreateShuffleVector(
1265 V1, V2,
1266 Mask: getSpliceMask(Imm: COffset->getZExtValue(), NumElts: FVTy->getNumElements()));
1267
1268 return CreateIntrinsic(ID: Intrinsic::vector_splice_left, OverloadTypes: V1->getType(),
1269 Args: {V1, V2, Offset}, FMFSource: {}, Name);
1270}
1271
1272Value *IRBuilderBase::CreateVectorSpliceRight(Value *V1, Value *V2,
1273 Value *Offset,
1274 const Twine &Name) {
1275 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1276 assert(V1->getType() == V2->getType() &&
1277 "Splice expects matching operand types!");
1278
1279 // Emit a shufflevector for fixed vectors with a constant offset
1280 if (auto *COffset = dyn_cast<ConstantInt>(Val: Offset))
1281 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType()))
1282 return CreateShuffleVector(
1283 V1, V2,
1284 Mask: getSpliceMask(Imm: -COffset->getZExtValue(), NumElts: FVTy->getNumElements()));
1285
1286 return CreateIntrinsic(ID: Intrinsic::vector_splice_right, OverloadTypes: V1->getType(),
1287 Args: {V1, V2, Offset}, FMFSource: {}, Name);
1288}
1289
1290Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1291 const Twine &Name) {
1292 auto EC = ElementCount::getFixed(MinVal: NumElts);
1293 return CreateVectorSplat(EC, V, Name);
1294}
1295
1296Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1297 const Twine &Name) {
1298 assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1299
1300 // First insert it into a poison vector so we can shuffle it.
1301 Value *Poison = PoisonValue::get(T: VectorType::get(ElementType: V->getType(), EC));
1302 V = CreateInsertElement(Vec: Poison, NewElt: V, Idx: getInt64(C: 0), Name: Name + ".splatinsert");
1303
1304 // Shuffle the value across the desired number of elements.
1305 SmallVector<int, 16> Zeros;
1306 Zeros.resize(N: EC.getKnownMinValue());
1307 return CreateShuffleVector(V, Mask: Zeros, Name: Name + ".splat");
1308}
1309
1310Value *IRBuilderBase::CreateVectorInterleave(ArrayRef<Value *> Ops,
1311 const Twine &Name) {
1312 assert(Ops.size() >= 2 && Ops.size() <= 8 &&
1313 "Unexpected number of operands to interleave");
1314
1315 // Make sure all operands are the same type.
1316 assert(isa<VectorType>(Ops[0]->getType()) && "Unexpected type");
1317
1318#ifndef NDEBUG
1319 for (unsigned I = 1; I < Ops.size(); I++) {
1320 assert(Ops[I]->getType() == Ops[0]->getType() &&
1321 "Vector interleave expects matching operand types!");
1322 }
1323#endif
1324
1325 unsigned IID = Intrinsic::getInterleaveIntrinsicID(Factor: Ops.size());
1326 auto *SubvecTy = cast<VectorType>(Val: Ops[0]->getType());
1327 Type *DestTy = VectorType::get(ElementType: SubvecTy->getElementType(),
1328 EC: SubvecTy->getElementCount() * Ops.size());
1329 return CreateIntrinsic(ID: IID, OverloadTypes: {DestTy}, Args: Ops, FMFSource: {}, Name);
1330}
1331
1332Value *IRBuilderBase::CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
1333 unsigned Dimension,
1334 unsigned LastIndex,
1335 MDNode *DbgInfo) {
1336 auto *BaseType = Base->getType();
1337 assert(isa<PointerType>(BaseType) &&
1338 "Invalid Base ptr type for preserve.array.access.index.");
1339
1340 Value *LastIndexV = getInt32(C: LastIndex);
1341 Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 0);
1342 SmallVector<Value *, 4> IdxList(Dimension, Zero);
1343 IdxList.push_back(Elt: LastIndexV);
1344
1345 Type *ResultType = GetElementPtrInst::getGEPReturnType(Ptr: Base, IdxList);
1346
1347 Value *DimV = getInt32(C: Dimension);
1348 CallInst *Fn = CreateIntrinsicWithoutFolding(
1349 ID: Intrinsic::preserve_array_access_index, OverloadTypes: {ResultType, BaseType},
1350 Args: {Base, DimV, LastIndexV});
1351 Fn->addParamAttr(
1352 ArgNo: 0, Attr: Attribute::get(Context&: Fn->getContext(), Kind: Attribute::ElementType, Ty: ElTy));
1353 if (DbgInfo)
1354 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1355
1356 return Fn;
1357}
1358
1359Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1360 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1361 assert(isa<PointerType>(Base->getType()) &&
1362 "Invalid Base ptr type for preserve.union.access.index.");
1363 auto *BaseType = Base->getType();
1364
1365 Value *DIIndex = getInt32(C: FieldIndex);
1366 CallInst *Fn =
1367 CreateIntrinsicWithoutFolding(ID: Intrinsic::preserve_union_access_index,
1368 OverloadTypes: {BaseType, BaseType}, Args: {Base, DIIndex});
1369 if (DbgInfo)
1370 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1371
1372 return Fn;
1373}
1374
1375Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1376 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1377 MDNode *DbgInfo) {
1378 auto *BaseType = Base->getType();
1379 assert(isa<PointerType>(BaseType) &&
1380 "Invalid Base ptr type for preserve.struct.access.index.");
1381
1382 Value *GEPIndex = getInt32(C: Index);
1383 Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 0);
1384 Type *ResultType =
1385 GetElementPtrInst::getGEPReturnType(Ptr: Base, IdxList: {Zero, GEPIndex});
1386
1387 Value *DIIndex = getInt32(C: FieldIndex);
1388 CallInst *Fn = CreateIntrinsicWithoutFolding(
1389 ID: Intrinsic::preserve_struct_access_index, OverloadTypes: {ResultType, BaseType},
1390 Args: {Base, GEPIndex, DIIndex});
1391 Fn->addParamAttr(
1392 ArgNo: 0, Attr: Attribute::get(Context&: Fn->getContext(), Kind: Attribute::ElementType, Ty: ElTy));
1393 if (DbgInfo)
1394 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1395
1396 return Fn;
1397}
1398
1399Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
1400 ConstantInt *TestV = getInt32(C: Test);
1401 return CreateIntrinsic(ID: Intrinsic::is_fpclass, OverloadTypes: {FPNum->getType()},
1402 Args: {FPNum, TestV});
1403}
1404
1405CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1406 Value *PtrValue,
1407 Value *AlignValue,
1408 Value *OffsetValue) {
1409 SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1410 if (OffsetValue)
1411 Vals.push_back(Elt: OffsetValue);
1412 OperandBundleDefT<Value *> AlignOpB("align", Vals);
1413 return CreateAssumption(OpBundles: {AlignOpB});
1414}
1415
1416CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1417 Value *PtrValue,
1418 uint64_t Alignment,
1419 Value *OffsetValue) {
1420 assert(isa<PointerType>(PtrValue->getType()) &&
1421 "trying to create an alignment assumption on a non-pointer?");
1422 assert(Alignment != 0 && "Invalid Alignment");
1423 Value *AlignValue = ConstantInt::get(Ty: getInt64Ty(), V: Alignment);
1424 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1425}
1426
1427CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1428 Value *PtrValue,
1429 Value *Alignment,
1430 Value *OffsetValue) {
1431 assert(isa<PointerType>(PtrValue->getType()) &&
1432 "trying to create an alignment assumption on a non-pointer?");
1433 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue: Alignment, OffsetValue);
1434}
1435
1436CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
1437 Value *SizeValue) {
1438 assert(isa<PointerType>(PtrValue->getType()) &&
1439 "trying to create a deferenceable assumption on a non-pointer?");
1440 SmallVector<Value *, 4> Vals({PtrValue, SizeValue});
1441 OperandBundleDefT<Value *> DereferenceableOpB("dereferenceable", Vals);
1442 return CreateAssumption(OpBundles: {DereferenceableOpB});
1443}
1444
1445CallInst *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
1446 assert(isa<PointerType>(PtrValue->getType()) &&
1447 "trying to create a nonnull assumption on a non-pointer?");
1448 return CreateAssumption(OpBundles: OperandBundleDef("nonnull", PtrValue));
1449}
1450
1451IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default;
1452IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default;
1453IRBuilderFolder::~IRBuilderFolder() = default;
1454void ConstantFolder::anchor() {}
1455void NoFolder::anchor() {}
1456