1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallVectorExtras.h"
17#include "llvm/IR/Constant.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/DerivedTypes.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GetElementPtrTypeIterator.h"
22#include "llvm/IR/GlobalValue.h"
23#include "llvm/IR/GlobalVariable.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/NoFolder.h"
29#include "llvm/IR/Operator.h"
30#include "llvm/IR/ProfDataUtils.h"
31#include "llvm/IR/Statepoint.h"
32#include "llvm/IR/Type.h"
33#include "llvm/IR/Value.h"
34#include "llvm/Support/Casting.h"
35#include <cassert>
36#include <cstdint>
37#include <optional>
38#include <vector>
39
40using namespace llvm;
41
42/// CreateGlobalString - Make a new global variable with an initializer that
43/// has array of i8 type filled in with the nul terminated string value
44/// specified. If Name is specified, it is the name of the global variable
45/// created.
46GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
47 const Twine &Name,
48 unsigned AddressSpace,
49 Module *M, bool AddNull) {
50 Constant *StrConstant = ConstantDataArray::getString(Context, Initializer: Str, AddNull);
51 if (!M)
52 M = getModule();
53 auto *GV = new GlobalVariable(
54 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
55 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
56 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
57 GV->setAlignment(M->getDataLayout().getPrefTypeAlign(Ty: getInt8Ty()));
58 return GV;
59}
60
61Type *IRBuilderBase::getCurrentFunctionReturnType() const {
62 BasicBlock *BB = GetInsertBlock();
63 assert(BB && BB->getParent() && "No current function!");
64 return BB->getParent()->getReturnType();
65}
66
67DebugLoc IRBuilderBase::getCurrentDebugLocation() const { return StoredDL; }
68void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
69 // We prefer to set our current debug location if any has been set, but if
70 // our debug location is empty and I has a valid location, we shouldn't
71 // overwrite it.
72 I->setDebugLoc(StoredDL.orElse(Other: I->getDebugLoc()));
73}
74
75Value *IRBuilderBase::CreateAggregateCast(Value *V, Type *DestTy) {
76 Type *SrcTy = V->getType();
77 if (SrcTy == DestTy)
78 return V;
79
80 if (SrcTy->isAggregateType()) {
81 unsigned NumElements;
82 if (SrcTy->isStructTy()) {
83 assert(DestTy->isStructTy() && "Expected StructType");
84 assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements() &&
85 "Expected StructTypes with equal number of elements");
86 NumElements = SrcTy->getStructNumElements();
87 } else {
88 assert(SrcTy->isArrayTy() && DestTy->isArrayTy() && "Expected ArrayType");
89 assert(SrcTy->getArrayNumElements() == DestTy->getArrayNumElements() &&
90 "Expected ArrayTypes with equal number of elements");
91 NumElements = SrcTy->getArrayNumElements();
92 }
93
94 Value *Result = PoisonValue::get(T: DestTy);
95 for (unsigned I = 0; I < NumElements; ++I) {
96 Type *ElementTy = SrcTy->isStructTy() ? DestTy->getStructElementType(N: I)
97 : DestTy->getArrayElementType();
98 Value *Element =
99 CreateAggregateCast(V: CreateExtractValue(Agg: V, Idxs: ArrayRef(I)), DestTy: ElementTy);
100
101 Result = CreateInsertValue(Agg: Result, Val: Element, Idxs: ArrayRef(I));
102 }
103 return Result;
104 }
105
106 return CreateBitOrPointerCast(V, DestTy);
107}
108
109Value *IRBuilderBase::CreateBitPreservingCastChain(const DataLayout &DL,
110 Value *V, Type *NewTy) {
111 Type *OldTy = V->getType();
112
113 if (OldTy == NewTy)
114 return V;
115
116 assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
117 "Integer types must be the exact same to convert.");
118
119 // A variant of bitcast that supports a mixture of fixed and scalable types
120 // that are know to have the same size.
121 auto CreateBitCastLike = [this](Value *In, Type *Ty) -> Value * {
122 Type *InTy = In->getType();
123 if (InTy == Ty)
124 return In;
125
126 if (isa<FixedVectorType>(Val: InTy) && isa<ScalableVectorType>(Val: Ty)) {
127 // For vscale_range(2) expand <4 x i32> to <vscale x 4 x i16> -->
128 // <4 x i32> to <vscale x 2 x i32> to <vscale x 4 x i16>
129 auto *VTy = VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: Ty), EltTy: InTy);
130 return CreateBitCast(
131 V: CreateInsertVector(DstType: VTy, SrcVec: PoisonValue::get(T: VTy), SubVec: In, Idx: getInt64(C: 0)), DestTy: Ty);
132 }
133
134 if (isa<ScalableVectorType>(Val: InTy) && isa<FixedVectorType>(Val: Ty)) {
135 // For vscale_range(2) expand <vscale x 4 x i16> to <4 x i32> -->
136 // <vscale x 4 x i16> to <vscale x 2 x i32> to <4 x i32>
137 auto *VTy = VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: InTy), EltTy: Ty);
138 return CreateExtractVector(DstType: Ty, SrcVec: CreateBitCast(V: In, DestTy: VTy), Idx: getInt64(C: 0));
139 }
140
141 return CreateBitCast(V: In, DestTy: Ty);
142 };
143
144 // See if we need inttoptr for this type pair. May require additional bitcast.
145 bool OldIsIntOrFP = OldTy->isIntOrIntVectorTy() || OldTy->isFPOrFPVectorTy();
146 if (OldIsIntOrFP && NewTy->isPtrOrPtrVectorTy()) {
147 // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
148 // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
149 // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*>
150 // Expand <2 x float> to i8* --> <2 x float> to i64 to i8*
151 // Directly handle i64 to i8*
152 return CreateIntToPtr(V: CreateBitCastLike(V, DL.getIntPtrType(NewTy)), DestTy: NewTy);
153 }
154
155 // See if we need ptrtoint for this type pair. May require additional bitcast.
156 bool NewIsIntOrFP = NewTy->isIntOrIntVectorTy() || NewTy->isFPOrFPVectorTy();
157 if (OldTy->isPtrOrPtrVectorTy() && NewIsIntOrFP) {
158 // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128
159 // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
160 // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32>
161 // Expand i8* to <2 x float> --> i8* to i64 to <2 x float>
162 // Expand i8* to i64 --> i8* to i64 to i64
163 return CreateBitCastLike(CreatePtrToInt(V, DestTy: DL.getIntPtrType(OldTy)), NewTy);
164 }
165
166 if (OldTy->isPtrOrPtrVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
167 unsigned OldAS = OldTy->getPointerAddressSpace();
168 unsigned NewAS = NewTy->getPointerAddressSpace();
169 // To convert pointers with different address spaces (they are already
170 // checked convertible, i.e. they have the same pointer size), so far we
171 // cannot use `bitcast` (which has restrict on the same address space) or
172 // `addrspacecast` (which is not always no-op casting). Instead, use a pair
173 // of no-op `ptrtoint`/`inttoptr` casts through an integer with the same bit
174 // size.
175 if (OldAS != NewAS) {
176 return CreateIntToPtr(
177 V: CreateBitCastLike(CreatePtrToInt(V, DestTy: DL.getIntPtrType(OldTy)),
178 DL.getIntPtrType(NewTy)),
179 DestTy: NewTy);
180 }
181 }
182
183 return CreateBitCastLike(V, NewTy);
184}
185
186CallInst *
187IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
188 const Twine &Name, FMFSource FMFSource,
189 ArrayRef<OperandBundleDef> OpBundles) {
190 CallInst *CI = CreateCall(Callee, Args: Ops, OpBundles, Name);
191 if (isa<FPMathOperator>(Val: CI))
192 CI->setFastMathFlags(FMFSource.get(Default: FMF));
193 return CI;
194}
195
196static Value *CreateVScaleMultiple(IRBuilderBase &B, Type *Ty, uint64_t Scale) {
197 Value *VScale = B.CreateVScale(Ty);
198 if (Scale == 1)
199 return VScale;
200
201 return B.CreateNUWMul(LHS: VScale, RHS: ConstantInt::get(Ty, V: Scale));
202}
203
204Value *IRBuilderBase::CreateElementCount(Type *Ty, ElementCount EC) {
205 if (EC.isFixed() || EC.isZero())
206 return ConstantInt::get(Ty, V: EC.getKnownMinValue());
207
208 return CreateVScaleMultiple(B&: *this, Ty, Scale: EC.getKnownMinValue());
209}
210
211Value *IRBuilderBase::CreateTypeSize(Type *Ty, TypeSize Size) {
212 if (Size.isFixed() || Size.isZero())
213 return ConstantInt::get(Ty, V: Size.getKnownMinValue());
214
215 return CreateVScaleMultiple(B&: *this, Ty, Scale: Size.getKnownMinValue());
216}
217
218Value *IRBuilderBase::CreateAllocationSize(Type *DestTy, AllocaInst *AI) {
219 const DataLayout &DL = getDataLayout();
220 TypeSize ElemSize = AI->getAllocationBaseSize(DL);
221 Value *Size = CreateTypeSize(Ty: DestTy, Size: ElemSize);
222 if (AI->isArrayAllocation())
223 Size = CreateMul(LHS: CreateZExtOrTrunc(V: AI->getArraySize(), DestTy), RHS: Size);
224 return Size;
225}
226
227Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
228 Type *STy = DstType->getScalarType();
229 if (isa<ScalableVectorType>(Val: DstType)) {
230 Type *StepVecType = DstType;
231 // TODO: We expect this special case (element type < 8 bits) to be
232 // temporary - once the intrinsic properly supports < 8 bits this code
233 // can be removed.
234 if (STy->getScalarSizeInBits() < 8)
235 StepVecType =
236 VectorType::get(ElementType: getInt8Ty(), Other: cast<ScalableVectorType>(Val: DstType));
237 Value *Res = CreateIntrinsic(ID: Intrinsic::stepvector, OverloadTypes: {StepVecType}, Args: {},
238 FMFSource: nullptr, Name);
239 if (StepVecType != DstType)
240 Res = CreateTrunc(V: Res, DestTy: DstType);
241 return Res;
242 }
243
244 unsigned NumEls = cast<FixedVectorType>(Val: DstType)->getNumElements();
245
246 // Create a vector of consecutive numbers from zero to VF.
247 // It's okay if the values wrap around.
248 SmallVector<Constant *, 8> Indices;
249 for (unsigned i = 0; i < NumEls; ++i)
250 Indices.push_back(
251 Elt: ConstantInt::get(Ty: STy, V: i, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
252
253 // Add the consecutive indices to the vector value.
254 return ConstantVector::get(V: Indices);
255}
256
257CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
258 MaybeAlign Align, bool isVolatile,
259 const AAMDNodes &AAInfo) {
260 Value *Ops[] = {Ptr, Val, Size, getInt1(V: isVolatile)};
261 Type *Tys[] = {Ptr->getType(), Size->getType()};
262
263 auto *CI = cast<MemSetInst>(
264 Val: CreateIntrinsicWithoutFolding(ID: Intrinsic::memset, OverloadTypes: Tys, Args: Ops));
265
266 if (Align)
267 CI->setDestAlignment(*Align);
268 CI->setAAMetadata(AAInfo);
269 return CI;
270}
271
272CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
273 Value *Val, Value *Size,
274 bool IsVolatile,
275 const AAMDNodes &AAInfo) {
276 Value *Ops[] = {Dst, Val, Size, getInt1(V: IsVolatile)};
277 Type *Tys[] = {Dst->getType(), Size->getType()};
278
279 auto *CI = cast<MemSetInst>(
280 Val: CreateIntrinsicWithoutFolding(ID: Intrinsic::memset_inline, OverloadTypes: Tys, Args: Ops));
281
282 if (DstAlign)
283 CI->setDestAlignment(*DstAlign);
284 CI->setAAMetadata(AAInfo);
285 return CI;
286}
287
288CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
289 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
290 const AAMDNodes &AAInfo) {
291
292 Value *Ops[] = {Ptr, Val, Size, getInt32(C: ElementSize)};
293 Type *Tys[] = {Ptr->getType(), Size->getType()};
294
295 auto *CI = cast<AnyMemSetInst>(Val: CreateIntrinsicWithoutFolding(
296 ID: Intrinsic::memset_element_unordered_atomic, OverloadTypes: Tys, Args: Ops));
297 CI->setDestAlignment(Alignment);
298 CI->setAAMetadata(AAInfo);
299 return CI;
300}
301
302CallInst *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
303 MaybeAlign DstAlign, Value *Src,
304 MaybeAlign SrcAlign, Value *Size,
305 bool isVolatile,
306 const AAMDNodes &AAInfo) {
307 assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||
308 IntrID == Intrinsic::memmove) &&
309 "Unexpected intrinsic ID");
310 Value *Ops[] = {Dst, Src, Size, getInt1(V: isVolatile)};
311 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
312
313 auto *MCI =
314 cast<MemTransferInst>(Val: CreateIntrinsicWithoutFolding(ID: IntrID, OverloadTypes: Tys, Args: Ops));
315
316 if (DstAlign)
317 MCI->setDestAlignment(*DstAlign);
318 if (SrcAlign)
319 MCI->setSourceAlignment(*SrcAlign);
320 MCI->setAAMetadata(AAInfo);
321 return MCI;
322}
323
324CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
325 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
326 uint32_t ElementSize, const AAMDNodes &AAInfo) {
327 assert(DstAlign >= ElementSize &&
328 "Pointer alignment must be at least element size");
329 assert(SrcAlign >= ElementSize &&
330 "Pointer alignment must be at least element size");
331 Value *Ops[] = {Dst, Src, Size, getInt32(C: ElementSize)};
332 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
333
334 auto *AMCI = cast<AnyMemCpyInst>(Val: CreateIntrinsicWithoutFolding(
335 ID: Intrinsic::memcpy_element_unordered_atomic, OverloadTypes: Tys, Args: Ops));
336
337 // Set the alignment of the pointer args.
338 AMCI->setDestAlignment(DstAlign);
339 AMCI->setSourceAlignment(SrcAlign);
340 AMCI->setAAMetadata(AAInfo);
341 return AMCI;
342}
343
344/// isConstantOne - Return true only if val is constant int 1
345static bool isConstantOne(const Value *Val) {
346 assert(Val && "isConstantOne does not work with nullptr Val");
347 const ConstantInt *CVal = dyn_cast<ConstantInt>(Val);
348 return CVal && CVal->isOne();
349}
350
351CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Value *AllocSize,
352 Value *ArraySize,
353 ArrayRef<OperandBundleDef> OpB,
354 Function *MallocF, const Twine &Name) {
355 // malloc(type) becomes:
356 // i8* malloc(typeSize)
357 // malloc(type, arraySize) becomes:
358 // i8* malloc(typeSize*arraySize)
359 if (!ArraySize)
360 ArraySize = ConstantInt::get(Ty: IntPtrTy, V: 1);
361 else if (ArraySize->getType() != IntPtrTy)
362 ArraySize = CreateIntCast(V: ArraySize, DestTy: IntPtrTy, isSigned: false);
363
364 if (!isConstantOne(Val: ArraySize)) {
365 if (isConstantOne(Val: AllocSize)) {
366 AllocSize = ArraySize; // Operand * 1 = Operand
367 } else {
368 // Multiply type size by the array size...
369 AllocSize = CreateMul(LHS: ArraySize, RHS: AllocSize, Name: "mallocsize");
370 }
371 }
372
373 assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");
374 // Create the call to Malloc.
375 Module *M = getModule();
376 Type *BPTy = PointerType::getUnqual(C&: Context);
377 FunctionCallee MallocFunc = MallocF;
378 if (!MallocFunc)
379 // prototype malloc as "void *malloc(size_t)"
380 MallocFunc = M->getOrInsertFunction(Name: "malloc", RetTy: BPTy, Args: IntPtrTy);
381 CallInst *MCall = CreateCall(Callee: MallocFunc, Args: AllocSize, OpBundles: OpB, Name);
382
383 MCall->setTailCall();
384 if (Function *F = dyn_cast<Function>(Val: MallocFunc.getCallee())) {
385 MCall->setCallingConv(F->getCallingConv());
386 F->setReturnDoesNotAlias();
387 }
388
389 assert(!MCall->getType()->isVoidTy() && "Malloc has void return type");
390
391 return MCall;
392}
393
394CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Value *AllocSize,
395 Value *ArraySize, Function *MallocF,
396 const Twine &Name) {
397
398 return CreateMalloc(IntPtrTy, AllocSize, ArraySize, OpB: {}, MallocF, Name);
399}
400
401/// CreateFree - Generate the IR for a call to the builtin free function.
402CallInst *IRBuilderBase::CreateFree(Value *Source,
403 ArrayRef<OperandBundleDef> Bundles) {
404 assert(Source->getType()->isPointerTy() &&
405 "Can not free something of nonpointer type!");
406
407 Module *M = getModule();
408
409 Type *VoidTy = Type::getVoidTy(C&: M->getContext());
410 Type *VoidPtrTy = PointerType::getUnqual(C&: M->getContext());
411 // prototype free as "void free(void*)"
412 FunctionCallee FreeFunc = M->getOrInsertFunction(Name: "free", RetTy: VoidTy, Args: VoidPtrTy);
413 CallInst *Result = CreateCall(Callee: FreeFunc, Args: Source, OpBundles: Bundles, Name: "");
414 Result->setTailCall();
415 if (Function *F = dyn_cast<Function>(Val: FreeFunc.getCallee()))
416 Result->setCallingConv(F->getCallingConv());
417
418 return Result;
419}
420
421CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
422 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
423 uint32_t ElementSize, const AAMDNodes &AAInfo) {
424 assert(DstAlign >= ElementSize &&
425 "Pointer alignment must be at least element size");
426 assert(SrcAlign >= ElementSize &&
427 "Pointer alignment must be at least element size");
428 Value *Ops[] = {Dst, Src, Size, getInt32(C: ElementSize)};
429 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
430
431 CallInst *CI = CreateIntrinsicWithoutFolding(
432 ID: Intrinsic::memmove_element_unordered_atomic, OverloadTypes: Tys, Args: Ops);
433
434 // Set the alignment of the pointer args.
435 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: DstAlign));
436 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: SrcAlign));
437 CI->setAAMetadata(AAInfo);
438 return CI;
439}
440
441Value *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
442 Value *Ops[] = {Src};
443 Type *Tys[] = { Src->getType() };
444 return CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops);
445}
446
447Value *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
448 Value *Ops[] = {Acc, Src};
449 return CreateIntrinsic(ID: Intrinsic::vector_reduce_fadd, OverloadTypes: {Src->getType()}, Args: Ops);
450}
451
452Value *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
453 Value *Ops[] = {Acc, Src};
454 return CreateIntrinsic(ID: Intrinsic::vector_reduce_fmul, OverloadTypes: {Src->getType()}, Args: Ops);
455}
456
457Value *IRBuilderBase::CreateAddReduce(Value *Src) {
458 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_add, Src);
459}
460
461Value *IRBuilderBase::CreateMulReduce(Value *Src) {
462 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_mul, Src);
463}
464
465Value *IRBuilderBase::CreateAndReduce(Value *Src) {
466 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_and, Src);
467}
468
469Value *IRBuilderBase::CreateOrReduce(Value *Src) {
470 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_or, Src);
471}
472
473Value *IRBuilderBase::CreateXorReduce(Value *Src) {
474 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_xor, Src);
475}
476
477Value *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
478 auto ID =
479 IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
480 return getReductionIntrinsic(ID, Src);
481}
482
483Value *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
484 auto ID =
485 IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
486 return getReductionIntrinsic(ID, Src);
487}
488
489Value *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
490 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmax, Src);
491}
492
493Value *IRBuilderBase::CreateFPMinReduce(Value *Src) {
494 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmin, Src);
495}
496
497Value *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
498 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmaximum, Src);
499}
500
501Value *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
502 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fminimum, Src);
503}
504
505Value *IRBuilderBase::CreateFPMaximumNumReduce(Value *Src) {
506 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fmaximumnum, Src);
507}
508
509Value *IRBuilderBase::CreateFPMinimumNumReduce(Value *Src) {
510 return getReductionIntrinsic(ID: Intrinsic::vector_reduce_fminimumnum, Src);
511}
512
513CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
514 assert(isa<PointerType>(Ptr->getType()) &&
515 "lifetime.start only applies to pointers.");
516 return CreateIntrinsicWithoutFolding(ID: Intrinsic::lifetime_start,
517 OverloadTypes: {Ptr->getType()}, Args: {Ptr});
518}
519
520CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
521 assert(isa<PointerType>(Ptr->getType()) &&
522 "lifetime.end only applies to pointers.");
523 return CreateIntrinsicWithoutFolding(ID: Intrinsic::lifetime_end,
524 OverloadTypes: {Ptr->getType()}, Args: {Ptr});
525}
526
527CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
528
529 assert(isa<PointerType>(Ptr->getType()) &&
530 "invariant.start only applies to pointers.");
531 if (!Size)
532 Size = getInt64(C: -1);
533 else
534 assert(Size->getType() == getInt64Ty() &&
535 "invariant.start requires the size to be an i64");
536
537 Value *Ops[] = {Size, Ptr};
538 // Fill in the single overloaded type: memory object type.
539 Type *ObjectPtr[1] = {Ptr->getType()};
540 return CreateIntrinsicWithoutFolding(ID: Intrinsic::invariant_start, OverloadTypes: ObjectPtr,
541 Args: Ops);
542}
543
544static MaybeAlign getAlign(Value *Ptr) {
545 if (auto *V = dyn_cast<GlobalVariable>(Val: Ptr))
546 return V->getAlign();
547 if (auto *A = dyn_cast<GlobalAlias>(Val: Ptr))
548 return getAlign(Ptr: A->getAliaseeObject());
549 return {};
550}
551
552CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
553 assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&
554 "threadlocal_address only applies to thread local variables.");
555 CallInst *CI = CreateIntrinsicWithoutFolding(
556 ID: llvm::Intrinsic::threadlocal_address, OverloadTypes: {Ptr->getType()}, Args: {Ptr});
557 if (MaybeAlign A = getAlign(Ptr)) {
558 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *A));
559 CI->addRetAttr(Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *A));
560 }
561 return CI;
562}
563
564CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
565 assert(Cond->getType() == getInt1Ty() &&
566 "an assumption condition must be of type i1");
567 return CreateIntrinsicWithoutFolding(ID: Intrinsic::assume, /*OverloadTypes=*/{},
568 Args: {Cond});
569}
570
571CallInst *
572IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
573 Value *Args[] = {ConstantInt::getTrue(Context&: getContext())};
574 return CreateIntrinsicWithoutFolding(
575 ID: Intrinsic::assume, /*OverloadTypes=*/{}, Args,
576 /*FMFSource=*/nullptr, /*Name=*/"", OpBundles);
577}
578
579Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
580 return CreateIntrinsicWithoutFolding(
581 ID: Intrinsic::experimental_noalias_scope_decl, OverloadTypes: {}, Args: {Scope});
582}
583
584/// Create a call to a Masked Load intrinsic.
585/// \p Ty - vector type to load
586/// \p Ptr - base pointer for the load
587/// \p Alignment - alignment of the source location
588/// \p Mask - vector of booleans which indicates what vector lanes should
589/// be accessed in memory
590/// \p PassThru - pass-through value that is used to fill the masked-off lanes
591/// of the result
592/// \p Name - name of the result variable
593CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
594 Value *Mask, Value *PassThru,
595 const Twine &Name) {
596 auto *PtrTy = cast<PointerType>(Val: Ptr->getType());
597 assert(Ty->isVectorTy() && "Type should be vector");
598 assert(Mask && "Mask should not be all-ones (null)");
599 if (!PassThru)
600 PassThru = PoisonValue::get(T: Ty);
601 Type *OverloadedTypes[] = { Ty, PtrTy };
602 Value *Ops[] = {Ptr, Mask, PassThru};
603 CallInst *CI =
604 CreateMaskedIntrinsic(Id: Intrinsic::masked_load, Ops, OverloadedTypes, Name);
605 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
606 return CI;
607}
608
609/// Create a call to a Masked Store intrinsic.
610/// \p Val - data to be stored,
611/// \p Ptr - base pointer for the store
612/// \p Alignment - alignment of the destination location
613/// \p Mask - vector of booleans which indicates what vector lanes should
614/// be accessed in memory
615CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
616 Align Alignment, Value *Mask) {
617 auto *PtrTy = cast<PointerType>(Val: Ptr->getType());
618 Type *DataTy = Val->getType();
619 assert(DataTy->isVectorTy() && "Val should be a vector");
620 assert(Mask && "Mask should not be all-ones (null)");
621 Type *OverloadedTypes[] = { DataTy, PtrTy };
622 Value *Ops[] = {Val, Ptr, Mask};
623 CallInst *CI =
624 CreateMaskedIntrinsic(Id: Intrinsic::masked_store, Ops, OverloadedTypes);
625 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
626 return CI;
627}
628
629/// Create a call to a Masked intrinsic, with given intrinsic Id,
630/// an array of operands - Ops, and an array of overloaded types -
631/// OverloadedTypes.
632CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
633 ArrayRef<Value *> Ops,
634 ArrayRef<Type *> OverloadedTypes,
635 const Twine &Name) {
636 return CreateIntrinsicWithoutFolding(ID: Id, OverloadTypes: OverloadedTypes, Args: Ops, FMFSource: {}, Name);
637}
638
639/// Create a call to a Masked Gather intrinsic.
640/// \p Ty - vector type to gather
641/// \p Ptrs - vector of pointers for loading
642/// \p Align - alignment for one element
643/// \p Mask - vector of booleans which indicates what vector lanes should
644/// be accessed in memory
645/// \p PassThru - pass-through value that is used to fill the masked-off lanes
646/// of the result
647/// \p Name - name of the result variable
648CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
649 Align Alignment, Value *Mask,
650 Value *PassThru,
651 const Twine &Name) {
652 auto *VecTy = cast<VectorType>(Val: Ty);
653 ElementCount NumElts = VecTy->getElementCount();
654 auto *PtrsTy = cast<VectorType>(Val: Ptrs->getType());
655 assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
656
657 if (!Mask)
658 Mask = getAllOnesMask(NumElts);
659
660 if (!PassThru)
661 PassThru = PoisonValue::get(T: Ty);
662
663 Type *OverloadedTypes[] = {Ty, PtrsTy};
664 Value *Ops[] = {Ptrs, Mask, PassThru};
665
666 // We specify only one type when we create this intrinsic. Types of other
667 // arguments are derived from this type.
668 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_gather, Ops,
669 OverloadedTypes, Name);
670 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
671 return CI;
672}
673
674/// Create a call to a Masked Scatter intrinsic.
675/// \p Data - data to be stored,
676/// \p Ptrs - the vector of pointers, where the \p Data elements should be
677/// stored
678/// \p Align - alignment for one element
679/// \p Mask - vector of booleans which indicates what vector lanes should
680/// be accessed in memory
681CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
682 Align Alignment, Value *Mask) {
683 auto *PtrsTy = cast<VectorType>(Val: Ptrs->getType());
684 auto *DataTy = cast<VectorType>(Val: Data->getType());
685 ElementCount NumElts = PtrsTy->getElementCount();
686
687 if (!Mask)
688 Mask = getAllOnesMask(NumElts);
689
690 Type *OverloadedTypes[] = {DataTy, PtrsTy};
691 Value *Ops[] = {Data, Ptrs, Mask};
692
693 // We specify only one type when we create this intrinsic. Types of other
694 // arguments are derived from this type.
695 CallInst *CI =
696 CreateMaskedIntrinsic(Id: Intrinsic::masked_scatter, Ops, OverloadedTypes);
697 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment));
698 return CI;
699}
700
701/// Create a call to Masked Expand Load intrinsic
702/// \p Ty - vector type to load
703/// \p Ptr - base pointer for the load
704/// \p Align - alignment of \p Ptr
705/// \p Mask - vector of booleans which indicates what vector lanes should
706/// be accessed in memory
707/// \p PassThru - pass-through value that is used to fill the masked-off lanes
708/// of the result
709/// \p Name - name of the result variable
710CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
711 MaybeAlign Align, Value *Mask,
712 Value *PassThru,
713 const Twine &Name) {
714 assert(Ty->isVectorTy() && "Type should be vector");
715 assert(Mask && "Mask should not be all-ones (null)");
716 if (!PassThru)
717 PassThru = PoisonValue::get(T: Ty);
718 Type *PtrTy = Ptr->getType();
719 Type *OverloadedTypes[] = {Ty, PtrTy};
720 Value *Ops[] = {Ptr, Mask, PassThru};
721 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_expandload, Ops,
722 OverloadedTypes, Name);
723 if (Align)
724 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *Align));
725 return CI;
726}
727
728/// Create a call to Masked Compress Store intrinsic
729/// \p Val - data to be stored,
730/// \p Ptr - base pointer for the store
731/// \p Align - alignment of \p Ptr
732/// \p Mask - vector of booleans which indicates what vector lanes should
733/// be accessed in memory
734CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
735 MaybeAlign Align,
736 Value *Mask) {
737 Type *DataTy = Val->getType();
738 assert(DataTy->isVectorTy() && "Val should be a vector");
739 assert(Mask && "Mask should not be all-ones (null)");
740 Type *PtrTy = Ptr->getType();
741 Type *OverloadedTypes[] = {DataTy, PtrTy};
742 Value *Ops[] = {Val, Ptr, Mask};
743 CallInst *CI = CreateMaskedIntrinsic(Id: Intrinsic::masked_compressstore, Ops,
744 OverloadedTypes);
745 if (Align)
746 CI->addParamAttr(ArgNo: 1, Attr: Attribute::getWithAlignment(Context&: CI->getContext(), Alignment: *Align));
747 return CI;
748}
749
750template <typename T0>
751static std::vector<Value *>
752getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
753 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
754 std::vector<Value *> Args;
755 Args.push_back(x: B.getInt64(C: ID));
756 Args.push_back(x: B.getInt32(C: NumPatchBytes));
757 Args.push_back(x: ActualCallee);
758 Args.push_back(B.getInt32(C: CallArgs.size()));
759 Args.push_back(x: B.getInt32(C: Flags));
760 llvm::append_range(Args, CallArgs);
761 // GC Transition and Deopt args are now always handled via operand bundle.
762 // They will be removed from the signature of gc.statepoint shortly.
763 Args.push_back(x: B.getInt32(C: 0));
764 Args.push_back(x: B.getInt32(C: 0));
765 // GC args are now encoded in the gc-live operand bundle
766 return Args;
767}
768
769template<typename T1, typename T2, typename T3>
770static std::vector<OperandBundleDef>
771getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
772 std::optional<ArrayRef<T2>> DeoptArgs,
773 ArrayRef<T3> GCArgs) {
774 std::vector<OperandBundleDef> Rval;
775 if (DeoptArgs)
776 Rval.emplace_back(args: "deopt", args: SmallVector<Value *, 16>(*DeoptArgs));
777 if (TransitionArgs)
778 Rval.emplace_back(args: "gc-transition",
779 args: SmallVector<Value *, 16>(*TransitionArgs));
780 if (GCArgs.size())
781 Rval.emplace_back(args: "gc-live", args: SmallVector<Value *, 16>(GCArgs));
782 return Rval;
783}
784
785template <typename T0, typename T1, typename T2, typename T3>
786static CallInst *CreateGCStatepointCallCommon(
787 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
788 FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
789 std::optional<ArrayRef<T1>> TransitionArgs,
790 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
791 const Twine &Name) {
792 Module *M = Builder->getModule();
793 // Fill in the one generic type'd argument (the function is also vararg)
794 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
795 M, id: Intrinsic::experimental_gc_statepoint,
796 OverloadTys: {ActualCallee.getCallee()->getType()});
797
798 std::vector<Value *> Args = getStatepointArgs(
799 *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
800
801 CallInst *CI = Builder->CreateCall(
802 FnStatepoint, Args,
803 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
804 CI->addParamAttr(ArgNo: 2,
805 Attr: Attribute::get(Context&: Builder->getContext(), Kind: Attribute::ElementType,
806 Ty: ActualCallee.getFunctionType()));
807 return CI;
808}
809
810CallInst *IRBuilderBase::CreateGCStatepointCall(
811 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
812 ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
813 ArrayRef<Value *> GCArgs, const Twine &Name) {
814 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
815 Builder: this, ID, NumPatchBytes, ActualCallee, Flags: uint32_t(StatepointFlags::None),
816 CallArgs, TransitionArgs: std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
817}
818
819CallInst *IRBuilderBase::CreateGCStatepointCall(
820 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
821 uint32_t Flags, ArrayRef<Value *> CallArgs,
822 std::optional<ArrayRef<Use>> TransitionArgs,
823 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
824 const Twine &Name) {
825 return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
826 Builder: this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
827 DeoptArgs, GCArgs, Name);
828}
829
830CallInst *IRBuilderBase::CreateGCStatepointCall(
831 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
832 ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
833 ArrayRef<Value *> GCArgs, const Twine &Name) {
834 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
835 Builder: this, ID, NumPatchBytes, ActualCallee, Flags: uint32_t(StatepointFlags::None),
836 CallArgs, TransitionArgs: std::nullopt, DeoptArgs, GCArgs, Name);
837}
838
839template <typename T0, typename T1, typename T2, typename T3>
840static InvokeInst *CreateGCStatepointInvokeCommon(
841 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
842 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
843 BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
844 std::optional<ArrayRef<T1>> TransitionArgs,
845 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
846 const Twine &Name) {
847 Module *M = Builder->getModule();
848 // Fill in the one generic type'd argument (the function is also vararg)
849 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
850 M, id: Intrinsic::experimental_gc_statepoint,
851 OverloadTys: {ActualInvokee.getCallee()->getType()});
852
853 std::vector<Value *> Args =
854 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
855 Flags, InvokeArgs);
856
857 InvokeInst *II = Builder->CreateInvoke(
858 FnStatepoint, NormalDest, UnwindDest, Args,
859 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
860 II->addParamAttr(ArgNo: 2,
861 Attr: Attribute::get(Context&: Builder->getContext(), Kind: Attribute::ElementType,
862 Ty: ActualInvokee.getFunctionType()));
863 return II;
864}
865
866InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
867 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
868 BasicBlock *NormalDest, BasicBlock *UnwindDest,
869 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
870 ArrayRef<Value *> GCArgs, const Twine &Name) {
871 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
872 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
873 Flags: uint32_t(StatepointFlags::None), InvokeArgs,
874 TransitionArgs: std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
875}
876
877InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
878 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
879 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
880 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
881 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
882 const Twine &Name) {
883 return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
884 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
885 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
886}
887
888InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
889 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
890 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
891 std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
892 const Twine &Name) {
893 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
894 Builder: this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
895 Flags: uint32_t(StatepointFlags::None), InvokeArgs, TransitionArgs: std::nullopt, DeoptArgs,
896 GCArgs, Name);
897}
898
899CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
900 Type *ResultType, const Twine &Name) {
901 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
902 Type *Types[] = {ResultType};
903
904 Value *Args[] = {Statepoint};
905 return CreateIntrinsicWithoutFolding(ID, OverloadTypes: Types, Args, FMFSource: {}, Name);
906}
907
908CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
909 int BaseOffset, int DerivedOffset,
910 Type *ResultType, const Twine &Name) {
911 Type *Types[] = {ResultType};
912
913 Value *Args[] = {Statepoint, getInt32(C: BaseOffset), getInt32(C: DerivedOffset)};
914 return CreateIntrinsicWithoutFolding(ID: Intrinsic::experimental_gc_relocate,
915 OverloadTypes: Types, Args, FMFSource: {}, Name);
916}
917
918CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
919 const Twine &Name) {
920 Type *PtrTy = DerivedPtr->getType();
921 return CreateIntrinsicWithoutFolding(
922 ID: Intrinsic::experimental_gc_get_pointer_base, OverloadTypes: PtrTy, Args: DerivedPtr, FMFSource: {}, Name);
923}
924
925CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
926 const Twine &Name) {
927 Type *PtrTy = DerivedPtr->getType();
928 return CreateIntrinsicWithoutFolding(
929 ID: Intrinsic::experimental_gc_get_pointer_offset, OverloadTypes: {PtrTy}, Args: {DerivedPtr}, FMFSource: {},
930 Name);
931}
932
933Value *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op,
934 FMFSource FMFSource,
935 const Twine &Name) {
936 Module *M = getModule();
937 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: Op->getType());
938 if (Value *V =
939 Folder.FoldIntrinsic(ID, Ops: Op, Ty: Fn->getReturnType(), FMF: FMFSource.get(Default: FMF),
940 CtxF: GetInsertBlock()->getParent()))
941 return V;
942 return createCallHelper(Callee: Fn, Ops: Op, Name, FMFSource);
943}
944
945Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
946 Value *RHS, FMFSource FMFSource,
947 const Twine &Name) {
948 Module *M = getModule();
949 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: {LHS->getType()});
950 if (Value *V = Folder.FoldIntrinsic(ID, Ops: {LHS, RHS}, Ty: Fn->getReturnType(),
951 FMF: FMFSource.get(Default: FMF),
952 CtxF: GetInsertBlock()->getParent()))
953 return V;
954 return createCallHelper(Callee: Fn, Ops: {LHS, RHS}, Name, FMFSource);
955}
956
957CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(
958 Intrinsic::ID ID, ArrayRef<Type *> OverloadTypes, ArrayRef<Value *> Args,
959 FMFSource FMFSource, const Twine &Name,
960 ArrayRef<OperandBundleDef> OpBundles) {
961 Module *M = getModule();
962 Function *Fn = Intrinsic::getOrInsertDeclaration(M, id: ID, OverloadTys: OverloadTypes);
963 return createCallHelper(Callee: Fn, Ops: Args, Name, FMFSource, OpBundles);
964}
965
966CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(Type *RetTy,
967 Intrinsic::ID ID,
968 ArrayRef<Value *> Args,
969 FMFSource FMFSource,
970 const Twine &Name) {
971 Module *M = getModule();
972 SmallVector<Type *> ArgTys = llvm::map_to_vector(C&: Args, F: &Value::getType);
973 Function *Fn = Intrinsic::getOrInsertDeclaration(M, IID: ID, RetTy, ArgTys);
974 return createCallHelper(Callee: Fn, Ops: Args, Name, FMFSource);
975}
976
977Value *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
978 ArrayRef<Type *> OverloadTypes,
979 ArrayRef<Value *> Args,
980 FMFSource FMFSource, const Twine &Name,
981 ArrayRef<OperandBundleDef> OpBundles,
982 function_ref<void(CallInst *)> SetFn) {
983 Type *RetTy = Intrinsic::getType(Context, id: ID, OverloadTys: OverloadTypes)->getReturnType();
984 if (Value *V = Folder.FoldIntrinsic(ID, Ops: Args, Ty: RetTy, FMF: FMFSource.get(Default: FMF),
985 CtxF: GetInsertBlock()->getParent()))
986 return V;
987 CallInst *CI = CreateIntrinsicWithoutFolding(ID, OverloadTypes, Args,
988 FMFSource, Name, OpBundles);
989 SetFn(CI);
990 return CI;
991}
992
993Value *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
994 ArrayRef<Value *> Args,
995 FMFSource FMFSource, const Twine &Name,
996 function_ref<void(CallInst *)> SetFn) {
997 if (Value *V = Folder.FoldIntrinsic(ID, Ops: Args, Ty: RetTy, FMF: FMFSource.get(Default: FMF),
998 CtxF: GetInsertBlock()->getParent()))
999 return V;
1000 CallInst *CI =
1001 CreateIntrinsicWithoutFolding(RetTy, ID, Args, FMFSource, Name);
1002 SetFn(CI);
1003 return CI;
1004}
1005
1006CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
1007 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
1008 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
1009 std::optional<fp::ExceptionBehavior> Except) {
1010 Value *RoundingV = getConstrainedFPRounding(Rounding);
1011 Value *ExceptV = getConstrainedFPExcept(Except);
1012
1013 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1014 CallInst *C = CreateIntrinsicWithoutFolding(
1015 ID, OverloadTypes: {L->getType()}, Args: {L, R, RoundingV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1016 setConstrainedFPCallAttr(C);
1017 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1018 return C;
1019}
1020
1021CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
1022 Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
1023 FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag,
1024 std::optional<RoundingMode> Rounding,
1025 std::optional<fp::ExceptionBehavior> Except) {
1026 Value *RoundingV = getConstrainedFPRounding(Rounding);
1027 Value *ExceptV = getConstrainedFPExcept(Except);
1028
1029 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1030
1031 llvm::SmallVector<Value *, 5> ExtArgs(Args);
1032 ExtArgs.push_back(Elt: RoundingV);
1033 ExtArgs.push_back(Elt: ExceptV);
1034 CallInst *C =
1035 CreateIntrinsicWithoutFolding(ID, OverloadTypes: Types, Args: ExtArgs, FMFSource: nullptr, Name, OpBundles: {});
1036 setConstrainedFPCallAttr(C);
1037 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1038 return C;
1039}
1040
1041CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
1042 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
1043 const Twine &Name, MDNode *FPMathTag,
1044 std::optional<fp::ExceptionBehavior> Except) {
1045 Value *ExceptV = getConstrainedFPExcept(Except);
1046
1047 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1048 CallInst *C = CreateIntrinsicWithoutFolding(
1049 ID, OverloadTypes: {L->getType()}, Args: {L, R, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1050 setConstrainedFPCallAttr(C);
1051 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1052 return C;
1053}
1054
1055Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1056 const Twine &Name, MDNode *FPMathTag) {
1057 if (Instruction::isBinaryOp(Opcode: Opc)) {
1058 assert(Ops.size() == 2 && "Invalid number of operands!");
1059 return CreateBinOp(Opc: static_cast<Instruction::BinaryOps>(Opc),
1060 LHS: Ops[0], RHS: Ops[1], Name, FPMathTag);
1061 }
1062 if (Instruction::isUnaryOp(Opcode: Opc)) {
1063 assert(Ops.size() == 1 && "Invalid number of operands!");
1064 return CreateUnOp(Opc: static_cast<Instruction::UnaryOps>(Opc),
1065 V: Ops[0], Name, FPMathTag);
1066 }
1067 llvm_unreachable("Unexpected opcode!");
1068}
1069
1070CallInst *IRBuilderBase::CreateConstrainedFPCast(
1071 Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource,
1072 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
1073 std::optional<fp::ExceptionBehavior> Except) {
1074 Value *ExceptV = getConstrainedFPExcept(Except);
1075
1076 FastMathFlags UseFMF = FMFSource.get(Default: FMF);
1077
1078 CallInst *C;
1079 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: ID)) {
1080 Value *RoundingV = getConstrainedFPRounding(Rounding);
1081 C = CreateIntrinsicWithoutFolding(
1082 ID, OverloadTypes: {DestTy, V->getType()}, Args: {V, RoundingV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1083 } else
1084 C = CreateIntrinsicWithoutFolding(ID, OverloadTypes: {DestTy, V->getType()}, Args: {V, ExceptV},
1085 FMFSource: nullptr, Name, OpBundles: {});
1086 setConstrainedFPCallAttr(C);
1087
1088 if (isa<FPMathOperator>(Val: C))
1089 setFPAttrs(I: C, FPMD: FPMathTag, FMF: UseFMF);
1090 return C;
1091}
1092
1093Value *IRBuilderBase::CreateFCmpHelper(CmpInst::Predicate P, Value *LHS,
1094 Value *RHS, const Twine &Name,
1095 MDNode *FPMathTag, FMFSource FMFSource,
1096 bool IsSignaling) {
1097 if (IsFPConstrained) {
1098 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
1099 : Intrinsic::experimental_constrained_fcmp;
1100 return CreateConstrainedFPCmp(ID, P, L: LHS, R: RHS, Name);
1101 }
1102
1103 if (auto *V = Folder.FoldCmp(P, LHS, RHS))
1104 return V;
1105 return Insert(
1106 I: setFPAttrs(I: new FCmpInst(P, LHS, RHS), FPMD: FPMathTag, FMF: FMFSource.get(Default: FMF)),
1107 Name);
1108}
1109
1110CallInst *IRBuilderBase::CreateConstrainedFPCmp(
1111 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
1112 const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
1113 Value *PredicateV = getConstrainedFPPredicate(Predicate: P);
1114 Value *ExceptV = getConstrainedFPExcept(Except);
1115
1116 CallInst *C = CreateIntrinsicWithoutFolding(
1117 ID, OverloadTypes: {L->getType()}, Args: {L, R, PredicateV, ExceptV}, FMFSource: nullptr, Name, OpBundles: {});
1118 setConstrainedFPCallAttr(C);
1119 return C;
1120}
1121
1122CallInst *IRBuilderBase::CreateConstrainedFPCall(
1123 Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
1124 std::optional<RoundingMode> Rounding,
1125 std::optional<fp::ExceptionBehavior> Except) {
1126 llvm::SmallVector<Value *, 6> UseArgs(Args);
1127
1128 if (Intrinsic::hasConstrainedFPRoundingModeOperand(QID: Callee->getIntrinsicID()))
1129 UseArgs.push_back(Elt: getConstrainedFPRounding(Rounding));
1130 UseArgs.push_back(Elt: getConstrainedFPExcept(Except));
1131
1132 CallInst *C = CreateCall(Callee, Args: UseArgs, Name);
1133 setConstrainedFPCallAttr(C);
1134 return C;
1135}
1136
1137Value *IRBuilderBase::CreateSelectWithUnknownProfile(Value *C, Value *True,
1138 Value *False,
1139 StringRef PassName,
1140 const Twine &Name) {
1141 Value *Ret = CreateSelectFMF(C, True, False, FMFSource: {}, Name);
1142 if (auto *SI = dyn_cast<SelectInst>(Val: Ret)) {
1143 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, PassName);
1144 }
1145 return Ret;
1146}
1147
1148Value *IRBuilderBase::CreateSelectFMFWithUnknownProfile(Value *C, Value *True,
1149 Value *False,
1150 FMFSource FMFSource,
1151 StringRef PassName,
1152 const Twine &Name) {
1153 Value *Ret = CreateSelectFMF(C, True, False, FMFSource, Name);
1154 if (auto *SI = dyn_cast<SelectInst>(Val: Ret))
1155 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, PassName);
1156 return Ret;
1157}
1158
1159Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
1160 const Twine &Name, Instruction *MDFrom) {
1161 return CreateSelectFMF(C, True, False, FMFSource: {}, Name, MDFrom);
1162}
1163
1164Value *IRBuilderBase::CreateSelectFMF(Value *C, Value *True, Value *False,
1165 FMFSource FMFSource, const Twine &Name,
1166 Instruction *MDFrom) {
1167 if (auto *V = Folder.FoldSelect(C, True, False, FMF: FMFSource.get(Default: FMF)))
1168 return V;
1169
1170 SelectInst *Sel = SelectInst::Create(C, S1: True, S2: False);
1171 if (MDFrom) {
1172 MDNode *Prof = MDFrom->getMetadata(KindID: LLVMContext::MD_prof);
1173 MDNode *Unpred = MDFrom->getMetadata(KindID: LLVMContext::MD_unpredictable);
1174 Sel = addBranchMetadata(I: Sel, Weights: Prof, Unpredictable: Unpred);
1175 }
1176 if (isa<FPMathOperator>(Val: Sel))
1177 setFPAttrs(I: Sel, /*MDNode=*/FPMD: nullptr, FMF: FMFSource.get(Default: FMF));
1178 return Insert(I: Sel, Name);
1179}
1180
1181Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name,
1182 bool IsNUW) {
1183 assert(LHS->getType() == RHS->getType() &&
1184 "Pointer subtraction operand types must match!");
1185 Value *LHSAddr = CreatePtrToAddr(V: LHS);
1186 Value *RHSAddr = CreatePtrToAddr(V: RHS);
1187 return CreateSub(LHS: LHSAddr, RHS: RHSAddr, Name, HasNUW: IsNUW);
1188}
1189Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
1190 const Twine &Name) {
1191 const DataLayout &DL = getDataLayout();
1192 TypeSize ElemSize = DL.getTypeAllocSize(Ty: ElemTy);
1193 if (ElemSize == TypeSize::getFixed(ExactSize: 1))
1194 return CreatePtrDiff(LHS, RHS, Name);
1195
1196 Value *Diff = CreatePtrDiff(LHS, RHS);
1197 return CreateExactSDiv(LHS: Diff, RHS: CreateTypeSize(Ty: Diff->getType(), Size: ElemSize), Name);
1198}
1199
1200Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
1201 assert(isa<PointerType>(Ptr->getType()) &&
1202 "launder.invariant.group only applies to pointers.");
1203 auto *PtrType = Ptr->getType();
1204 Module *M = getModule();
1205 Function *FnLaunderInvariantGroup = Intrinsic::getOrInsertDeclaration(
1206 M, id: Intrinsic::launder_invariant_group, OverloadTys: {PtrType});
1207
1208 assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&
1209 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1210 PtrType &&
1211 "LaunderInvariantGroup should take and return the same type");
1212
1213 return CreateCall(Callee: FnLaunderInvariantGroup, Args: {Ptr});
1214}
1215
1216Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1217 auto *Ty = cast<VectorType>(Val: V->getType());
1218 if (isa<ScalableVectorType>(Val: Ty)) {
1219 Module *M = getModule();
1220 Function *F =
1221 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::vector_reverse, OverloadTys: Ty);
1222 return Insert(I: CallInst::Create(Func: F, Args: V), Name);
1223 }
1224 // Keep the original behaviour for fixed vector
1225 SmallVector<int, 8> ShuffleMask;
1226 int NumElts = Ty->getElementCount().getKnownMinValue();
1227 for (int i = 0; i < NumElts; ++i)
1228 ShuffleMask.push_back(Elt: NumElts - i - 1);
1229 return CreateShuffleVector(V, Mask: ShuffleMask, Name);
1230}
1231
1232static SmallVector<int, 8> getSpliceMask(int64_t Imm, unsigned NumElts) {
1233 unsigned Idx = (NumElts + Imm) % NumElts;
1234 SmallVector<int, 8> Mask;
1235 for (unsigned I = 0; I < NumElts; ++I)
1236 Mask.push_back(Elt: Idx + I);
1237 return Mask;
1238}
1239
1240Value *IRBuilderBase::CreateVectorSpliceLeft(Value *V1, Value *V2,
1241 Value *Offset, const Twine &Name) {
1242 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1243 assert(V1->getType() == V2->getType() &&
1244 "Splice expects matching operand types!");
1245
1246 // Emit a shufflevector for fixed vectors with a constant offset
1247 if (auto *COffset = dyn_cast<ConstantInt>(Val: Offset))
1248 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType()))
1249 return CreateShuffleVector(
1250 V1, V2,
1251 Mask: getSpliceMask(Imm: COffset->getZExtValue(), NumElts: FVTy->getNumElements()));
1252
1253 return CreateIntrinsic(ID: Intrinsic::vector_splice_left, OverloadTypes: V1->getType(),
1254 Args: {V1, V2, Offset}, FMFSource: {}, Name);
1255}
1256
1257Value *IRBuilderBase::CreateVectorSpliceRight(Value *V1, Value *V2,
1258 Value *Offset,
1259 const Twine &Name) {
1260 assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1261 assert(V1->getType() == V2->getType() &&
1262 "Splice expects matching operand types!");
1263
1264 // Emit a shufflevector for fixed vectors with a constant offset
1265 if (auto *COffset = dyn_cast<ConstantInt>(Val: Offset))
1266 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType()))
1267 return CreateShuffleVector(
1268 V1, V2,
1269 Mask: getSpliceMask(Imm: -COffset->getZExtValue(), NumElts: FVTy->getNumElements()));
1270
1271 return CreateIntrinsic(ID: Intrinsic::vector_splice_right, OverloadTypes: V1->getType(),
1272 Args: {V1, V2, Offset}, FMFSource: {}, Name);
1273}
1274
1275Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1276 const Twine &Name) {
1277 auto EC = ElementCount::getFixed(MinVal: NumElts);
1278 return CreateVectorSplat(EC, V, Name);
1279}
1280
1281Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1282 const Twine &Name) {
1283 assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1284
1285 // First insert it into a poison vector so we can shuffle it.
1286 Value *Poison = PoisonValue::get(T: VectorType::get(ElementType: V->getType(), EC));
1287 V = CreateInsertElement(Vec: Poison, NewElt: V, Idx: getInt64(C: 0), Name: Name + ".splatinsert");
1288
1289 // Shuffle the value across the desired number of elements.
1290 SmallVector<int, 16> Zeros;
1291 Zeros.resize(N: EC.getKnownMinValue());
1292 return CreateShuffleVector(V, Mask: Zeros, Name: Name + ".splat");
1293}
1294
1295Value *IRBuilderBase::CreateVectorInterleave(ArrayRef<Value *> Ops,
1296 const Twine &Name) {
1297 assert(Ops.size() >= 2 && Ops.size() <= 8 &&
1298 "Unexpected number of operands to interleave");
1299
1300 // Make sure all operands are the same type.
1301 assert(isa<VectorType>(Ops[0]->getType()) && "Unexpected type");
1302
1303#ifndef NDEBUG
1304 for (unsigned I = 1; I < Ops.size(); I++) {
1305 assert(Ops[I]->getType() == Ops[0]->getType() &&
1306 "Vector interleave expects matching operand types!");
1307 }
1308#endif
1309
1310 unsigned IID = Intrinsic::getInterleaveIntrinsicID(Factor: Ops.size());
1311 auto *SubvecTy = cast<VectorType>(Val: Ops[0]->getType());
1312 Type *DestTy = VectorType::get(ElementType: SubvecTy->getElementType(),
1313 EC: SubvecTy->getElementCount() * Ops.size());
1314 return CreateIntrinsic(ID: IID, OverloadTypes: {DestTy}, Args: Ops, FMFSource: {}, Name);
1315}
1316
1317Value *IRBuilderBase::CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
1318 unsigned Dimension,
1319 unsigned LastIndex,
1320 MDNode *DbgInfo) {
1321 auto *BaseType = Base->getType();
1322 assert(isa<PointerType>(BaseType) &&
1323 "Invalid Base ptr type for preserve.array.access.index.");
1324
1325 Value *LastIndexV = getInt32(C: LastIndex);
1326 Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 0);
1327 SmallVector<Value *, 4> IdxList(Dimension, Zero);
1328 IdxList.push_back(Elt: LastIndexV);
1329
1330 Type *ResultType = GetElementPtrInst::getGEPReturnType(Ptr: Base, IdxList);
1331
1332 Value *DimV = getInt32(C: Dimension);
1333 CallInst *Fn = CreateIntrinsicWithoutFolding(
1334 ID: Intrinsic::preserve_array_access_index, OverloadTypes: {ResultType, BaseType},
1335 Args: {Base, DimV, LastIndexV});
1336 Fn->addParamAttr(
1337 ArgNo: 0, Attr: Attribute::get(Context&: Fn->getContext(), Kind: Attribute::ElementType, Ty: ElTy));
1338 if (DbgInfo)
1339 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1340
1341 return Fn;
1342}
1343
1344Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1345 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1346 assert(isa<PointerType>(Base->getType()) &&
1347 "Invalid Base ptr type for preserve.union.access.index.");
1348 auto *BaseType = Base->getType();
1349
1350 Value *DIIndex = getInt32(C: FieldIndex);
1351 CallInst *Fn =
1352 CreateIntrinsicWithoutFolding(ID: Intrinsic::preserve_union_access_index,
1353 OverloadTypes: {BaseType, BaseType}, Args: {Base, DIIndex});
1354 if (DbgInfo)
1355 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1356
1357 return Fn;
1358}
1359
1360Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1361 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1362 MDNode *DbgInfo) {
1363 auto *BaseType = Base->getType();
1364 assert(isa<PointerType>(BaseType) &&
1365 "Invalid Base ptr type for preserve.struct.access.index.");
1366
1367 Value *GEPIndex = getInt32(C: Index);
1368 Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 0);
1369 Type *ResultType =
1370 GetElementPtrInst::getGEPReturnType(Ptr: Base, IdxList: {Zero, GEPIndex});
1371
1372 Value *DIIndex = getInt32(C: FieldIndex);
1373 CallInst *Fn = CreateIntrinsicWithoutFolding(
1374 ID: Intrinsic::preserve_struct_access_index, OverloadTypes: {ResultType, BaseType},
1375 Args: {Base, GEPIndex, DIIndex});
1376 Fn->addParamAttr(
1377 ArgNo: 0, Attr: Attribute::get(Context&: Fn->getContext(), Kind: Attribute::ElementType, Ty: ElTy));
1378 if (DbgInfo)
1379 Fn->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: DbgInfo);
1380
1381 return Fn;
1382}
1383
1384Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
1385 ConstantInt *TestV = getInt32(C: Test);
1386 return CreateIntrinsic(ID: Intrinsic::is_fpclass, OverloadTypes: {FPNum->getType()},
1387 Args: {FPNum, TestV});
1388}
1389
1390CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1391 Value *PtrValue,
1392 Value *AlignValue,
1393 Value *OffsetValue) {
1394 SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1395 if (OffsetValue)
1396 Vals.push_back(Elt: OffsetValue);
1397 OperandBundleDefT<Value *> AlignOpB("align", Vals);
1398 return CreateAssumption(OpBundles: {AlignOpB});
1399}
1400
1401CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1402 Value *PtrValue,
1403 uint64_t Alignment,
1404 Value *OffsetValue) {
1405 assert(isa<PointerType>(PtrValue->getType()) &&
1406 "trying to create an alignment assumption on a non-pointer?");
1407 assert(Alignment != 0 && "Invalid Alignment");
1408 Value *AlignValue = ConstantInt::get(Ty: getInt64Ty(), V: Alignment);
1409 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1410}
1411
1412CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1413 Value *PtrValue,
1414 Value *Alignment,
1415 Value *OffsetValue) {
1416 assert(isa<PointerType>(PtrValue->getType()) &&
1417 "trying to create an alignment assumption on a non-pointer?");
1418 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue: Alignment, OffsetValue);
1419}
1420
1421CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
1422 Value *SizeValue) {
1423 assert(isa<PointerType>(PtrValue->getType()) &&
1424 "trying to create a deferenceable assumption on a non-pointer?");
1425 SmallVector<Value *, 4> Vals({PtrValue, SizeValue});
1426 OperandBundleDefT<Value *> DereferenceableOpB("dereferenceable", Vals);
1427 return CreateAssumption(OpBundles: {DereferenceableOpB});
1428}
1429
1430CallInst *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
1431 assert(isa<PointerType>(PtrValue->getType()) &&
1432 "trying to create a nonnull assumption on a non-pointer?");
1433 return CreateAssumption(OpBundles: OperandBundleDef("nonnull", PtrValue));
1434}
1435
1436CallInst *IRBuilderBase::CreateRangeAssumption(Value *Val,
1437 ConstantRange Range) {
1438 assert(isa<IntegerType>(Val->getType()) &&
1439 "trying to create a range assumption on a non-integer?");
1440 assert(!Range.isFullSet() &&
1441 "trying to create a range assumption which doesn't assume anything?");
1442 return CreateAssumption(
1443 OpBundles: OperandBundleDef("range", ArrayRef<Value *>{Val, getInt(AI: Range.getLower()),
1444 getInt(AI: Range.getUpper())}));
1445}
1446
1447Value *IRBuilderBase::CreateGEP(Type *Ty, Value *OrigPtr,
1448 ArrayRef<Value *> IdxList, const Twine &Name,
1449 GEPNoWrapFlags NW) {
1450 Value *Ptr = OrigPtr;
1451 const DataLayout &DL = getDataLayout();
1452 if (auto *V = Folder.FoldGEP(DL, Ty, Ptr, IdxList, NW))
1453 return V;
1454
1455 if (!CanonicalizeGEPs)
1456 return Insert(I: GetElementPtrInst::Create(PointeeType: Ty, Ptr, IdxList, NW), Name);
1457
1458 unsigned IdxSize = DL.getIndexTypeSizeInBits(Ty: Ptr->getType());
1459 if (Ty->isIntegerTy(BitWidth: 8) && IdxList[0]->getType()->isIntOrIntVectorTy(BitWidth: IdxSize))
1460 return CreatePtrAdd(Ptr, Offset: IdxList[0], Name, NW);
1461
1462 auto GTI = gep_type_begin(Op0: Ty, A: IdxList), GTE = gep_type_end(Ty, A: IdxList);
1463 for (; GTI != GTE; ++GTI) {
1464 Value *Op = GTI.getOperand();
1465 Type *IdxTy = Op->getType()->getWithNewBitWidth(NewBitWidth: IdxSize);
1466 if (auto *OpC = dyn_cast<Constant>(Val: Op)) {
1467 // Skip zero indices, except for vector splats.
1468 bool IsSplat = IdxTy->isVectorTy() && !Ptr->getType()->isVectorTy();
1469 if (OpC->isNullValue() && !IsSplat)
1470 continue;
1471
1472 if (StructType *STy = GTI.getStructTypeOrNull()) {
1473 uint64_t OpValue = OpC->getUniqueInteger().getZExtValue();
1474 uint64_t Offset = DL.getStructLayout(Ty: STy)->getElementOffset(Idx: OpValue);
1475 if (!Offset && !IsSplat)
1476 continue;
1477
1478 Constant *OffsetC = ConstantInt::get(Ty: IdxTy, V: Offset, /*IsSigned=*/false,
1479 /*ImplicitTrunc=*/true);
1480 Ptr = CreatePtrAdd(Ptr, Offset: OffsetC, Name, NW);
1481 continue;
1482 }
1483 }
1484
1485 if (Op->getType() != IdxTy)
1486 Op = CreateSExtOrTrunc(V: Op, DestTy: IdxTy);
1487
1488 TypeSize Stride = GTI.getSequentialElementStride(DL);
1489 auto *OpC = dyn_cast<ConstantInt>(Val: Op);
1490 if (OpC && !Stride.isScalable()) {
1491 APInt StrideInt(IdxSize, Stride.getFixedValue(), /*IsSigned=*/false,
1492 /*ImplicitTrunc=*/true);
1493 Ptr = CreatePtrAdd(
1494 Ptr, Offset: ConstantInt::get(Ty: IdxTy, V: OpC->getValue() * StrideInt), Name, NW);
1495 continue;
1496 }
1497
1498 Ptr = CreateScaledPtrAdd(Ptr, Scale: Stride, Offset: Op, Name, NW);
1499 }
1500
1501 // Make sure to always generate at least one GEP.
1502 if (Ptr == OrigPtr)
1503 return CreatePtrAddWithUnsignedOffset(Ptr, Offset: 0, Name, NW);
1504 return Ptr;
1505}
1506
1507void IRBuilderFolder::anchor() {}
1508void ConstantFolder::anchor() {}
1509void NoFolder::anchor() {}
1510