1//===-- AMDGPULowerBufferFatPointers.cpp ---------------------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers operations on buffer fat pointers (addrspace 7) to
10// operations on buffer resources (addrspace 8) and is needed for correct
11// codegen.
12//
13// # Background
14//
15// Address space 7 (the buffer fat pointer) is a 160-bit pointer that consists
16// of a 128-bit buffer descriptor and a 32-bit offset into that descriptor.
17// The buffer resource part needs to be it needs to be a "raw" buffer resource
18// (it must have a stride of 0 and bounds checks must be in raw buffer mode
19// or disabled).
20//
21// When these requirements are met, a buffer resource can be treated as a
22// typical (though quite wide) pointer that follows typical LLVM pointer
23// semantics. This allows the frontend to reason about such buffers (which are
24// often encountered in the context of SPIR-V kernels).
25//
26// However, because of their non-power-of-2 size, these fat pointers cannot be
27// present during translation to MIR (though this restriction may be lifted
28// during the transition to GlobalISel). Therefore, this pass is needed in order
29// to correctly implement these fat pointers.
30//
31// The resource intrinsics take the resource part (the address space 8 pointer)
32// and the offset part (the 32-bit integer) as separate arguments. In addition,
33// many users of these buffers manipulate the offset while leaving the resource
34// part alone. For these reasons, we want to typically separate the resource
35// and offset parts into separate variables, but combine them together when
36// encountering cases where this is required, such as by inserting these values
37// into aggretates or moving them to memory.
38//
39// Therefore, at a high level, `ptr addrspace(7) %x` becomes `ptr addrspace(8)
40// %x.rsrc` and `i32 %x.off`, which will be combined into `{ptr addrspace(8),
41// i32} %x = {%x.rsrc, %x.off}` if needed. Similarly, `vector<Nxp7>` becomes
42// `{vector<Nxp8>, vector<Nxi32 >}` and its component parts.
43//
44// # Implementation
45//
46// This pass proceeds in three main phases:
47//
48// ## Rewriting loads and stores of p7 and memcpy()-like handling
49//
50// The first phase is to rewrite away all loads and stors of `ptr addrspace(7)`,
51// including aggregates containing such pointers, to ones that use `i160`. This
52// is handled by `StoreFatPtrsAsIntsAndExpandMemcpyVisitor` , which visits
53// loads, stores, and allocas and, if the loaded or stored type contains `ptr
54// addrspace(7)`, rewrites it to use i160, `ptrtoint`ing before stores and
55// `inttoptr`ing after loads. Vectors of pointers work the same way. Since i160
56// and p7 differ in size and alignment, aggregates are split into one access per
57// leaf, and allocas and GEPs use byte offsets and sizes from the original type.
58//
59// Such a transformation allows the later phases of the pass to not need
60// to handle buffer fat pointers moving to and from memory, where we load
61// have to handle the incompatibility between a `{Nxp8, Nxi32}` representation
62// and `Nxi60` directly. Instead, that transposing action (where the vectors
63// of resources and vectors of offsets are concatentated before being stored to
64// memory) are handled through implementing `inttoptr` and `ptrtoint` only.
65//
66// Atomics operations on `ptr addrspace(7)` values are not supported, as the
67// hardware does not include a 160-bit atomic.
68//
69// In order to save on O(N) work and to ensure that the contents type
70// legalizer correctly splits up wide loads, also unconditionally lower
71// memcpy-like intrinsics into loops here.
72//
73// ## Buffer contents type legalization
74//
75// The underlying buffer intrinsics only support types up to 128 bits long,
76// and don't support complex types. If buffer operations were
77// standard pointer operations that could be represented as MIR-level loads,
78// this would be handled by the various legalization schemes in instruction
79// selection. However, because we have to do the conversion from `load` and
80// `store` to intrinsics at LLVM IR level, we must perform that legalization
81// ourselves.
82//
83// This involves a combination of
84// - Converting arrays to vectors where possible
85// - Otherwise, splitting loads and stores of aggregates into loads/stores of
86// each component.
87// - Zero-extending things to fill a whole number of bytes
88// - Casting values of types that don't neatly correspond to supported machine
89// value
90// (for example, an i96 or i256) into ones that would work (
91// like <3 x i32> and <8 x i32>, respectively)
92// - Splitting values that are too long (such as aforementioned <8 x i32>) into
93// multiple operations.
94//
95// ## Type remapping
96//
97// We use a `ValueMapper` to mangle uses of [vectors of] buffer fat pointers
98// to the corresponding struct type, which has a resource part and an offset
99// part.
100//
101// This uses a `BufferFatPtrToStructTypeMap` and a `FatPtrConstMaterializer`
102// to, usually by way of `setType`ing values. Constants are handled here
103// because there isn't a good way to fix them up later.
104//
105// This has the downside of leaving the IR in an invalid state (for example,
106// the instruction `getelementptr {ptr addrspace(8), i32} %p, ...` will exist),
107// but all such invalid states will be resolved by the third phase.
108//
109// Functions that don't take buffer fat pointers are modified in place. Those
110// that do take such pointers have their basic blocks moved to a new function
111// with arguments that are {ptr addrspace(8), i32} arguments and return values.
112// This phase also records intrinsics so that they can be remangled or deleted
113// later.
114//
115// ## Splitting pointer structs
116//
117// The meat of this pass consists of defining semantics for operations that
118// produce or consume [vectors of] buffer fat pointers in terms of their
119// resource and offset parts. This is accomplished throgh the `SplitPtrStructs`
120// visitor.
121//
122// In the first pass through each function that is being lowered, the splitter
123// inserts new instructions to implement the split-structures behavior, which is
124// needed for correctness and performance. It records a list of "split users",
125// instructions that are being replaced by operations on the resource and offset
126// parts.
127//
128// Split users do not necessarily need to produce parts themselves (
129// a `load float, ptr addrspace(7)` does not, for example), but, if they do not
130// generate fat buffer pointers, they must RAUW in their replacement
131// instructions during the initial visit.
132//
133// When these new instructions are created, they use the split parts recorded
134// for their initial arguments in order to generate their replacements, creating
135// a parallel set of instructions that does not refer to the original fat
136// pointer values but instead to their resource and offset components.
137//
138// Instructions, such as `extractvalue`, that produce buffer fat pointers from
139// sources that do not have split parts, have such parts generated using
140// `extractvalue`. This is also the initial handling of PHI nodes, which
141// are then cleaned up.
142//
143// ### Conditionals
144//
145// PHI nodes are initially given resource parts via `extractvalue`. However,
146// this is not an efficient rewrite of such nodes, as, in most cases, the
147// resource part in a conditional or loop remains constant throughout the loop
148// and only the offset varies. Failing to optimize away these constant resources
149// would cause additional registers to be sent around loops and might lead to
150// waterfall loops being generated for buffer operations due to the
151// "non-uniform" resource argument.
152//
153// Therefore, after all instructions have been visited, the pointer splitter
154// post-processes all encountered conditionals. Given a PHI node or select,
155// getPossibleRsrcRoots() collects all values that the resource parts of that
156// conditional's input could come from as well as collecting all conditional
157// instructions encountered during the search. If, after filtering out the
158// initial node itself, the set of encountered conditionals is a subset of the
159// potential roots and there is a single potential resource that isn't in the
160// conditional set, that value is the only possible value the resource argument
161// could have throughout the control flow.
162//
163// If that condition is met, then a PHI node can have its resource part changed
164// to the singleton value and then be replaced by a PHI on the offsets.
165// Otherwise, each PHI node is split into two, one for the resource part and one
166// for the offset part, which replace the temporary `extractvalue` instructions
167// that were added during the first pass.
168//
169// Similar logic applies to `select`, where
170// `%z = select i1 %cond, %cond, ptr addrspace(7) %x, ptr addrspace(7) %y`
171// can be split into `%z.rsrc = %x.rsrc` and
172// `%z.off = select i1 %cond, ptr i32 %x.off, i32 %y.off`
173// if both `%x` and `%y` have the same resource part, but two `select`
174// operations will be needed if they do not.
175//
176// ### Final processing
177//
178// After conditionals have been cleaned up, the IR for each function is
179// rewritten to remove all the old instructions that have been split up.
180//
181// Any instruction that used to produce a buffer fat pointer (and therefore now
182// produces a resource-and-offset struct after type remapping) is
183// replaced as follows:
184// 1. All debug value annotations are cloned to reflect that the resource part
185// and offset parts are computed separately and constitute different
186// fragments of the underlying source language variable.
187// 2. All uses that were themselves split are replaced by a `poison` of the
188// struct type, as they will themselves be erased soon. This rule, combined
189// with debug handling, should leave the use lists of split instructions
190// empty in almost all cases.
191// 3. If a user of the original struct-valued result remains, the structure
192// needed for the new types to work is constructed out of the newly-defined
193// parts, and the original instruction is replaced by this structure
194// before being erased. Instructions requiring this construction include
195// `ret` and `insertvalue`.
196//
197// # Consequences
198//
199// This pass does not alter the CFG.
200//
201// Alias analysis information will become coarser, as the LLVM alias analyzer
202// cannot handle the buffer intrinsics. Specifically, while we can determine
203// that the following two loads do not alias:
204// ```
205// %y = getelementptr i32, ptr addrspace(7) %x, i32 1
206// %a = load i32, ptr addrspace(7) %x
207// %b = load i32, ptr addrspace(7) %y
208// ```
209// we cannot (except through some code that runs during scheduling) determine
210// that the rewritten loads below do not alias.
211// ```
212// %y.off = add i32 %x.off, 1
213// %a = call @llvm.amdgcn.raw.ptr.buffer.load(ptr addrspace(8) %x.rsrc, i32
214// %x.off, ...)
215// %b = call @llvm.amdgcn.raw.ptr.buffer.load(ptr addrspace(8)
216// %x.rsrc, i32 %y.off, ...)
217// ```
218// However, existing alias information is preserved.
219//===----------------------------------------------------------------------===//
220
221#include "AMDGPU.h"
222#include "AMDGPUTargetMachine.h"
223#include "GCNSubtarget.h"
224#include "SIDefines.h"
225#include "llvm/ADT/SetOperations.h"
226#include "llvm/ADT/SmallVector.h"
227#include "llvm/Analysis/InstSimplifyFolder.h"
228#include "llvm/Analysis/ScalarEvolution.h"
229#include "llvm/Analysis/ScalarEvolutionExpressions.h"
230#include "llvm/Analysis/TargetTransformInfo.h"
231#include "llvm/Analysis/Utils/Local.h"
232#include "llvm/CodeGen/TargetPassConfig.h"
233#include "llvm/IR/AttributeMask.h"
234#include "llvm/IR/Constants.h"
235#include "llvm/IR/DebugInfo.h"
236#include "llvm/IR/DerivedTypes.h"
237#include "llvm/IR/IRBuilder.h"
238#include "llvm/IR/InstIterator.h"
239#include "llvm/IR/InstVisitor.h"
240#include "llvm/IR/Instructions.h"
241#include "llvm/IR/IntrinsicInst.h"
242#include "llvm/IR/Intrinsics.h"
243#include "llvm/IR/IntrinsicsAMDGPU.h"
244#include "llvm/IR/Metadata.h"
245#include "llvm/IR/PassManager.h"
246#include "llvm/IR/PatternMatch.h"
247#include "llvm/IR/ReplaceConstant.h"
248#include "llvm/InitializePasses.h"
249#include "llvm/Pass.h"
250#include "llvm/Support/AMDGPUAddrSpace.h"
251#include "llvm/Support/Alignment.h"
252#include "llvm/Support/AtomicOrdering.h"
253#include "llvm/Support/Debug.h"
254#include "llvm/Support/ErrorHandling.h"
255#include "llvm/Support/MathExtras.h"
256#include "llvm/Transforms/Utils/Cloning.h"
257#include "llvm/Transforms/Utils/Local.h"
258#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
259#include "llvm/Transforms/Utils/ValueMapper.h"
260
261#define DEBUG_TYPE "amdgpu-lower-buffer-fat-pointers"
262
263using namespace llvm;
264
265using GetTTIFn = function_ref<const TargetTransformInfo *(Function &)>;
266using GetSEFn = function_ref<ScalarEvolution *(Function &)>;
267
268static constexpr unsigned BufferOffsetWidth = 32;
269
270namespace {
271/// Recursively replace instances of ptr addrspace(7) and vector<Nxptr
272/// addrspace(7)> with some other type as defined by the relevant subclass.
273class BufferFatPtrTypeLoweringBase : public ValueMapTypeRemapper {
274 DenseMap<Type *, Type *> Map;
275
276 Type *remapTypeImpl(Type *Ty);
277
278protected:
279 virtual Type *remapScalar(PointerType *PT) = 0;
280 virtual Type *remapVector(VectorType *VT) = 0;
281
282 const DataLayout &DL;
283
284public:
285 BufferFatPtrTypeLoweringBase(const DataLayout &DL) : DL(DL) {}
286 Type *remapType(Type *SrcTy) override;
287 void clear() { Map.clear(); }
288};
289
290/// Remap ptr addrspace(7) to i160 and vector<Nxptr addrspace(7)> to
291/// vector<Nxi60> in order to correctly handling loading/storing these values
292/// from memory.
293class BufferFatPtrToIntTypeMap : public BufferFatPtrTypeLoweringBase {
294 using BufferFatPtrTypeLoweringBase::BufferFatPtrTypeLoweringBase;
295
296protected:
297 Type *remapScalar(PointerType *PT) override { return DL.getIntPtrType(PT); }
298 Type *remapVector(VectorType *VT) override { return DL.getIntPtrType(VT); }
299};
300
301/// Remap ptr addrspace(7) to {ptr addrspace(8), i32} (the resource and offset
302/// parts of the pointer) so that we can easily rewrite operations on these
303/// values that aren't loading them from or storing them to memory.
304class BufferFatPtrToStructTypeMap : public BufferFatPtrTypeLoweringBase {
305 using BufferFatPtrTypeLoweringBase::BufferFatPtrTypeLoweringBase;
306
307protected:
308 Type *remapScalar(PointerType *PT) override;
309 Type *remapVector(VectorType *VT) override;
310};
311} // namespace
312
313// This code is adapted from the type remapper in lib/Linker/IRMover.cpp
314Type *BufferFatPtrTypeLoweringBase::remapTypeImpl(Type *Ty) {
315 Type **Entry = &Map[Ty];
316 if (*Entry)
317 return *Entry;
318 if (auto *PT = dyn_cast<PointerType>(Val: Ty)) {
319 if (PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER) {
320 return *Entry = remapScalar(PT);
321 }
322 }
323 if (auto *VT = dyn_cast<VectorType>(Val: Ty)) {
324 auto *PT = dyn_cast<PointerType>(Val: VT->getElementType());
325 if (PT && PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER) {
326 return *Entry = remapVector(VT);
327 }
328 return *Entry = Ty;
329 }
330 // Whether the type is one that is structurally uniqued - that is, if it is
331 // not a named struct (the only kind of type where multiple structurally
332 // identical types that have a distinct `Type*`)
333 StructType *TyAsStruct = dyn_cast<StructType>(Val: Ty);
334 bool IsUniqued = !TyAsStruct || TyAsStruct->isLiteral();
335 // Base case for ints, floats, opaque pointers, and so on, which don't
336 // require recursion.
337 if (Ty->getNumContainedTypes() == 0 && IsUniqued)
338 return *Entry = Ty;
339 bool Changed = false;
340 SmallVector<Type *> ElementTypes(Ty->getNumContainedTypes(), nullptr);
341 for (unsigned int I = 0, E = Ty->getNumContainedTypes(); I < E; ++I) {
342 Type *OldElem = Ty->getContainedType(i: I);
343 Type *NewElem = remapTypeImpl(Ty: OldElem);
344 ElementTypes[I] = NewElem;
345 Changed |= (OldElem != NewElem);
346 }
347 // Recursive calls to remapTypeImpl() may have invalidated pointer.
348 Entry = &Map[Ty];
349 if (!Changed) {
350 return *Entry = Ty;
351 }
352 if (auto *ArrTy = dyn_cast<ArrayType>(Val: Ty))
353 return *Entry = ArrayType::get(ElementType: ElementTypes[0], NumElements: ArrTy->getNumElements());
354 if (auto *FnTy = dyn_cast<FunctionType>(Val: Ty))
355 return *Entry = FunctionType::get(Result: ElementTypes[0],
356 Params: ArrayRef(ElementTypes).slice(N: 1),
357 isVarArg: FnTy->isVarArg());
358 if (auto *STy = dyn_cast<StructType>(Val: Ty)) {
359 // Genuine opaque types don't have a remapping.
360 if (STy->isOpaque())
361 return *Entry = Ty;
362 bool IsPacked = STy->isPacked();
363 if (IsUniqued)
364 return *Entry = StructType::get(Context&: Ty->getContext(), Elements: ElementTypes, isPacked: IsPacked);
365 SmallString<16> Name(STy->getName());
366 STy->setName("");
367 return *Entry = StructType::create(Context&: Ty->getContext(), Elements: ElementTypes, Name,
368 isPacked: IsPacked);
369 }
370 llvm_unreachable("Unknown type of type that contains elements");
371}
372
373Type *BufferFatPtrTypeLoweringBase::remapType(Type *SrcTy) {
374 return remapTypeImpl(Ty: SrcTy);
375}
376
377Type *BufferFatPtrToStructTypeMap::remapScalar(PointerType *PT) {
378 LLVMContext &Ctx = PT->getContext();
379 return StructType::get(elt1: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::BUFFER_RESOURCE),
380 elts: IntegerType::get(C&: Ctx, NumBits: BufferOffsetWidth));
381}
382
383Type *BufferFatPtrToStructTypeMap::remapVector(VectorType *VT) {
384 ElementCount EC = VT->getElementCount();
385 LLVMContext &Ctx = VT->getContext();
386 Type *RsrcVec =
387 VectorType::get(ElementType: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::BUFFER_RESOURCE), EC);
388 Type *OffVec = VectorType::get(ElementType: IntegerType::get(C&: Ctx, NumBits: BufferOffsetWidth), EC);
389 return StructType::get(elt1: RsrcVec, elts: OffVec);
390}
391
392static bool isBufferFatPtrOrVector(Type *Ty) {
393 if (auto *PT = dyn_cast<PointerType>(Val: Ty->getScalarType()))
394 return PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER;
395 return false;
396}
397
398// True if the type is {ptr addrspace(8), i32} or a struct containing vectors of
399// those types. Used to quickly skip instructions we don't need to process.
400static bool isSplitFatPtr(Type *Ty) {
401 auto *ST = dyn_cast<StructType>(Val: Ty);
402 if (!ST)
403 return false;
404 if (!ST->isLiteral() || ST->getNumElements() != 2)
405 return false;
406 auto *MaybeRsrc =
407 dyn_cast<PointerType>(Val: ST->getElementType(N: 0)->getScalarType());
408 auto *MaybeOff =
409 dyn_cast<IntegerType>(Val: ST->getElementType(N: 1)->getScalarType());
410 return MaybeRsrc && MaybeOff &&
411 MaybeRsrc->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE &&
412 MaybeOff->getBitWidth() == BufferOffsetWidth;
413}
414
415// True if the result type or any argument types are buffer fat pointers.
416static bool isBufferFatPtrConst(Constant *C) {
417 Type *T = C->getType();
418 return isBufferFatPtrOrVector(Ty: T) || any_of(Range: C->operands(), P: [](const Use &U) {
419 return isBufferFatPtrOrVector(Ty: U.get()->getType());
420 });
421}
422
423namespace {
424/// Convert [vectors of] buffer fat pointers to integers when they are read from
425/// or stored to memory. This ensures that these pointers will have the same
426/// memory layout as before they are lowered, even though they will no longer
427/// have their previous layout in registers/in the program (they'll be broken
428/// down into resource and offset parts). This has the downside of imposing
429/// marshalling costs when reading or storing these values, but since placing
430/// such pointers into memory is an uncommon operation at best, we feel that
431/// this cost is acceptable for better performance in the common case.
432class StoreFatPtrsAsIntsAndExpandMemcpyVisitor
433 : public InstVisitor<StoreFatPtrsAsIntsAndExpandMemcpyVisitor, bool> {
434 BufferFatPtrToIntTypeMap *TypeMap;
435
436 IRBuilder<InstSimplifyFolder> IRB;
437
438 const DataLayout &DL;
439
440 // Used for memcpy() lowering.
441 const TargetTransformInfo *TTI;
442 ScalarEvolution *SE;
443
444 Value *applyOffset(Value *Ptr, uint64_t Off);
445 // Visits each maximal subtree of `Ty` that is fat-ptr-free or is itself a
446 // [vector of] fat pointer(s), at its offset in `Ty`'s original layout.
447 void forEachAggLeaf(
448 Type *Ty, SmallVectorImpl<unsigned> &AggIdxs, uint64_t Off,
449 const Twine &Name,
450 function_ref<void(Type *LeafTy, Type *IntLeafTy, ArrayRef<unsigned> Idxs,
451 uint64_t Off, const Twine &Name)>
452 Visit);
453
454public:
455 StoreFatPtrsAsIntsAndExpandMemcpyVisitor(BufferFatPtrToIntTypeMap *TypeMap,
456 Module &M)
457 : TypeMap(TypeMap), IRB(M, InstSimplifyFolder(M.getDataLayout())),
458 DL(M.getDataLayout()) {}
459 bool processFunction(Function &F, const TargetTransformInfo *TTI,
460 ScalarEvolution *SE);
461
462 bool visitInstruction(Instruction &I) { return false; }
463 bool visitAllocaInst(AllocaInst &I);
464 bool visitLoadInst(LoadInst &LI);
465 bool visitStoreInst(StoreInst &SI);
466 bool visitGetElementPtrInst(GetElementPtrInst &I);
467
468 bool visitMemCpyInst(MemCpyInst &MCI);
469 bool visitMemMoveInst(MemMoveInst &MMI);
470 bool visitMemSetInst(MemSetInst &MSI);
471 bool visitMemSetPatternInst(MemSetPatternInst &MSPI);
472};
473} // namespace
474
475Value *StoreFatPtrsAsIntsAndExpandMemcpyVisitor::applyOffset(Value *Ptr,
476 uint64_t Off) {
477 // The InstSimplifyFolder gives back `Ptr` itself when `Off` is 0.
478 return IRB.CreatePtrAddWithUnsignedOffset(
479 Ptr, Offset: Off, Name: Ptr->getName() + ".off." + Twine(Off),
480 NW: GEPNoWrapFlags::noUnsignedWrap());
481}
482
483void StoreFatPtrsAsIntsAndExpandMemcpyVisitor::forEachAggLeaf(
484 Type *Ty, SmallVectorImpl<unsigned> &AggIdxs, uint64_t Off,
485 const Twine &Name,
486 function_ref<void(Type *LeafTy, Type *IntLeafTy, ArrayRef<unsigned> Idxs,
487 uint64_t Off, const Twine &Name)>
488 Visit) {
489 Type *IntTy = TypeMap->remapType(SrcTy: Ty);
490 if (isBufferFatPtrOrVector(Ty) || Ty == IntTy) {
491 // Zero-sized leaves ({} or [0 x T]) access no bytes; skip them.
492 if (DL.getTypeStoreSize(Ty) != 0)
493 Visit(Ty, IntTy, AggIdxs, Off, Name);
494 return;
495 }
496 auto Recurse = [&](unsigned I, Type *ElemTy, uint64_t ElemOff) {
497 AggIdxs.push_back(Elt: I);
498 forEachAggLeaf(Ty: ElemTy, AggIdxs, Off: Off + ElemOff, Name: Name + "." + Twine(I),
499 Visit);
500 AggIdxs.pop_back();
501 };
502 if (auto *ST = dyn_cast<StructType>(Val: Ty)) {
503 const StructLayout *Layout = DL.getStructLayout(Ty: ST);
504 for (auto [I, ElemTy, ElemOff] :
505 enumerate(First: ST->elements(), Rest: Layout->getMemberOffsets()))
506 Recurse(I, ElemTy, ElemOff.getFixedValue());
507 return;
508 }
509 auto *AT = cast<ArrayType>(Val: Ty);
510 Type *ElemTy = AT->getElementType();
511 uint64_t Stride = DL.getTypeAllocSize(Ty: ElemTy).getFixedValue();
512 for (unsigned I : seq<unsigned>(Size: AT->getNumElements()))
513 Recurse(I, ElemTy, I * Stride);
514}
515
516bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::processFunction(
517 Function &F, const TargetTransformInfo *TTI, ScalarEvolution *SE) {
518 this->TTI = TTI;
519 this->SE = SE;
520 bool Changed = false;
521 // Process memcpy-like instructions after the main iteration because they can
522 // invalidate iterators.
523 SmallVector<WeakTrackingVH> CanBecomeLoops;
524 for (Instruction &I : make_early_inc_range(Range: instructions(F))) {
525 if (isa<MemTransferInst, MemSetInst, MemSetPatternInst>(Val: I))
526 CanBecomeLoops.push_back(Elt: &I);
527 else
528 Changed |= visit(I);
529 }
530 for (WeakTrackingVH VH : make_early_inc_range(Range&: CanBecomeLoops)) {
531 Changed |= visit(I: cast<Instruction>(Val&: VH));
532 }
533 this->TTI = nullptr;
534 this->SE = nullptr;
535 return Changed;
536}
537
538bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitAllocaInst(AllocaInst &I) {
539 Type *Ty = I.getAllocatedType();
540 Type *NewTy = TypeMap->remapType(SrcTy: Ty);
541 if (Ty == NewTy)
542 return false;
543 // i160 is smaller than ptr addrspace(7) (24 bytes vs. 32); fall back to a
544 // byte array of the original size so sizes computed from Ty stay in bounds.
545 TypeSize AllocSize = DL.getTypeAllocSize(Ty);
546 if (AllocSize.isFixed() && DL.getTypeAllocSize(Ty: NewTy) != AllocSize)
547 NewTy = ArrayType::get(ElementType: IRB.getInt8Ty(), NumElements: AllocSize.getFixedValue());
548 I.setAllocatedType(NewTy);
549 return true;
550}
551
552bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitGetElementPtrInst(
553 GetElementPtrInst &I) {
554 Type *Ty = I.getSourceElementType();
555 if (Ty == TypeMap->remapType(SrcTy: Ty))
556 return false;
557 // Lower to a byte offset now, before remapping changes p7's layout (see file
558 // header).
559 IRB.SetInsertPoint(&I);
560 Value *Off = emitGEPOffset(Builder: &IRB, DL, GEP: &I);
561 Value *NewGEP = IRB.CreatePtrAdd(Ptr: I.getPointerOperand(), Offset: Off, Name: I.getName(),
562 NW: I.getNoWrapFlags());
563 I.replaceAllUsesWith(V: NewGEP);
564 I.eraseFromParent();
565 return true;
566}
567
568bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitLoadInst(LoadInst &LI) {
569 Type *Ty = LI.getType();
570 Type *IntTy = TypeMap->remapType(SrcTy: Ty);
571 if (Ty == IntTy)
572 return false;
573
574 IRB.SetInsertPoint(&LI);
575 if (!isBufferFatPtrOrVector(Ty)) {
576 // i160 has the same 20-byte store size as p7, so loading each leaf at
577 // its original-layout offset accesses the same bytes as the unlowered load.
578 Value *Agg = PoisonValue::get(T: Ty);
579 AAMDNodes AATags = LI.getAAMetadata();
580 SmallVector<unsigned> AggIdxs;
581 forEachAggLeaf(
582 Ty, AggIdxs, Off: 0, Name: LI.getName(),
583 Visit: [&](Type *LeafTy, Type *IntLeafTy, ArrayRef<unsigned> Idxs,
584 uint64_t Off, const Twine &Name) {
585 Value *Ptr = applyOffset(Ptr: LI.getPointerOperand(), Off);
586 LoadInst *NewLI = IRB.CreateAlignedLoad(
587 Ty: IntLeafTy, Ptr, Align: commonAlignment(A: LI.getAlign(), Offset: Off), Name);
588 NewLI->setVolatile(LI.isVolatile());
589 copyMetadataForLoad(Dest&: *NewLI, Source: LI);
590 NewLI->setAAMetadata(AATags.adjustForAccess(Offset: Off, AccessTy: IntLeafTy, DL));
591 Value *V = NewLI;
592 if (LeafTy != IntLeafTy)
593 V = IRB.CreateIntToPtr(V: NewLI, DestTy: LeafTy, Name: Name + ".ptr");
594 Agg = IRB.CreateInsertValue(Agg, Val: V, Idxs, Name: Name + ".agg");
595 });
596 LI.replaceAllUsesWith(V: Agg);
597 LI.eraseFromParent();
598 return true;
599 }
600 auto *NLI = cast<LoadInst>(Val: LI.clone());
601 NLI->mutateType(Ty: IntTy);
602 NLI = IRB.Insert(I: NLI);
603 NLI->takeName(V: &LI);
604
605 Value *CastBack = IRB.CreateIntToPtr(V: NLI, DestTy: Ty, Name: NLI->getName() + ".ptr");
606 LI.replaceAllUsesWith(V: CastBack);
607 LI.eraseFromParent();
608 return true;
609}
610
611bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitStoreInst(StoreInst &SI) {
612 Value *V = SI.getValueOperand();
613 Type *Ty = V->getType();
614 Type *IntTy = TypeMap->remapType(SrcTy: Ty);
615 if (Ty == IntTy)
616 return false;
617
618 IRB.SetInsertPoint(&SI);
619 if (!isBufferFatPtrOrVector(Ty)) {
620 // Store each leaf at its byte offset in the original layout; see
621 // visitLoadInst.
622 AAMDNodes AATags = SI.getAAMetadata();
623 SmallVector<unsigned> AggIdxs;
624 forEachAggLeaf(
625 Ty, AggIdxs, Off: 0, Name: V->getName(),
626 Visit: [&](Type *LeafTy, Type *IntLeafTy, ArrayRef<unsigned> Idxs,
627 uint64_t Off, const Twine &Name) {
628 Value *Leaf = IRB.CreateExtractValue(Agg: V, Idxs, Name);
629 if (LeafTy != IntLeafTy)
630 Leaf = IRB.CreatePtrToInt(V: Leaf, DestTy: IntLeafTy, Name: Name + ".int");
631 auto *NewSI = cast<StoreInst>(Val: SI.clone());
632 NewSI->setAlignment(commonAlignment(A: SI.getAlign(), Offset: Off));
633 NewSI->setOperand(i_nocapture: 0, Val_nocapture: Leaf);
634 NewSI->setOperand(i_nocapture: 1, Val_nocapture: applyOffset(Ptr: SI.getPointerOperand(), Off));
635 // Each leaf covers only part of the original assignment.
636 NewSI->setMetadata(KindID: LLVMContext::MD_DIAssignID, Node: nullptr);
637 IRB.Insert(I: NewSI);
638 NewSI->setAAMetadata(AATags.adjustForAccess(Offset: Off, AccessTy: IntLeafTy, DL));
639 });
640 SI.eraseFromParent();
641 return true;
642 }
643 Value *IntV = IRB.CreatePtrToInt(V, DestTy: IntTy, Name: V->getName() + ".int");
644 for (auto *Dbg : at::getDVRAssignmentMarkers(Inst: &SI))
645 Dbg->setRawLocation(ValueAsMetadata::get(V: IntV));
646
647 SI.setOperand(i_nocapture: 0, Val_nocapture: IntV);
648 return true;
649}
650
651bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitMemCpyInst(
652 MemCpyInst &MCI) {
653 // TODO: Allow memcpy.p7.p3 as a synonym for the direct-to-LDS copy, which'll
654 // need loop expansion here.
655 if (MCI.getSourceAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER &&
656 MCI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
657 return false;
658 llvm::expandMemCpyAsLoop(MemCpy: &MCI, TTI: *TTI, SE);
659 MCI.eraseFromParent();
660 return true;
661}
662
663bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitMemMoveInst(
664 MemMoveInst &MMI) {
665 if (MMI.getSourceAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER &&
666 MMI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
667 return false;
668 reportFatalUsageError(
669 reason: "memmove() on buffer descriptors is not implemented because pointer "
670 "comparison on buffer descriptors isn't implemented\n");
671}
672
673bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitMemSetInst(
674 MemSetInst &MSI) {
675 if (MSI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
676 return false;
677 llvm::expandMemSetAsLoop(MemSet: &MSI, TTI);
678 MSI.eraseFromParent();
679 return true;
680}
681
682bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitMemSetPatternInst(
683 MemSetPatternInst &MSPI) {
684 if (MSPI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
685 return false;
686 llvm::expandMemSetPatternAsLoop(MemSet: &MSPI, TTI: *TTI);
687 MSPI.eraseFromParent();
688 return true;
689}
690
691namespace {
692/// Convert loads/stores of types that the buffer intrinsics can't handle into
693/// one ore more such loads/stores that consist of legal types.
694///
695/// Do this by
696/// 1. Recursing into structs (and arrays that don't share a memory layout with
697/// vectors) since the intrinsics can't handle complex types.
698/// 2. Converting arrays of non-aggregate, byte-sized types into their
699/// corresponding vectors
700/// 3. Bitcasting unsupported types, namely overly-long scalars and byte
701/// vectors, into vectors of supported types.
702/// 4. Splitting up excessively long reads/writes into multiple operations.
703///
704/// Note that this doesn't handle complex data strucures, but, in the future,
705/// the aggregate load splitter from SROA could be refactored to allow for that
706/// case.
707///
708/// Note that, if we can prove that the initial value of the pointer offset is 0
709/// and that the load/store won't wrap from the left or won't have bounds checks
710/// that straddle a word boundary, we can emit some of the strict bounds
711/// checking pessimizations even in strict OOB mode, and we attempt to do so.
712class LegalizeBufferContentTypesVisitor
713 : public InstVisitor<LegalizeBufferContentTypesVisitor, bool> {
714 friend class InstVisitor<LegalizeBufferContentTypesVisitor, bool>;
715
716 IRBuilder<InstSimplifyFolder> IRB;
717
718 const DataLayout &DL;
719
720 ScalarEvolution *SE = nullptr;
721
722 // Map base (non-GEP'd) pointers to the number of records they have, if known.
723 // If a pointer is known to have a starting offset of 0 but it wasn't known to
724 // have a number of records (ex. it was `addrspacecast` from a buffer
725 // resource), it will be present in this map, but the key will be null.
726 // Otherwise, there will be no map entry.
727 ValueToValueMapTy ZeroBasePointerToNumRecords;
728
729 // Subtarget info, needed for determining what cache control bits to set.
730 const TargetMachine *TM;
731 const GCNSubtarget *ST = nullptr;
732
733 /// If T is [N x U], where U is a scalar type, return the vector type
734 /// <N x U>, otherwise, return T.
735 Type *scalarArrayTypeAsVector(Type *MaybeArrayType);
736 Value *arrayToVector(Value *V, Type *TargetType, const Twine &Name);
737 Value *vectorToArray(Value *V, Type *OrigType, const Twine &Name);
738
739 /// Analyze how a given buffer access could be out of bounds. Used to optimize
740 /// the strict splitting used in strict bounds checking mode.
741 struct OobProperties {
742 // Offset is far enough from all-1s that we won't get wrapping around to 0.
743 bool NoWrapFromMax = false;
744 // Offset is either entirely in-bounds or entirely out of bounds.
745 bool NoPartialOOB = false;
746
747 OobProperties() = delete;
748 // Needed for some Clangs.
749 OobProperties(bool NoWrapFromMax, bool NoPartialOOB)
750 : NoWrapFromMax(NoWrapFromMax), NoPartialOOB(NoPartialOOB) {}
751 };
752 OobProperties analyzeOobProperties(Value *Ptr, Type *Ty, uint64_t ByteOffset);
753
754 /// Break up the loads of a struct into the loads of its components
755
756 /// Convert a vector or scalar type that can't be operated on by buffer
757 /// intrinsics to one that would be legal through bitcasts and/or truncation.
758 /// Uses the wider of i32, i16, or i8 where possible, clamping to the maximum
759 /// allowed width under the alignment rules and subtarget flags.
760 Type *legalNonAggregateForMemOp(Type *T, uint64_t MaxWidth);
761 Value *makeLegalNonAggregate(Value *V, Type *TargetType, const Twine &Name);
762 Value *makeIllegalNonAggregate(Value *V, Type *OrigType, const Twine &Name);
763
764 struct VecSlice {
765 uint64_t Index = 0;
766 uint64_t Length = 0;
767 VecSlice() = delete;
768 // Needed for some Clangs
769 VecSlice(uint64_t Index, uint64_t Length) : Index(Index), Length(Length) {}
770 };
771 /// Return the [index, length] pairs into which `T` needs to be cut to form
772 /// legal buffer load or store operations. Clears `Slices`. Creates an empty
773 /// `Slices` for non-vector inputs and creates one slice if no slicing will be
774 /// needed. No slice may be larger than `MaxWidth`.
775 void getVecSlices(Type *T, uint64_t MaxWidth,
776 SmallVectorImpl<VecSlice> &Slices);
777
778 Value *extractSlice(Value *Vec, VecSlice S, const Twine &Name);
779 Value *insertSlice(Value *Whole, Value *Part, VecSlice S, const Twine &Name);
780
781 /// In most cases, return `LegalType`. However, when given an input that would
782 /// normally be a legal type for the buffer intrinsics to return but that
783 /// isn't hooked up through SelectionDAG, return a type of the same width that
784 /// can be used with the relevant intrinsics. Specifically, handle the cases:
785 /// - <1 x T> => T for all T
786 /// - <N x i8> <=> i16, i32, 2xi32, 4xi32 (as needed)
787 /// - <N x T> where T is under 32 bits and the total size is 96 bits <=> <3 x
788 /// i32>
789 Type *intrinsicTypeFor(Type *LegalType);
790
791 bool visitLoadImpl(LoadInst &OrigLI, Type *PartType,
792 SmallVectorImpl<uint32_t> &AggIdxs, uint64_t AggByteOffset,
793 Value *&Result, const Twine &Name);
794 /// Return value is (Changed, ModifiedInPlace)
795 std::pair<bool, bool> visitStoreImpl(StoreInst &OrigSI, Type *PartType,
796 SmallVectorImpl<uint32_t> &AggIdxs,
797 uint64_t AggByteOffset,
798 const Twine &Name);
799
800 bool visitInstruction(Instruction &I) { return false; }
801 bool visitLoadInst(LoadInst &LI);
802 bool visitStoreInst(StoreInst &SI);
803
804 // Record base pointer data and num_records (if known).
805 bool visitIntrinsicInst(IntrinsicInst &II);
806 bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASCI);
807
808public:
809 LegalizeBufferContentTypesVisitor(Module &M, const TargetMachine *TM)
810 : IRB(M, InstSimplifyFolder(M.getDataLayout())), DL(M.getDataLayout()),
811 TM(TM) {}
812 bool processFunction(Function &F, ScalarEvolution *SE);
813};
814} // namespace
815
816Type *LegalizeBufferContentTypesVisitor::scalarArrayTypeAsVector(Type *T) {
817 ArrayType *AT = dyn_cast<ArrayType>(Val: T);
818 if (!AT)
819 return T;
820 Type *ET = AT->getElementType();
821 if (!ET->isSingleValueType() || isa<VectorType>(Val: ET))
822 reportFatalUsageError(reason: "loading non-scalar arrays from buffer fat pointers "
823 "should have recursed");
824 if (!DL.typeSizeEqualsStoreSize(Ty: AT))
825 reportFatalUsageError(
826 reason: "loading padded arrays from buffer fat pinters should have recursed");
827 return FixedVectorType::get(ElementType: ET, NumElts: AT->getNumElements());
828}
829
830Value *LegalizeBufferContentTypesVisitor::arrayToVector(Value *V,
831 Type *TargetType,
832 const Twine &Name) {
833 Value *VectorRes = PoisonValue::get(T: TargetType);
834 auto *VT = cast<FixedVectorType>(Val: TargetType);
835 unsigned EC = VT->getNumElements();
836 for (auto I : iota_range<unsigned>(0, EC, /*Inclusive=*/false)) {
837 Value *Elem = IRB.CreateExtractValue(Agg: V, Idxs: I, Name: Name + ".elem." + Twine(I));
838 VectorRes = IRB.CreateInsertElement(Vec: VectorRes, NewElt: Elem, Idx: I,
839 Name: Name + ".as.vec." + Twine(I));
840 }
841 return VectorRes;
842}
843
844Value *LegalizeBufferContentTypesVisitor::vectorToArray(Value *V,
845 Type *OrigType,
846 const Twine &Name) {
847 Value *ArrayRes = PoisonValue::get(T: OrigType);
848 ArrayType *AT = cast<ArrayType>(Val: OrigType);
849 unsigned EC = AT->getNumElements();
850 for (auto I : iota_range<unsigned>(0, EC, /*Inclusive=*/false)) {
851 Value *Elem = IRB.CreateExtractElement(Vec: V, Idx: I, Name: Name + ".elem." + Twine(I));
852 ArrayRes = IRB.CreateInsertValue(Agg: ArrayRes, Val: Elem, Idxs: I,
853 Name: Name + ".as.array." + Twine(I));
854 }
855 return ArrayRes;
856}
857
858LegalizeBufferContentTypesVisitor::OobProperties
859LegalizeBufferContentTypesVisitor::analyzeOobProperties(Value *Ptr, Type *Ty,
860 uint64_t ByteOffset) {
861 OobProperties Result(false, false);
862
863 if (ST->hasRelaxedBufferOOBMode())
864 return OobProperties(true, true);
865
866 if (!SE)
867 return Result;
868 if (!SE->isSCEVable(Ty: Ptr->getType()))
869 return Result;
870 const SCEV *PtrOp = SE->getSCEV(V: Ptr);
871 if (ByteOffset > 0)
872 PtrOp = SE->getAddExpr(LHS: PtrOp, RHS: SE->getConstant(V: IRB.getInt32(C: ByteOffset)));
873 const auto *PtrBase = dyn_cast<SCEVUnknown>(Val: SE->getPointerBase(V: PtrOp));
874 if (!PtrBase)
875 return Result;
876 Value *PtrBaseVal = PtrBase->getValue();
877 // We don't know if the offset field started at 0, so there's no safe analysis
878 // we can do. If it weren't for the fact that nuw / inbounds / ... are
879 // properties of the pointer, we might be able to use hem, but loads where the
880 // address computation for sub-parts of the loaded type wraps the address
881 // space are explicitly in scope here so there's not much we can do inside
882 // functions that can't "see" the fat pointer creation.
883 auto NumRecordsIfKnown = ZeroBasePointerToNumRecords.find(Val: PtrBaseVal);
884 if (NumRecordsIfKnown == ZeroBasePointerToNumRecords.end())
885 return Result;
886
887 unsigned TypeSize = DL.getTypeStoreSize(Ty).getKnownMinValue();
888 const SCEV *PtrDiff = SE->getMinusSCEV(LHS: PtrOp, RHS: PtrBase);
889 APInt MaxNoWrapOffset = APInt::getAllOnes(numBits: BufferOffsetWidth) - TypeSize;
890 if (SE->isKnownNonNegative(S: PtrDiff) ||
891 SE->getUnsignedRangeMax(S: PtrDiff).ule(RHS: MaxNoWrapOffset))
892 Result.NoWrapFromMax = true;
893
894 // If we know that the pointer is zero-based but not what its upper bound is,
895 // we'll need to split up underaligned loads of small types.
896 if (!NumRecordsIfKnown->second)
897 return Result;
898 const SCEV *NumRecords = SE->getSCEV(V: NumRecordsIfKnown->second);
899
900 // We'll normalize all bounds to the num_records width on the hardware.
901 std::optional<unsigned> MaybeNumRecordsWidth =
902 ST->getBufferResourceNumRecordsWidth();
903 if (!MaybeNumRecordsWidth)
904 return Result;
905 unsigned NumRecordsWidth = *MaybeNumRecordsWidth;
906 Type *NumRecordsTy = IRB.getIntNTy(N: NumRecordsWidth);
907 // Compare in i64 so wraparound is visible as a negative.
908 Type *CompareTy = IRB.getInt64Ty();
909 const SCEV *Bound = SE->getNoopOrZeroExtend(
910 V: SE->getTruncateOrZeroExtend(V: NumRecords, Ty: NumRecordsTy), Ty: CompareTy);
911
912 // All-1s is (per ISA or as a consequence of the bounds check rules, depending
913 // on architecture) no bounds check.
914 if (Bound == SE->getConstant(Val: APInt::getMaxValue(numBits: NumRecordsWidth)
915 .zext(width: CompareTy->getIntegerBitWidth())))
916 Result.NoPartialOOB = true;
917
918 const SCEV *BoundsDiff =
919 SE->getMinusSCEV(LHS: Bound, RHS: SE->getNoopOrZeroExtend(V: PtrDiff, Ty: CompareTy));
920
921 if (SE->getSignedRangeMin(S: BoundsDiff).sge(RHS: TypeSize) ||
922 SE->isKnownNonPositive(S: BoundsDiff))
923 Result.NoPartialOOB = true;
924 return Result;
925}
926
927Type *LegalizeBufferContentTypesVisitor::legalNonAggregateForMemOp(
928 Type *T, uint64_t MaxWidth) {
929 TypeSize Size = DL.getTypeStoreSizeInBits(Ty: T);
930 // Implicitly zero-extend to the next byte if needed.
931 if (!DL.typeSizeEqualsStoreSize(Ty: T))
932 T = IRB.getIntNTy(N: Size.getFixedValue());
933 Type *ElemTy = T->getScalarType();
934 if (isa<PointerType, ScalableVectorType>(Val: ElemTy)) {
935 // Pointers are always big enough, and we'll let scalable vectors through to
936 // fail in codegen.
937 return T;
938 }
939 unsigned ElemSize = DL.getTypeSizeInBits(Ty: ElemTy).getFixedValue();
940 if (isPowerOf2_32(Value: ElemSize) && ElemSize >= 16 && ElemSize <= MaxWidth) {
941 // [vectors of] anything that's 16/32/64/128 bits can be cast and split into
942 // legal buffer operations, except that we might need to cut them into
943 // smaller values if we're not allowed to do unaligned vector loads.
944 return T;
945 }
946 Type *BestVectorElemType = nullptr;
947 if (Size.isKnownMultipleOf(RHS: 32) && MaxWidth >= 32)
948 BestVectorElemType = IRB.getInt32Ty();
949 else if (Size.isKnownMultipleOf(RHS: 16) && MaxWidth >= 16)
950 BestVectorElemType = IRB.getInt16Ty();
951 else
952 BestVectorElemType = IRB.getInt8Ty();
953 unsigned NumCastElems =
954 Size.getFixedValue() / BestVectorElemType->getIntegerBitWidth();
955 if (NumCastElems == 1)
956 return BestVectorElemType;
957 return FixedVectorType::get(ElementType: BestVectorElemType, NumElts: NumCastElems);
958}
959
960Value *LegalizeBufferContentTypesVisitor::makeLegalNonAggregate(
961 Value *V, Type *TargetType, const Twine &Name) {
962 Type *SourceType = V->getType();
963 TypeSize SourceSize = DL.getTypeSizeInBits(Ty: SourceType);
964 TypeSize TargetSize = DL.getTypeSizeInBits(Ty: TargetType);
965 if (SourceSize != TargetSize) {
966 Type *ShortScalarTy = IRB.getIntNTy(N: SourceSize.getFixedValue());
967 Type *ByteScalarTy = IRB.getIntNTy(N: TargetSize.getFixedValue());
968 Value *AsScalar = IRB.CreateBitCast(V, DestTy: ShortScalarTy, Name: Name + ".as.scalar");
969 Value *Zext = IRB.CreateZExt(V: AsScalar, DestTy: ByteScalarTy, Name: Name + ".zext");
970 V = Zext;
971 SourceType = ByteScalarTy;
972 }
973 return IRB.CreateBitCast(V, DestTy: TargetType, Name: Name + ".legal");
974}
975
976Value *LegalizeBufferContentTypesVisitor::makeIllegalNonAggregate(
977 Value *V, Type *OrigType, const Twine &Name) {
978 Type *LegalType = V->getType();
979 TypeSize LegalSize = DL.getTypeSizeInBits(Ty: LegalType);
980 TypeSize OrigSize = DL.getTypeSizeInBits(Ty: OrigType);
981 if (LegalSize != OrigSize) {
982 Type *ShortScalarTy = IRB.getIntNTy(N: OrigSize.getFixedValue());
983 Type *ByteScalarTy = IRB.getIntNTy(N: LegalSize.getFixedValue());
984 Value *AsScalar = IRB.CreateBitCast(V, DestTy: ByteScalarTy, Name: Name + ".bytes.cast");
985 Value *Trunc = IRB.CreateTrunc(V: AsScalar, DestTy: ShortScalarTy, Name: Name + ".trunc");
986 return IRB.CreateBitCast(V: Trunc, DestTy: OrigType, Name: Name + ".orig");
987 }
988 return IRB.CreateBitCast(V, DestTy: OrigType, Name: Name + ".real.ty");
989}
990
991Type *LegalizeBufferContentTypesVisitor::intrinsicTypeFor(Type *LegalType) {
992 auto *VT = dyn_cast<FixedVectorType>(Val: LegalType);
993 if (!VT)
994 return LegalType;
995 Type *ET = VT->getElementType();
996 // Explicitly return the element type of 1-element vectors because the
997 // underlying intrinsics don't like <1 x T> even though it's a synonym for T.
998 if (VT->getNumElements() == 1)
999 return ET;
1000 if (DL.getTypeSizeInBits(Ty: LegalType) == 96 && DL.getTypeSizeInBits(Ty: ET) < 32)
1001 return FixedVectorType::get(ElementType: IRB.getInt32Ty(), NumElts: 3);
1002 if (ET->isIntegerTy(BitWidth: 8)) {
1003 switch (VT->getNumElements()) {
1004 default:
1005 return LegalType; // Let it crash later
1006 case 1:
1007 return IRB.getInt8Ty();
1008 case 2:
1009 return IRB.getInt16Ty();
1010 case 4:
1011 return IRB.getInt32Ty();
1012 case 8:
1013 return FixedVectorType::get(ElementType: IRB.getInt32Ty(), NumElts: 2);
1014 case 16:
1015 return FixedVectorType::get(ElementType: IRB.getInt32Ty(), NumElts: 4);
1016 }
1017 }
1018 return LegalType;
1019}
1020
1021void LegalizeBufferContentTypesVisitor::getVecSlices(
1022 Type *T, uint64_t MaxWidth, SmallVectorImpl<VecSlice> &Slices) {
1023 Slices.clear();
1024 auto *VT = dyn_cast<FixedVectorType>(Val: T);
1025 if (!VT)
1026 return;
1027
1028 uint64_t ElemBitWidth =
1029 DL.getTypeSizeInBits(Ty: VT->getElementType()).getFixedValue();
1030
1031 uint64_t ElemsPer4Words = 128 / ElemBitWidth;
1032 uint64_t ElemsPer2Words = ElemsPer4Words / 2;
1033 uint64_t ElemsPerWord = ElemsPer2Words / 2;
1034 uint64_t ElemsPerShort = ElemsPerWord / 2;
1035 uint64_t ElemsPerByte = ElemsPerShort / 2;
1036 // If the elements evenly pack into 32-bit words, we can use 3-word stores,
1037 // such as for <6 x bfloat> or <3 x i32>, but we can't dot his for, for
1038 // example, <3 x i64>, since that's not slicing.
1039 uint64_t ElemsPer3Words = ElemsPerWord * 3;
1040
1041 uint64_t TotalElems = VT->getNumElements();
1042 uint64_t Index = 0;
1043 auto TrySlice = [&](unsigned MaybeLen, unsigned Width) {
1044 if (MaybeLen > 0 && Width <= MaxWidth && Index + MaybeLen <= TotalElems) {
1045 VecSlice Slice{/*Index=*/Index, /*Length=*/MaybeLen};
1046 Slices.push_back(Elt: Slice);
1047 Index += MaybeLen;
1048 return true;
1049 }
1050 return false;
1051 };
1052 while (Index < TotalElems) {
1053 TrySlice(ElemsPer4Words, 128) || TrySlice(ElemsPer3Words, 96) ||
1054 TrySlice(ElemsPer2Words, 64) || TrySlice(ElemsPerWord, 32) ||
1055 TrySlice(ElemsPerShort, 16) || TrySlice(ElemsPerByte, 8);
1056 }
1057}
1058
1059Value *LegalizeBufferContentTypesVisitor::extractSlice(Value *Vec, VecSlice S,
1060 const Twine &Name) {
1061 auto *VecVT = dyn_cast<FixedVectorType>(Val: Vec->getType());
1062 if (!VecVT)
1063 return Vec;
1064 if (S.Length == VecVT->getNumElements() && S.Index == 0)
1065 return Vec;
1066 if (S.Length == 1)
1067 return IRB.CreateExtractElement(Vec, Idx: S.Index,
1068 Name: Name + ".slice." + Twine(S.Index));
1069 SmallVector<int> Mask = llvm::to_vector(
1070 Range: llvm::iota_range<int>(S.Index, S.Index + S.Length, /*Inclusive=*/false));
1071 return IRB.CreateShuffleVector(V: Vec, Mask, Name: Name + ".slice." + Twine(S.Index));
1072}
1073
1074Value *LegalizeBufferContentTypesVisitor::insertSlice(Value *Whole, Value *Part,
1075 VecSlice S,
1076 const Twine &Name) {
1077 auto *WholeVT = dyn_cast<FixedVectorType>(Val: Whole->getType());
1078 if (!WholeVT)
1079 return Part;
1080 if (S.Length == WholeVT->getNumElements() && S.Index == 0)
1081 return Part;
1082 if (S.Length == 1) {
1083 return IRB.CreateInsertElement(Vec: Whole, NewElt: Part, Idx: S.Index,
1084 Name: Name + ".slice." + Twine(S.Index));
1085 }
1086 int NumElems = cast<FixedVectorType>(Val: Whole->getType())->getNumElements();
1087
1088 // Extend the slice with poisons to make the main shufflevector happy.
1089 SmallVector<int> ExtPartMask(NumElems, -1);
1090 for (auto [I, E] : llvm::enumerate(
1091 First: MutableArrayRef<int>(ExtPartMask).take_front(N: S.Length))) {
1092 E = I;
1093 }
1094 Value *ExtPart = IRB.CreateShuffleVector(V: Part, Mask: ExtPartMask,
1095 Name: Name + ".ext." + Twine(S.Index));
1096
1097 SmallVector<int> Mask =
1098 llvm::to_vector(Range: llvm::iota_range<int>(0, NumElems, /*Inclusive=*/false));
1099 for (auto [I, E] :
1100 llvm::enumerate(First: MutableArrayRef<int>(Mask).slice(N: S.Index, M: S.Length)))
1101 E = I + NumElems;
1102 return IRB.CreateShuffleVector(V1: Whole, V2: ExtPart, Mask,
1103 Name: Name + ".parts." + Twine(S.Index));
1104}
1105
1106bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
1107 LoadInst &OrigLI, Type *PartType, SmallVectorImpl<uint32_t> &AggIdxs,
1108 uint64_t AggByteOff, Value *&Result, const Twine &Name) {
1109 if (auto *ST = dyn_cast<StructType>(Val: PartType)) {
1110 const StructLayout *Layout = DL.getStructLayout(Ty: ST);
1111 bool Changed = false;
1112 for (auto [I, ElemTy, Offset] :
1113 llvm::enumerate(First: ST->elements(), Rest: Layout->getMemberOffsets())) {
1114 AggIdxs.push_back(Elt: I);
1115 Changed |= visitLoadImpl(OrigLI, PartType: ElemTy, AggIdxs,
1116 AggByteOff: AggByteOff + Offset.getFixedValue(), Result,
1117 Name: Name + "." + Twine(I));
1118 AggIdxs.pop_back();
1119 }
1120 return Changed;
1121 }
1122 if (auto *AT = dyn_cast<ArrayType>(Val: PartType)) {
1123 Type *ElemTy = AT->getElementType();
1124 if (!ElemTy->isSingleValueType() || !DL.typeSizeEqualsStoreSize(Ty: ElemTy) ||
1125 ElemTy->isVectorTy()) {
1126 TypeSize ElemAllocSize = DL.getTypeAllocSize(Ty: ElemTy);
1127 bool Changed = false;
1128 for (auto I : llvm::iota_range<uint32_t>(0, AT->getNumElements(),
1129 /*Inclusive=*/false)) {
1130 AggIdxs.push_back(Elt: I);
1131 Changed |= visitLoadImpl(OrigLI, PartType: ElemTy, AggIdxs,
1132 AggByteOff: AggByteOff + I * ElemAllocSize.getFixedValue(),
1133 Result, Name: Name + Twine(I));
1134 AggIdxs.pop_back();
1135 }
1136 return Changed;
1137 }
1138 }
1139
1140 // Typical case
1141
1142 Align PartAlign = commonAlignment(A: OrigLI.getAlign(), Offset: AggByteOff);
1143 Type *ArrayAsVecType = scalarArrayTypeAsVector(T: PartType);
1144 OobProperties OobProps =
1145 analyzeOobProperties(Ptr: OrigLI.getPointerOperand(), Ty: PartType, ByteOffset: AggByteOff);
1146 uint64_t MaxWidth = ST->getMaxBufferIntrinsicWidth(
1147 DL, T: ArrayAsVecType, A: PartAlign, NoWrapFromMax: OobProps.NoWrapFromMax,
1148 NoPartialOOB: OobProps.NoPartialOOB);
1149 Type *LegalType = legalNonAggregateForMemOp(T: ArrayAsVecType, MaxWidth);
1150
1151 SmallVector<VecSlice> Slices;
1152 getVecSlices(T: LegalType, MaxWidth, Slices);
1153 bool HasSlices = Slices.size() > 1;
1154 bool IsAggPart = !AggIdxs.empty();
1155 Value *LoadsRes;
1156 if (!HasSlices && !IsAggPart) {
1157 Type *LoadableType = intrinsicTypeFor(LegalType);
1158 if (LoadableType == PartType)
1159 return false;
1160
1161 IRB.SetInsertPoint(&OrigLI);
1162 auto *NLI = cast<LoadInst>(Val: OrigLI.clone());
1163 NLI->mutateType(Ty: LoadableType);
1164 NLI = IRB.Insert(I: NLI);
1165 NLI->setName(Name + ".loadable");
1166
1167 LoadsRes = IRB.CreateBitCast(V: NLI, DestTy: LegalType, Name: Name + ".from.loadable");
1168 } else {
1169 IRB.SetInsertPoint(&OrigLI);
1170 LoadsRes = PoisonValue::get(T: LegalType);
1171 Value *OrigPtr = OrigLI.getPointerOperand();
1172 // If we're needing to spill something into more than one load, its legal
1173 // type will be a vector (ex. an i256 load will have LegalType = <8 x i32>).
1174 // But if we're already a scalar (which can happen if we're splitting up a
1175 // struct), the element type will be the legal type itself.
1176 Type *ElemType = LegalType->getScalarType();
1177 unsigned ElemBytes = DL.getTypeStoreSize(Ty: ElemType);
1178 AAMDNodes AANodes = OrigLI.getAAMetadata();
1179 if (IsAggPart && Slices.empty())
1180 Slices.push_back(Elt: VecSlice{/*Index=*/0, /*Length=*/1});
1181 for (VecSlice S : Slices) {
1182 Type *SliceType =
1183 S.Length != 1 ? FixedVectorType::get(ElementType: ElemType, NumElts: S.Length) : ElemType;
1184 int64_t ByteOffset = AggByteOff + S.Index * ElemBytes;
1185 // You can't reasonably expect loads to wrap around the edge of memory.
1186 Value *NewPtr = IRB.CreateGEP(
1187 Ty: IRB.getInt8Ty(), Ptr: OrigLI.getPointerOperand(), IdxList: IRB.getInt32(C: ByteOffset),
1188 Name: OrigPtr->getName() + ".off.ptr." + Twine(ByteOffset),
1189 NW: ST->hasRelaxedBufferOOBMode() ? GEPNoWrapFlags::noUnsignedWrap()
1190 : GEPNoWrapFlags::none());
1191 Type *LoadableType = intrinsicTypeFor(LegalType: SliceType);
1192 LoadInst *NewLI = IRB.CreateAlignedLoad(
1193 Ty: LoadableType, Ptr: NewPtr, Align: commonAlignment(A: OrigLI.getAlign(), Offset: ByteOffset),
1194 Name: Name + ".off." + Twine(ByteOffset));
1195 copyMetadataForLoad(Dest&: *NewLI, Source: OrigLI);
1196 NewLI->setAAMetadata(
1197 AANodes.adjustForAccess(Offset: ByteOffset, AccessTy: LoadableType, DL));
1198 NewLI->setAtomic(Ordering: OrigLI.getOrdering(), SSID: OrigLI.getSyncScopeID());
1199 NewLI->setVolatile(OrigLI.isVolatile());
1200 Value *Loaded = IRB.CreateBitCast(V: NewLI, DestTy: SliceType,
1201 Name: NewLI->getName() + ".from.loadable");
1202 LoadsRes = insertSlice(Whole: LoadsRes, Part: Loaded, S, Name);
1203 }
1204 }
1205 if (LegalType != ArrayAsVecType)
1206 LoadsRes = makeIllegalNonAggregate(V: LoadsRes, OrigType: ArrayAsVecType, Name);
1207 if (ArrayAsVecType != PartType)
1208 LoadsRes = vectorToArray(V: LoadsRes, OrigType: PartType, Name);
1209
1210 if (IsAggPart)
1211 Result = IRB.CreateInsertValue(Agg: Result, Val: LoadsRes, Idxs: AggIdxs, Name);
1212 else
1213 Result = LoadsRes;
1214 return true;
1215}
1216
1217bool LegalizeBufferContentTypesVisitor::visitLoadInst(LoadInst &LI) {
1218 if (LI.getPointerAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
1219 return false;
1220
1221 SmallVector<uint32_t> AggIdxs;
1222 Type *OrigType = LI.getType();
1223 Value *Result = PoisonValue::get(T: OrigType);
1224 bool Changed = visitLoadImpl(OrigLI&: LI, PartType: OrigType, AggIdxs, AggByteOff: 0, Result, Name: LI.getName());
1225 if (!Changed)
1226 return false;
1227 Result->takeName(V: &LI);
1228 LI.replaceAllUsesWith(V: Result);
1229 LI.eraseFromParent();
1230 return Changed;
1231}
1232
1233std::pair<bool, bool> LegalizeBufferContentTypesVisitor::visitStoreImpl(
1234 StoreInst &OrigSI, Type *PartType, SmallVectorImpl<uint32_t> &AggIdxs,
1235 uint64_t AggByteOff, const Twine &Name) {
1236 if (auto *ST = dyn_cast<StructType>(Val: PartType)) {
1237 const StructLayout *Layout = DL.getStructLayout(Ty: ST);
1238 bool Changed = false;
1239 for (auto [I, ElemTy, Offset] :
1240 llvm::enumerate(First: ST->elements(), Rest: Layout->getMemberOffsets())) {
1241 AggIdxs.push_back(Elt: I);
1242 Changed |= visitStoreImpl(OrigSI, PartType: ElemTy, AggIdxs,
1243 AggByteOff: AggByteOff + Offset.getFixedValue(),
1244 Name: Name + "." + Twine(I))
1245 .first;
1246 AggIdxs.pop_back();
1247 }
1248 return std::make_pair(x&: Changed, /*ModifiedInPlace=*/y: false);
1249 }
1250 if (auto *AT = dyn_cast<ArrayType>(Val: PartType)) {
1251 Type *ElemTy = AT->getElementType();
1252 if (!ElemTy->isSingleValueType() || !DL.typeSizeEqualsStoreSize(Ty: ElemTy) ||
1253 ElemTy->isVectorTy()) {
1254 TypeSize ElemAllocSize = DL.getTypeAllocSize(Ty: ElemTy);
1255 bool Changed = false;
1256 for (auto I : llvm::iota_range<uint32_t>(0, AT->getNumElements(),
1257 /*Inclusive=*/false)) {
1258 AggIdxs.push_back(Elt: I);
1259 Changed |=
1260 visitStoreImpl(OrigSI, PartType: ElemTy, AggIdxs,
1261 AggByteOff: AggByteOff + I * ElemAllocSize.getFixedValue(),
1262 Name: Name + Twine(I))
1263 .first;
1264 AggIdxs.pop_back();
1265 }
1266 return std::make_pair(x&: Changed, /*ModifiedInPlace=*/y: false);
1267 }
1268 }
1269
1270 Value *OrigData = OrigSI.getValueOperand();
1271 Value *NewData = OrigData;
1272
1273 bool IsAggPart = !AggIdxs.empty();
1274 if (IsAggPart)
1275 NewData = IRB.CreateExtractValue(Agg: NewData, Idxs: AggIdxs, Name);
1276
1277 Type *ArrayAsVecType = scalarArrayTypeAsVector(T: PartType);
1278 if (ArrayAsVecType != PartType) {
1279 NewData = arrayToVector(V: NewData, TargetType: ArrayAsVecType, Name);
1280 }
1281
1282 Align PartAlign = commonAlignment(A: OrigSI.getAlign(), Offset: AggByteOff);
1283 OobProperties OobProps =
1284 analyzeOobProperties(Ptr: OrigSI.getPointerOperand(), Ty: PartType, ByteOffset: AggByteOff);
1285 uint64_t MaxWidth = ST->getMaxBufferIntrinsicWidth(
1286 DL, T: ArrayAsVecType, A: PartAlign, NoWrapFromMax: OobProps.NoWrapFromMax,
1287 NoPartialOOB: OobProps.NoPartialOOB);
1288 Type *LegalType = legalNonAggregateForMemOp(T: ArrayAsVecType, MaxWidth);
1289 if (LegalType != ArrayAsVecType) {
1290 NewData = makeLegalNonAggregate(V: NewData, TargetType: LegalType, Name);
1291 }
1292
1293 SmallVector<VecSlice> Slices;
1294 getVecSlices(T: LegalType, MaxWidth, Slices);
1295 bool NeedToSplit = Slices.size() > 1 || IsAggPart;
1296 if (!NeedToSplit) {
1297 Type *StorableType = intrinsicTypeFor(LegalType);
1298 if (StorableType == PartType)
1299 return std::make_pair(/*Changed=*/x: false, /*ModifiedInPlace=*/y: false);
1300 NewData = IRB.CreateBitCast(V: NewData, DestTy: StorableType, Name: Name + ".storable");
1301 OrigSI.setOperand(i_nocapture: 0, Val_nocapture: NewData);
1302 return std::make_pair(/*Changed=*/x: true, /*ModifiedInPlace=*/y: true);
1303 }
1304
1305 Value *OrigPtr = OrigSI.getPointerOperand();
1306 Type *ElemType = LegalType->getScalarType();
1307 if (IsAggPart && Slices.empty())
1308 Slices.push_back(Elt: VecSlice{/*Index=*/0, /*Length=*/1});
1309 unsigned ElemBytes = DL.getTypeStoreSize(Ty: ElemType);
1310 AAMDNodes AANodes = OrigSI.getAAMetadata();
1311 for (VecSlice S : Slices) {
1312 Type *SliceType =
1313 S.Length != 1 ? FixedVectorType::get(ElementType: ElemType, NumElts: S.Length) : ElemType;
1314 int64_t ByteOffset = AggByteOff + S.Index * ElemBytes;
1315 Value *NewPtr = IRB.CreateGEP(
1316 Ty: IRB.getInt8Ty(), Ptr: OrigPtr, IdxList: IRB.getInt32(C: ByteOffset),
1317 Name: OrigPtr->getName() + ".part." + Twine(S.Index),
1318 NW: ST->hasRelaxedBufferOOBMode() ? GEPNoWrapFlags::noUnsignedWrap()
1319 : GEPNoWrapFlags::none());
1320 Value *DataSlice = extractSlice(Vec: NewData, S, Name);
1321 Type *StorableType = intrinsicTypeFor(LegalType: SliceType);
1322 DataSlice = IRB.CreateBitCast(V: DataSlice, DestTy: StorableType,
1323 Name: DataSlice->getName() + ".storable");
1324 auto *NewSI = cast<StoreInst>(Val: OrigSI.clone());
1325 NewSI->setAlignment(commonAlignment(A: OrigSI.getAlign(), Offset: ByteOffset));
1326 IRB.Insert(I: NewSI);
1327 NewSI->setOperand(i_nocapture: 0, Val_nocapture: DataSlice);
1328 NewSI->setOperand(i_nocapture: 1, Val_nocapture: NewPtr);
1329 NewSI->setAAMetadata(AANodes.adjustForAccess(Offset: ByteOffset, AccessTy: StorableType, DL));
1330 }
1331 return std::make_pair(/*Changed=*/x: true, /*ModifiedInPlace=*/y: false);
1332}
1333
1334bool LegalizeBufferContentTypesVisitor::visitStoreInst(StoreInst &SI) {
1335 if (SI.getPointerAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
1336 return false;
1337 IRB.SetInsertPoint(&SI);
1338 SmallVector<uint32_t> AggIdxs;
1339 Value *OrigData = SI.getValueOperand();
1340 auto [Changed, ModifiedInPlace] =
1341 visitStoreImpl(OrigSI&: SI, PartType: OrigData->getType(), AggIdxs, AggByteOff: 0, Name: OrigData->getName());
1342 if (Changed && !ModifiedInPlace)
1343 SI.eraseFromParent();
1344 return Changed;
1345}
1346
1347bool LegalizeBufferContentTypesVisitor::visitAddrSpaceCastInst(
1348 AddrSpaceCastInst &AI) {
1349 if (AI.getSrcAddressSpace() != AMDGPUAS::BUFFER_RESOURCE ||
1350 AI.getDestAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
1351 return false;
1352 Value *Src = AI.getPointerOperand();
1353 auto Record = ZeroBasePointerToNumRecords.find(Val: Src);
1354 if (Record != ZeroBasePointerToNumRecords.end())
1355 ZeroBasePointerToNumRecords.insert(KV: {&AI, Record->second});
1356 else
1357 ZeroBasePointerToNumRecords.insert(KV: {&AI, nullptr});
1358 return false;
1359}
1360
1361bool LegalizeBufferContentTypesVisitor::visitIntrinsicInst(IntrinsicInst &II) {
1362 if (II.getIntrinsicID() != Intrinsic::amdgcn_make_buffer_rsrc)
1363 return false;
1364 ZeroBasePointerToNumRecords.insert(KV: {&II, II.getOperand(i_nocapture: 2)});
1365 return false;
1366}
1367
1368bool LegalizeBufferContentTypesVisitor::processFunction(Function &F,
1369 ScalarEvolution *SE) {
1370 this->SE = SE;
1371 ST = &TM->getSubtarget<GCNSubtarget>(F);
1372 bool Changed = false;
1373 for (Instruction &I : make_early_inc_range(Range: instructions(F))) {
1374 Changed |= visit(I);
1375 }
1376 ZeroBasePointerToNumRecords.clear();
1377 this->SE = nullptr;
1378 return Changed;
1379}
1380
1381/// Return the ptr addrspace(8) and i32 (resource and offset parts) in a lowered
1382/// buffer fat pointer constant.
1383static std::pair<Constant *, Constant *>
1384splitLoweredFatBufferConst(Constant *C) {
1385 assert(isSplitFatPtr(C->getType()) && "Not a split fat buffer pointer");
1386 return std::make_pair(x: C->getAggregateElement(Elt: 0u), y: C->getAggregateElement(Elt: 1u));
1387}
1388
1389namespace {
1390/// Handle the remapping of ptr addrspace(7) constants.
1391class FatPtrConstMaterializer final : public ValueMaterializer {
1392 BufferFatPtrToStructTypeMap *TypeMap;
1393 // An internal mapper that is used to recurse into the arguments of constants.
1394 // While the documentation for `ValueMapper` specifies not to use it
1395 // recursively, examination of the logic in mapValue() shows that it can
1396 // safely be used recursively when handling constants, like it does in its own
1397 // logic.
1398 ValueMapper InternalMapper;
1399
1400 Constant *materializeBufferFatPtrConst(Constant *C);
1401
1402public:
1403 // UnderlyingMap is the value map this materializer will be filling.
1404 FatPtrConstMaterializer(BufferFatPtrToStructTypeMap *TypeMap,
1405 ValueToValueMapTy &UnderlyingMap)
1406 : TypeMap(TypeMap),
1407 InternalMapper(UnderlyingMap, RF_None, TypeMap, this) {}
1408 ~FatPtrConstMaterializer() = default;
1409
1410 Value *materialize(Value *V) override;
1411};
1412} // namespace
1413
1414Constant *FatPtrConstMaterializer::materializeBufferFatPtrConst(Constant *C) {
1415 Type *SrcTy = C->getType();
1416 auto *NewTy = dyn_cast<StructType>(Val: TypeMap->remapType(SrcTy));
1417 if (C->isNullValue())
1418 return ConstantAggregateZero::getNullValue(Ty: NewTy);
1419 if (isa<PoisonValue>(Val: C)) {
1420 return ConstantStruct::get(T: NewTy,
1421 V: {PoisonValue::get(T: NewTy->getElementType(N: 0)),
1422 PoisonValue::get(T: NewTy->getElementType(N: 1))});
1423 }
1424 if (isa<UndefValue>(Val: C)) {
1425 return ConstantStruct::get(T: NewTy,
1426 V: {UndefValue::get(T: NewTy->getElementType(N: 0)),
1427 UndefValue::get(T: NewTy->getElementType(N: 1))});
1428 }
1429
1430 if (auto *VC = dyn_cast<ConstantVector>(Val: C)) {
1431 if (Constant *S = VC->getSplatValue()) {
1432 Constant *NewS = InternalMapper.mapConstant(C: *S);
1433 if (!NewS)
1434 return nullptr;
1435 auto [Rsrc, Off] = splitLoweredFatBufferConst(C: NewS);
1436 auto EC = VC->getType()->getElementCount();
1437 return ConstantStruct::get(T: NewTy, V: {ConstantVector::getSplat(EC, Elt: Rsrc),
1438 ConstantVector::getSplat(EC, Elt: Off)});
1439 }
1440 SmallVector<Constant *> Rsrcs;
1441 SmallVector<Constant *> Offs;
1442 for (Value *Op : VC->operand_values()) {
1443 auto *NewOp = dyn_cast_or_null<Constant>(Val: InternalMapper.mapValue(V: *Op));
1444 if (!NewOp)
1445 return nullptr;
1446 auto [Rsrc, Off] = splitLoweredFatBufferConst(C: NewOp);
1447 Rsrcs.push_back(Elt: Rsrc);
1448 Offs.push_back(Elt: Off);
1449 }
1450 Constant *RsrcVec = ConstantVector::get(V: Rsrcs);
1451 Constant *OffVec = ConstantVector::get(V: Offs);
1452 return ConstantStruct::get(T: NewTy, V: {RsrcVec, OffVec});
1453 }
1454
1455 if (isa<GlobalValue>(Val: C))
1456 reportFatalUsageError(reason: "global values containing ptr addrspace(7) (buffer "
1457 "fat pointer) values are not supported");
1458
1459 if (isa<ConstantExpr>(Val: C))
1460 reportFatalUsageError(
1461 reason: "constant exprs containing ptr addrspace(7) (buffer "
1462 "fat pointer) values should have been expanded earlier");
1463
1464 return nullptr;
1465}
1466
1467Value *FatPtrConstMaterializer::materialize(Value *V) {
1468 Constant *C = dyn_cast<Constant>(Val: V);
1469 if (!C)
1470 return nullptr;
1471 // Structs and other types that happen to contain fat pointers get remapped
1472 // by the mapValue() logic.
1473 if (!isBufferFatPtrConst(C))
1474 return nullptr;
1475 return materializeBufferFatPtrConst(C);
1476}
1477
1478using PtrParts = std::pair<Value *, Value *>;
1479namespace {
1480// The visitor returns the resource and offset parts for an instruction if they
1481// can be computed, or (nullptr, nullptr) for cases that don't have a meaningful
1482// value mapping.
1483class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
1484 ValueToValueMapTy RsrcParts;
1485 ValueToValueMapTy OffParts;
1486
1487 // Track instructions that have been rewritten into a user of the component
1488 // parts of their ptr addrspace(7) input. Instructions that produced
1489 // ptr addrspace(7) parts should **not** be RAUW'd before being added to this
1490 // set, as that replacement will be handled in a post-visit step. However,
1491 // instructions that yield values that aren't fat pointers (ex. ptrtoint)
1492 // should RAUW themselves with new instructions that use the split parts
1493 // of their arguments during processing.
1494 DenseSet<Instruction *> SplitUsers;
1495
1496 // Nodes that need a second look once we've computed the parts for all other
1497 // instructions to see if, for example, we really need to phi on the resource
1498 // part.
1499 SmallVector<Instruction *> Conditionals;
1500 // Temporary instructions produced while lowering conditionals that should be
1501 // killed.
1502 SmallVector<Instruction *> ConditionalTemps;
1503
1504 // Subtarget info, needed for determining what cache control bits to set.
1505 const TargetMachine *TM;
1506 const GCNSubtarget *ST = nullptr;
1507
1508 IRBuilder<InstSimplifyFolder> IRB;
1509
1510 // Copy metadata between instructions if applicable.
1511 void copyMetadata(Value *Dest, Value *Src);
1512
1513 // Get the resource and offset parts of the value V, inserting appropriate
1514 // extractvalue calls if needed.
1515 PtrParts getPtrParts(Value *V);
1516
1517 // Given an instruction that could produce multiple resource parts (a PHI or
1518 // select), collect the set of possible instructions that could have provided
1519 // its resource parts that it could have (the `Roots`) and the set of
1520 // conditional instructions visited during the search (`Seen`). If, after
1521 // removing the root of the search from `Seen` and `Roots`, `Seen` is a subset
1522 // of `Roots` and `Roots - Seen` contains one element, the resource part of
1523 // that element can replace the resource part of all other elements in `Seen`.
1524 void getPossibleRsrcRoots(Instruction *I, SmallPtrSetImpl<Value *> &Roots,
1525 SmallPtrSetImpl<Value *> &Seen);
1526 void processConditionals();
1527
1528 // If an instruction hav been split into resource and offset parts,
1529 // delete that instruction. If any of its uses have not themselves been split
1530 // into parts (for example, an insertvalue), construct the structure
1531 // that the type rewrites declared should be produced by the dying instruction
1532 // and use that.
1533 // Also, kill the temporary extractvalue operations produced by the two-stage
1534 // lowering of PHIs and conditionals.
1535 void killAndReplaceSplitInstructions(SmallVectorImpl<Instruction *> &Origs);
1536
1537 void setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx);
1538 void insertPreMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
1539 void insertPostMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
1540 Value *handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, Type *Ty,
1541 Align Alignment, AtomicOrdering Order,
1542 bool IsVolatile, SyncScope::ID SSID);
1543
1544public:
1545 SplitPtrStructs(Module &M, const TargetMachine *TM)
1546 : TM(TM), IRB(M, InstSimplifyFolder(M.getDataLayout())) {}
1547
1548 void processFunction(Function &F);
1549
1550 PtrParts visitInstruction(Instruction &I);
1551 PtrParts visitLoadInst(LoadInst &LI);
1552 PtrParts visitStoreInst(StoreInst &SI);
1553 PtrParts visitAtomicRMWInst(AtomicRMWInst &AI);
1554 PtrParts visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI);
1555 PtrParts visitGetElementPtrInst(GetElementPtrInst &GEP);
1556
1557 PtrParts visitPtrToAddrInst(PtrToAddrInst &PA);
1558 PtrParts visitPtrToIntInst(PtrToIntInst &PI);
1559 PtrParts visitIntToPtrInst(IntToPtrInst &IP);
1560 PtrParts visitAddrSpaceCastInst(AddrSpaceCastInst &I);
1561 PtrParts visitICmpInst(ICmpInst &Cmp);
1562 PtrParts visitFreezeInst(FreezeInst &I);
1563
1564 PtrParts visitExtractElementInst(ExtractElementInst &I);
1565 PtrParts visitInsertElementInst(InsertElementInst &I);
1566 PtrParts visitShuffleVectorInst(ShuffleVectorInst &I);
1567
1568 PtrParts visitPHINode(PHINode &PHI);
1569 PtrParts visitSelectInst(SelectInst &SI);
1570
1571 PtrParts visitIntrinsicInst(IntrinsicInst &II);
1572};
1573} // namespace
1574
1575void SplitPtrStructs::copyMetadata(Value *Dest, Value *Src) {
1576 auto *DestI = dyn_cast<Instruction>(Val: Dest);
1577 auto *SrcI = dyn_cast<Instruction>(Val: Src);
1578
1579 if (!DestI || !SrcI)
1580 return;
1581
1582 DestI->copyMetadata(SrcInst: *SrcI);
1583}
1584
1585PtrParts SplitPtrStructs::getPtrParts(Value *V) {
1586 assert(isSplitFatPtr(V->getType()) && "it's not meaningful to get the parts "
1587 "of something that wasn't rewritten");
1588 auto *RsrcEntry = &RsrcParts[V];
1589 auto *OffEntry = &OffParts[V];
1590 if (*RsrcEntry && *OffEntry)
1591 return {*RsrcEntry, *OffEntry};
1592
1593 if (auto *C = dyn_cast<Constant>(Val: V)) {
1594 auto [Rsrc, Off] = splitLoweredFatBufferConst(C);
1595 return {*RsrcEntry = Rsrc, *OffEntry = Off};
1596 }
1597
1598 IRBuilder<InstSimplifyFolder>::InsertPointGuard Guard(IRB);
1599 if (auto *I = dyn_cast<Instruction>(Val: V)) {
1600 LLVM_DEBUG(dbgs() << "Recursing to split parts of " << *I << "\n");
1601 auto [Rsrc, Off] = visit(I&: *I);
1602 if (Rsrc && Off)
1603 return {*RsrcEntry = Rsrc, *OffEntry = Off};
1604 // We'll be creating the new values after the relevant instruction.
1605 // This instruction generates a value and so isn't a terminator.
1606 IRB.SetInsertPoint(*I->getInsertionPointAfterDef());
1607 IRB.SetCurrentDebugLocation(I->getDebugLoc());
1608 } else if (auto *A = dyn_cast<Argument>(Val: V)) {
1609 IRB.SetInsertPointPastAllocas(A->getParent());
1610 IRB.SetCurrentDebugLocation(DebugLoc());
1611 }
1612 Value *Rsrc = IRB.CreateExtractValue(Agg: V, Idxs: 0, Name: V->getName() + ".rsrc");
1613 Value *Off = IRB.CreateExtractValue(Agg: V, Idxs: 1, Name: V->getName() + ".off");
1614 return {*RsrcEntry = Rsrc, *OffEntry = Off};
1615}
1616
1617/// Returns the instruction that defines the resource part of the value V.
1618/// Note that this is not getUnderlyingObject(), since that looks through
1619/// operations like ptrmask which might modify the resource part.
1620///
1621/// We can limit ourselves to just looking through GEPs followed by looking
1622/// through addrspacecasts because only those two operations preserve the
1623/// resource part, and because operations on an `addrspace(8)` (which is the
1624/// legal input to this addrspacecast) would produce a different resource part.
1625static Value *rsrcPartRoot(Value *V) {
1626 while (auto *GEP = dyn_cast<GEPOperator>(Val: V))
1627 V = GEP->getPointerOperand();
1628 while (auto *ASC = dyn_cast<AddrSpaceCastOperator>(Val: V))
1629 V = ASC->getPointerOperand();
1630 return V;
1631}
1632
1633void SplitPtrStructs::getPossibleRsrcRoots(Instruction *I,
1634 SmallPtrSetImpl<Value *> &Roots,
1635 SmallPtrSetImpl<Value *> &Seen) {
1636 if (auto *PHI = dyn_cast<PHINode>(Val: I)) {
1637 if (!Seen.insert(Ptr: I).second)
1638 return;
1639 for (Value *In : PHI->incoming_values()) {
1640 In = rsrcPartRoot(V: In);
1641 Roots.insert(Ptr: In);
1642 if (isa<PHINode, SelectInst>(Val: In))
1643 getPossibleRsrcRoots(I: cast<Instruction>(Val: In), Roots, Seen);
1644 }
1645 } else if (auto *SI = dyn_cast<SelectInst>(Val: I)) {
1646 if (!Seen.insert(Ptr: SI).second)
1647 return;
1648 Value *TrueVal = rsrcPartRoot(V: SI->getTrueValue());
1649 Value *FalseVal = rsrcPartRoot(V: SI->getFalseValue());
1650 Roots.insert(Ptr: TrueVal);
1651 Roots.insert(Ptr: FalseVal);
1652 if (isa<PHINode, SelectInst>(Val: TrueVal))
1653 getPossibleRsrcRoots(I: cast<Instruction>(Val: TrueVal), Roots, Seen);
1654 if (isa<PHINode, SelectInst>(Val: FalseVal))
1655 getPossibleRsrcRoots(I: cast<Instruction>(Val: FalseVal), Roots, Seen);
1656 } else {
1657 llvm_unreachable("getPossibleRsrcParts() only works on phi and select");
1658 }
1659}
1660
1661void SplitPtrStructs::processConditionals() {
1662 SmallDenseMap<Value *, Value *> FoundRsrcs;
1663 SmallPtrSet<Value *, 4> Roots;
1664 SmallPtrSet<Value *, 4> Seen;
1665 for (Instruction *I : Conditionals) {
1666 // These have to exist by now because we've visited these nodes.
1667 Value *Rsrc = RsrcParts[I];
1668 Value *Off = OffParts[I];
1669 assert(Rsrc && Off && "must have visited conditionals by now");
1670
1671 std::optional<Value *> MaybeRsrc;
1672 auto MaybeFoundRsrc = FoundRsrcs.find(Val: I);
1673 if (MaybeFoundRsrc != FoundRsrcs.end()) {
1674 MaybeRsrc = MaybeFoundRsrc->second;
1675 } else {
1676 IRBuilder<InstSimplifyFolder>::InsertPointGuard Guard(IRB);
1677 Roots.clear();
1678 Seen.clear();
1679 getPossibleRsrcRoots(I, Roots, Seen);
1680 LLVM_DEBUG(dbgs() << "Processing conditional: " << *I << "\n");
1681#ifndef NDEBUG
1682 for (Value *V : Roots)
1683 LLVM_DEBUG(dbgs() << "Root: " << *V << "\n");
1684 for (Value *V : Seen)
1685 LLVM_DEBUG(dbgs() << "Seen: " << *V << "\n");
1686#endif
1687 // If we are our own possible root, then we shouldn't block our
1688 // replacement with a valid incoming value.
1689 Roots.erase(Ptr: I);
1690 // We don't want to block the optimization for conditionals that don't
1691 // refer to themselves but did see themselves during the traversal.
1692 Seen.erase(Ptr: I);
1693
1694 if (set_is_subset(S1: Seen, S2: Roots)) {
1695 auto Diff = set_difference(S1: Roots, S2: Seen);
1696 if (Diff.size() == 1) {
1697 Value *RootVal = *Diff.begin();
1698 // Handle the case where previous loops already looked through
1699 // an addrspacecast.
1700 if (isSplitFatPtr(Ty: RootVal->getType()))
1701 MaybeRsrc = getPtrParts(V: RootVal).first;
1702 else
1703 MaybeRsrc = RootVal;
1704 }
1705 }
1706 }
1707
1708 if (auto *PHI = dyn_cast<PHINode>(Val: I)) {
1709 Value *NewRsrc;
1710 StructType *PHITy = cast<StructType>(Val: PHI->getType());
1711 IRB.SetInsertPoint(*PHI->getInsertionPointAfterDef());
1712 IRB.SetCurrentDebugLocation(PHI->getDebugLoc());
1713 if (MaybeRsrc) {
1714 NewRsrc = *MaybeRsrc;
1715 } else {
1716 Type *RsrcTy = PHITy->getElementType(N: 0);
1717 auto *RsrcPHI = IRB.CreatePHI(Ty: RsrcTy, NumReservedValues: PHI->getNumIncomingValues());
1718 RsrcPHI->takeName(V: Rsrc);
1719 for (auto [V, BB] : llvm::zip(t: PHI->incoming_values(), u: PHI->blocks())) {
1720 Value *VRsrc = getPtrParts(V).first;
1721 RsrcPHI->addIncoming(V: VRsrc, BB);
1722 }
1723 copyMetadata(Dest: RsrcPHI, Src: PHI);
1724 NewRsrc = RsrcPHI;
1725 }
1726
1727 Type *OffTy = PHITy->getElementType(N: 1);
1728 auto *NewOff = IRB.CreatePHI(Ty: OffTy, NumReservedValues: PHI->getNumIncomingValues());
1729 NewOff->takeName(V: Off);
1730 for (auto [V, BB] : llvm::zip(t: PHI->incoming_values(), u: PHI->blocks())) {
1731 assert(OffParts.count(V) && "An offset part had to be created by now");
1732 Value *VOff = getPtrParts(V).second;
1733 NewOff->addIncoming(V: VOff, BB);
1734 }
1735 copyMetadata(Dest: NewOff, Src: PHI);
1736
1737 // Note: We don't eraseFromParent() the temporaries because we don't want
1738 // to put the corrections maps in an inconstent state. That'll be handed
1739 // during the rest of the killing. Also, `ValueToValueMapTy` guarantees
1740 // that references in that map will be updated as well.
1741 // Note that if the temporary instruction got `InstSimplify`'d away, it
1742 // might be something like a block argument.
1743 if (auto *RsrcInst = dyn_cast<Instruction>(Val: Rsrc)) {
1744 ConditionalTemps.push_back(Elt: RsrcInst);
1745 RsrcInst->replaceAllUsesWith(V: NewRsrc);
1746 }
1747 if (auto *OffInst = dyn_cast<Instruction>(Val: Off)) {
1748 ConditionalTemps.push_back(Elt: OffInst);
1749 OffInst->replaceAllUsesWith(V: NewOff);
1750 }
1751
1752 // Save on recomputing the cycle traversals in known-root cases.
1753 if (MaybeRsrc)
1754 for (Value *V : Seen)
1755 FoundRsrcs[V] = NewRsrc;
1756 } else if (isa<SelectInst>(Val: I)) {
1757 if (MaybeRsrc) {
1758 if (auto *RsrcInst = dyn_cast<Instruction>(Val: Rsrc)) {
1759 // Guard against conditionals that were already folded away.
1760 if (RsrcInst != *MaybeRsrc) {
1761 ConditionalTemps.push_back(Elt: RsrcInst);
1762 RsrcInst->replaceAllUsesWith(V: *MaybeRsrc);
1763 }
1764 }
1765 for (Value *V : Seen)
1766 FoundRsrcs[V] = *MaybeRsrc;
1767 }
1768 } else {
1769 llvm_unreachable("Only PHIs and selects go in the conditionals list");
1770 }
1771 }
1772}
1773
1774void SplitPtrStructs::killAndReplaceSplitInstructions(
1775 SmallVectorImpl<Instruction *> &Origs) {
1776 for (Instruction *I : ConditionalTemps)
1777 I->eraseFromParent();
1778
1779 for (Instruction *I : Origs) {
1780 if (!SplitUsers.contains(V: I))
1781 continue;
1782
1783 SmallVector<DbgVariableRecord *> Dbgs;
1784 findDbgValues(V: I, DbgVariableRecords&: Dbgs);
1785 for (DbgVariableRecord *Dbg : Dbgs) {
1786 auto &DL = I->getDataLayout();
1787 assert(isSplitFatPtr(I->getType()) &&
1788 "We should've RAUW'd away loads, stores, etc. at this point");
1789 DbgVariableRecord *OffDbg = Dbg->clone();
1790 auto [Rsrc, Off] = getPtrParts(V: I);
1791
1792 int64_t RsrcSz = DL.getTypeSizeInBits(Ty: Rsrc->getType());
1793 int64_t OffSz = DL.getTypeSizeInBits(Ty: Off->getType());
1794
1795 std::optional<DIExpression *> RsrcExpr =
1796 DIExpression::createFragmentExpression(Expr: Dbg->getExpression(), OffsetInBits: 0,
1797 SizeInBits: RsrcSz);
1798 std::optional<DIExpression *> OffExpr =
1799 DIExpression::createFragmentExpression(Expr: Dbg->getExpression(), OffsetInBits: RsrcSz,
1800 SizeInBits: OffSz);
1801 if (OffExpr) {
1802 OffDbg->setExpression(*OffExpr);
1803 OffDbg->replaceVariableLocationOp(OldValue: I, NewValue: Off);
1804 OffDbg->insertBefore(InsertBefore: Dbg);
1805 } else {
1806 OffDbg->eraseFromParent();
1807 }
1808 if (RsrcExpr) {
1809 Dbg->setExpression(*RsrcExpr);
1810 Dbg->replaceVariableLocationOp(OldValue: I, NewValue: Rsrc);
1811 } else {
1812 Dbg->replaceVariableLocationOp(OldValue: I, NewValue: PoisonValue::get(T: I->getType()));
1813 }
1814 }
1815
1816 Value *Poison = PoisonValue::get(T: I->getType());
1817 I->replaceUsesWithIf(New: Poison, ShouldReplace: [&](const Use &U) -> bool {
1818 if (const auto *UI = dyn_cast<Instruction>(Val: U.getUser()))
1819 return SplitUsers.contains(V: UI);
1820 return false;
1821 });
1822
1823 if (I->use_empty()) {
1824 I->eraseFromParent();
1825 continue;
1826 }
1827 IRB.SetInsertPoint(*I->getInsertionPointAfterDef());
1828 IRB.SetCurrentDebugLocation(I->getDebugLoc());
1829 auto [Rsrc, Off] = getPtrParts(V: I);
1830 Value *Struct = PoisonValue::get(T: I->getType());
1831 Struct = IRB.CreateInsertValue(Agg: Struct, Val: Rsrc, Idxs: 0);
1832 Struct = IRB.CreateInsertValue(Agg: Struct, Val: Off, Idxs: 1);
1833 copyMetadata(Dest: Struct, Src: I);
1834 Struct->takeName(V: I);
1835 I->replaceAllUsesWith(V: Struct);
1836 I->eraseFromParent();
1837 }
1838}
1839
1840void SplitPtrStructs::setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx) {
1841 LLVMContext &Ctx = Intr->getContext();
1842 Intr->addParamAttr(ArgNo: RsrcArgIdx, Attr: Attribute::getWithAlignment(Context&: Ctx, Alignment: A));
1843}
1844
1845void SplitPtrStructs::insertPreMemOpFence(AtomicOrdering Order,
1846 SyncScope::ID SSID) {
1847 switch (Order) {
1848 case AtomicOrdering::Release:
1849 case AtomicOrdering::AcquireRelease:
1850 case AtomicOrdering::SequentiallyConsistent:
1851 IRB.CreateFence(Ordering: AtomicOrdering::Release, SSID);
1852 break;
1853 default:
1854 break;
1855 }
1856}
1857
1858void SplitPtrStructs::insertPostMemOpFence(AtomicOrdering Order,
1859 SyncScope::ID SSID) {
1860 switch (Order) {
1861 case AtomicOrdering::Acquire:
1862 case AtomicOrdering::AcquireRelease:
1863 case AtomicOrdering::SequentiallyConsistent:
1864 IRB.CreateFence(Ordering: AtomicOrdering::Acquire, SSID);
1865 break;
1866 default:
1867 break;
1868 }
1869}
1870
1871Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
1872 Type *Ty, Align Alignment,
1873 AtomicOrdering Order, bool IsVolatile,
1874 SyncScope::ID SSID) {
1875 IRB.SetInsertPoint(I);
1876
1877 auto [Rsrc, Off] = getPtrParts(V: Ptr);
1878 SmallVector<Value *, 5> Args;
1879 if (Arg)
1880 Args.push_back(Elt: Arg);
1881 Args.push_back(Elt: Rsrc);
1882 Args.push_back(Elt: Off);
1883 insertPreMemOpFence(Order, SSID);
1884 // soffset is always 0 for these cases, where we always want any offset to be
1885 // part of bounds checking and we don't know which parts of the GEPs is
1886 // uniform.
1887 Args.push_back(Elt: IRB.getInt32(C: 0));
1888
1889 uint32_t Aux = 0;
1890 if (IsVolatile)
1891 Aux |= AMDGPU::CPol::VOLATILE;
1892 Args.push_back(Elt: IRB.getInt32(C: Aux));
1893
1894 Intrinsic::ID IID = Intrinsic::not_intrinsic;
1895 if (isa<LoadInst>(Val: I))
1896 IID = Order == AtomicOrdering::NotAtomic
1897 ? Intrinsic::amdgcn_raw_ptr_buffer_load
1898 : Intrinsic::amdgcn_raw_ptr_atomic_buffer_load;
1899 else if (isa<StoreInst>(Val: I))
1900 IID = Intrinsic::amdgcn_raw_ptr_buffer_store;
1901 else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
1902 switch (RMW->getOperation()) {
1903 case AtomicRMWInst::Xchg:
1904 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap;
1905 break;
1906 case AtomicRMWInst::Add:
1907 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_add;
1908 break;
1909 case AtomicRMWInst::Sub:
1910 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub;
1911 break;
1912 case AtomicRMWInst::And:
1913 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_and;
1914 break;
1915 case AtomicRMWInst::Or:
1916 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_or;
1917 break;
1918 case AtomicRMWInst::Xor:
1919 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor;
1920 break;
1921 case AtomicRMWInst::Max:
1922 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax;
1923 break;
1924 case AtomicRMWInst::Min:
1925 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin;
1926 break;
1927 case AtomicRMWInst::UMax:
1928 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax;
1929 break;
1930 case AtomicRMWInst::UMin:
1931 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin;
1932 break;
1933 case AtomicRMWInst::FAdd:
1934 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd;
1935 break;
1936 case AtomicRMWInst::FMax:
1937 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax;
1938 break;
1939 case AtomicRMWInst::FMin:
1940 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin;
1941 break;
1942 case AtomicRMWInst::USubCond:
1943 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32;
1944 break;
1945 case AtomicRMWInst::USubSat:
1946 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32;
1947 break;
1948 case AtomicRMWInst::FSub: {
1949 reportFatalUsageError(
1950 reason: "atomic floating point subtraction not supported for "
1951 "buffer resources and should've been expanded away");
1952 break;
1953 }
1954 case AtomicRMWInst::FMaximum: {
1955 reportFatalUsageError(
1956 reason: "atomic floating point fmaximum not supported for "
1957 "buffer resources and should've been expanded away");
1958 break;
1959 }
1960 case AtomicRMWInst::FMinimum: {
1961 reportFatalUsageError(
1962 reason: "atomic floating point fminimum not supported for "
1963 "buffer resources and should've been expanded away");
1964 break;
1965 }
1966 case AtomicRMWInst::FMaximumNum: {
1967 reportFatalUsageError(
1968 reason: "atomic floating point fmaximumnum not supported for "
1969 "buffer resources and should've been expanded away");
1970 break;
1971 }
1972 case AtomicRMWInst::FMinimumNum: {
1973 reportFatalUsageError(
1974 reason: "atomic floating point fminimumnum not supported for "
1975 "buffer resources and should've been expanded away");
1976 break;
1977 }
1978 case AtomicRMWInst::Nand:
1979 reportFatalUsageError(
1980 reason: "atomic nand not supported for buffer resources and "
1981 "should've been expanded away");
1982 break;
1983 case AtomicRMWInst::UIncWrap:
1984 case AtomicRMWInst::UDecWrap:
1985 reportFatalUsageError(
1986 reason: "wrapping increment/decrement not supported for "
1987 "buffer resources and should've been expanded away");
1988 break;
1989 case AtomicRMWInst::BAD_BINOP:
1990 llvm_unreachable("Not sure how we got a bad binop");
1991 }
1992 }
1993
1994 CallInst *Call = IRB.CreateIntrinsicWithoutFolding(ID: IID, OverloadTypes: Ty, Args);
1995 copyMetadata(Dest: Call, Src: I);
1996 setAlign(Intr: Call, A: Alignment, RsrcArgIdx: Arg ? 1 : 0);
1997 Call->takeName(V: I);
1998
1999 insertPostMemOpFence(Order, SSID);
2000 // The "no moving p7 directly" rewrites ensure that this load or store won't
2001 // itself need to be split into parts.
2002 SplitUsers.insert(V: I);
2003 I->replaceAllUsesWith(V: Call);
2004 return Call;
2005}
2006
2007PtrParts SplitPtrStructs::visitInstruction(Instruction &I) {
2008 return {nullptr, nullptr};
2009}
2010
2011PtrParts SplitPtrStructs::visitLoadInst(LoadInst &LI) {
2012 if (!isSplitFatPtr(Ty: LI.getPointerOperandType()))
2013 return {nullptr, nullptr};
2014 handleMemoryInst(I: &LI, Arg: nullptr, Ptr: LI.getPointerOperand(), Ty: LI.getType(),
2015 Alignment: LI.getAlign(), Order: LI.getOrdering(), IsVolatile: LI.isVolatile(),
2016 SSID: LI.getSyncScopeID());
2017 return {nullptr, nullptr};
2018}
2019
2020PtrParts SplitPtrStructs::visitStoreInst(StoreInst &SI) {
2021 if (!isSplitFatPtr(Ty: SI.getPointerOperandType()))
2022 return {nullptr, nullptr};
2023 Value *Arg = SI.getValueOperand();
2024 handleMemoryInst(I: &SI, Arg, Ptr: SI.getPointerOperand(), Ty: Arg->getType(),
2025 Alignment: SI.getAlign(), Order: SI.getOrdering(), IsVolatile: SI.isVolatile(),
2026 SSID: SI.getSyncScopeID());
2027 return {nullptr, nullptr};
2028}
2029
2030PtrParts SplitPtrStructs::visitAtomicRMWInst(AtomicRMWInst &AI) {
2031 if (!isSplitFatPtr(Ty: AI.getPointerOperand()->getType()))
2032 return {nullptr, nullptr};
2033 Value *Arg = AI.getValOperand();
2034 handleMemoryInst(I: &AI, Arg, Ptr: AI.getPointerOperand(), Ty: Arg->getType(),
2035 Alignment: AI.getAlign(), Order: AI.getOrdering(), IsVolatile: AI.isVolatile(),
2036 SSID: AI.getSyncScopeID());
2037 return {nullptr, nullptr};
2038}
2039
2040// Unlike load, store, and RMW, cmpxchg needs special handling to account
2041// for the boolean argument.
2042PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
2043 Value *Ptr = AI.getPointerOperand();
2044 if (!isSplitFatPtr(Ty: Ptr->getType()))
2045 return {nullptr, nullptr};
2046 IRB.SetInsertPoint(&AI);
2047
2048 Type *Ty = AI.getNewValOperand()->getType();
2049 AtomicOrdering Order = AI.getMergedOrdering();
2050 SyncScope::ID SSID = AI.getSyncScopeID();
2051 bool IsNonTemporal = AI.getMetadata(KindID: LLVMContext::MD_nontemporal);
2052
2053 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2054 insertPreMemOpFence(Order, SSID);
2055
2056 uint32_t Aux = 0;
2057 if (IsNonTemporal)
2058 Aux |= AMDGPU::CPol::SLC;
2059 if (AI.isVolatile())
2060 Aux |= AMDGPU::CPol::VOLATILE;
2061 CallInst *Call = IRB.CreateIntrinsicWithoutFolding(
2062 ID: Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap, OverloadTypes: Ty,
2063 Args: {AI.getNewValOperand(), AI.getCompareOperand(), Rsrc, Off,
2064 IRB.getInt32(C: 0), IRB.getInt32(C: Aux)});
2065 copyMetadata(Dest: Call, Src: &AI);
2066 setAlign(Intr: Call, A: AI.getAlign(), RsrcArgIdx: 2);
2067 Call->takeName(V: &AI);
2068 insertPostMemOpFence(Order, SSID);
2069
2070 Value *Res = PoisonValue::get(T: AI.getType());
2071 Res = IRB.CreateInsertValue(Agg: Res, Val: Call, Idxs: 0);
2072 Value *Succeeded = IRB.CreateICmpEQ(LHS: Call, RHS: AI.getCompareOperand());
2073 Res = IRB.CreateInsertValue(Agg: Res, Val: Succeeded, Idxs: 1);
2074 SplitUsers.insert(V: &AI);
2075 AI.replaceAllUsesWith(V: Res);
2076 return {nullptr, nullptr};
2077}
2078
2079PtrParts SplitPtrStructs::visitGetElementPtrInst(GetElementPtrInst &GEP) {
2080 using namespace llvm::PatternMatch;
2081 Value *Ptr = GEP.getPointerOperand();
2082 if (!isSplitFatPtr(Ty: Ptr->getType()))
2083 return {nullptr, nullptr};
2084 IRB.SetInsertPoint(&GEP);
2085
2086 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2087 const DataLayout &DL = GEP.getDataLayout();
2088 bool IsNUW = GEP.hasNoUnsignedWrap();
2089 bool IsNUSW = GEP.hasNoUnsignedSignedWrap();
2090
2091 StructType *ResTy = cast<StructType>(Val: GEP.getType());
2092 Type *ResRsrcTy = ResTy->getElementType(N: 0);
2093 VectorType *ResRsrcVecTy = dyn_cast<VectorType>(Val: ResRsrcTy);
2094 bool BroadcastsPtr = ResRsrcVecTy && !isa<VectorType>(Val: Off->getType());
2095
2096 // In order to call emitGEPOffset() and thus not have to reimplement it,
2097 // we need the GEP result to have ptr addrspace(7) type.
2098 Type *FatPtrTy =
2099 ResRsrcTy->getWithNewType(EltTy: IRB.getPtrTy(AddrSpace: AMDGPUAS::BUFFER_FAT_POINTER));
2100 GEP.mutateType(Ty: FatPtrTy);
2101 Value *OffAccum = emitGEPOffset(Builder: &IRB, DL, GEP: &GEP);
2102 GEP.mutateType(Ty: ResTy);
2103
2104 if (BroadcastsPtr) {
2105 Rsrc = IRB.CreateVectorSplat(EC: ResRsrcVecTy->getElementCount(), V: Rsrc,
2106 Name: Rsrc->getName());
2107 Off = IRB.CreateVectorSplat(EC: ResRsrcVecTy->getElementCount(), V: Off,
2108 Name: Off->getName());
2109 }
2110 if (match(V: OffAccum, P: m_Zero())) { // Constant-zero offset
2111 SplitUsers.insert(V: &GEP);
2112 return {Rsrc, Off};
2113 }
2114
2115 bool HasNonNegativeOff = false;
2116 if (auto *CI = dyn_cast<ConstantInt>(Val: OffAccum)) {
2117 HasNonNegativeOff = !CI->isNegative();
2118 }
2119 Value *NewOff;
2120 if (match(V: Off, P: m_Zero())) {
2121 NewOff = OffAccum;
2122 } else {
2123 NewOff = IRB.CreateAdd(LHS: Off, RHS: OffAccum, Name: "",
2124 /*hasNUW=*/HasNUW: IsNUW || (IsNUSW && HasNonNegativeOff),
2125 /*hasNSW=*/HasNSW: false);
2126 }
2127 copyMetadata(Dest: NewOff, Src: &GEP);
2128 NewOff->takeName(V: &GEP);
2129 SplitUsers.insert(V: &GEP);
2130 return {Rsrc, NewOff};
2131}
2132
2133PtrParts SplitPtrStructs::visitPtrToIntInst(PtrToIntInst &PI) {
2134 Value *Ptr = PI.getPointerOperand();
2135 if (!isSplitFatPtr(Ty: Ptr->getType()))
2136 return {nullptr, nullptr};
2137 IRB.SetInsertPoint(&PI);
2138
2139 Type *ResTy = PI.getType();
2140 unsigned Width = ResTy->getScalarSizeInBits();
2141
2142 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2143 const DataLayout &DL = PI.getDataLayout();
2144 unsigned FatPtrWidth = DL.getPointerSizeInBits(AS: AMDGPUAS::BUFFER_FAT_POINTER);
2145
2146 Value *Res;
2147 if (Width <= BufferOffsetWidth) {
2148 Res = IRB.CreateIntCast(V: Off, DestTy: ResTy, /*isSigned=*/false,
2149 Name: PI.getName() + ".off");
2150 } else {
2151 Value *RsrcInt = IRB.CreatePtrToInt(V: Rsrc, DestTy: ResTy, Name: PI.getName() + ".rsrc");
2152 Value *Shl = IRB.CreateShl(
2153 LHS: RsrcInt,
2154 RHS: ConstantExpr::getIntegerValue(Ty: ResTy, V: APInt(Width, BufferOffsetWidth)),
2155 Name: "", HasNUW: Width >= FatPtrWidth, HasNSW: Width > FatPtrWidth);
2156 Value *OffCast = IRB.CreateIntCast(V: Off, DestTy: ResTy, /*isSigned=*/false,
2157 Name: PI.getName() + ".off");
2158 Res = IRB.CreateOr(LHS: Shl, RHS: OffCast);
2159 }
2160
2161 copyMetadata(Dest: Res, Src: &PI);
2162 Res->takeName(V: &PI);
2163 SplitUsers.insert(V: &PI);
2164 PI.replaceAllUsesWith(V: Res);
2165 return {nullptr, nullptr};
2166}
2167
2168PtrParts SplitPtrStructs::visitPtrToAddrInst(PtrToAddrInst &PA) {
2169 Value *Ptr = PA.getPointerOperand();
2170 if (!isSplitFatPtr(Ty: Ptr->getType()))
2171 return {nullptr, nullptr};
2172 IRB.SetInsertPoint(&PA);
2173
2174 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2175 Value *Res = IRB.CreateIntCast(V: Off, DestTy: PA.getType(), /*isSigned=*/false);
2176 copyMetadata(Dest: Res, Src: &PA);
2177 Res->takeName(V: &PA);
2178 SplitUsers.insert(V: &PA);
2179 PA.replaceAllUsesWith(V: Res);
2180 return {nullptr, nullptr};
2181}
2182
2183PtrParts SplitPtrStructs::visitIntToPtrInst(IntToPtrInst &IP) {
2184 if (!isSplitFatPtr(Ty: IP.getType()))
2185 return {nullptr, nullptr};
2186 IRB.SetInsertPoint(&IP);
2187 const DataLayout &DL = IP.getDataLayout();
2188 unsigned RsrcPtrWidth = DL.getPointerSizeInBits(AS: AMDGPUAS::BUFFER_RESOURCE);
2189 Value *Int = IP.getOperand(i_nocapture: 0);
2190 Type *IntTy = Int->getType();
2191 Type *RsrcIntTy = IntTy->getWithNewBitWidth(NewBitWidth: RsrcPtrWidth);
2192 unsigned Width = IntTy->getScalarSizeInBits();
2193
2194 auto *RetTy = cast<StructType>(Val: IP.getType());
2195 Type *RsrcTy = RetTy->getElementType(N: 0);
2196 Type *OffTy = RetTy->getElementType(N: 1);
2197 // inttoptr zero-extends, so narrow inputs contribute nothing to the resource
2198 // part.
2199 Value *RsrcInt;
2200 if (Width <= BufferOffsetWidth) {
2201 RsrcInt = Constant::getNullValue(Ty: RsrcIntTy);
2202 } else {
2203 Value *RsrcPart =
2204 IRB.CreateLShr(LHS: Int, RHS: ConstantInt::get(Ty: IntTy, V: BufferOffsetWidth));
2205 RsrcInt = IRB.CreateIntCast(V: RsrcPart, DestTy: RsrcIntTy, /*isSigned=*/false);
2206 }
2207 Value *Rsrc = IRB.CreateIntToPtr(V: RsrcInt, DestTy: RsrcTy, Name: IP.getName() + ".rsrc");
2208 Value *Off =
2209 IRB.CreateIntCast(V: Int, DestTy: OffTy, /*IsSigned=*/isSigned: false, Name: IP.getName() + ".off");
2210
2211 copyMetadata(Dest: Rsrc, Src: &IP);
2212 SplitUsers.insert(V: &IP);
2213 return {Rsrc, Off};
2214}
2215
2216PtrParts SplitPtrStructs::visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
2217 // TODO(krzysz00): handle casts from ptr addrspace(7) to global pointers
2218 // by computing the effective address.
2219 if (!isSplitFatPtr(Ty: I.getType()))
2220 return {nullptr, nullptr};
2221 IRB.SetInsertPoint(&I);
2222 Value *In = I.getPointerOperand();
2223 // No-op casts preserve parts
2224 if (In->getType() == I.getType()) {
2225 auto [Rsrc, Off] = getPtrParts(V: In);
2226 SplitUsers.insert(V: &I);
2227 return {Rsrc, Off};
2228 }
2229
2230 auto *ResTy = cast<StructType>(Val: I.getType());
2231 Type *RsrcTy = ResTy->getElementType(N: 0);
2232 Type *OffTy = ResTy->getElementType(N: 1);
2233 Value *ZeroOff = Constant::getNullValue(Ty: OffTy);
2234
2235 // Special case for null pointers, undef, and poison, which can be created by
2236 // address space propagation.
2237 auto *InConst = dyn_cast<Constant>(Val: In);
2238 if (InConst && InConst->isNullValue()) {
2239 Value *NullRsrc = Constant::getNullValue(Ty: RsrcTy);
2240 SplitUsers.insert(V: &I);
2241 return {NullRsrc, ZeroOff};
2242 }
2243 if (isa<PoisonValue>(Val: In)) {
2244 Value *PoisonRsrc = PoisonValue::get(T: RsrcTy);
2245 Value *PoisonOff = PoisonValue::get(T: OffTy);
2246 SplitUsers.insert(V: &I);
2247 return {PoisonRsrc, PoisonOff};
2248 }
2249 if (isa<UndefValue>(Val: In)) {
2250 Value *UndefRsrc = UndefValue::get(T: RsrcTy);
2251 Value *UndefOff = UndefValue::get(T: OffTy);
2252 SplitUsers.insert(V: &I);
2253 return {UndefRsrc, UndefOff};
2254 }
2255
2256 if (I.getSrcAddressSpace() != AMDGPUAS::BUFFER_RESOURCE)
2257 reportFatalUsageError(
2258 reason: "only buffer resources (addrspace 8) and null/poison pointers can be "
2259 "cast to buffer fat pointers (addrspace 7)");
2260 SplitUsers.insert(V: &I);
2261 return {In, ZeroOff};
2262}
2263
2264PtrParts SplitPtrStructs::visitICmpInst(ICmpInst &Cmp) {
2265 Value *Lhs = Cmp.getOperand(i_nocapture: 0);
2266 if (!isSplitFatPtr(Ty: Lhs->getType()))
2267 return {nullptr, nullptr};
2268 Value *Rhs = Cmp.getOperand(i_nocapture: 1);
2269 IRB.SetInsertPoint(&Cmp);
2270 ICmpInst::Predicate Pred = Cmp.getPredicate();
2271
2272 assert((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2273 "Pointer comparison is only equal or unequal");
2274 auto [LhsRsrc, LhsOff] = getPtrParts(V: Lhs);
2275 auto [RhsRsrc, RhsOff] = getPtrParts(V: Rhs);
2276 Value *Res = IRB.CreateICmp(P: Pred, LHS: LhsOff, RHS: RhsOff);
2277 copyMetadata(Dest: Res, Src: &Cmp);
2278 Res->takeName(V: &Cmp);
2279 SplitUsers.insert(V: &Cmp);
2280 Cmp.replaceAllUsesWith(V: Res);
2281 return {nullptr, nullptr};
2282}
2283
2284PtrParts SplitPtrStructs::visitFreezeInst(FreezeInst &I) {
2285 if (!isSplitFatPtr(Ty: I.getType()))
2286 return {nullptr, nullptr};
2287 IRB.SetInsertPoint(&I);
2288 auto [Rsrc, Off] = getPtrParts(V: I.getOperand(i_nocapture: 0));
2289
2290 Value *RsrcRes = IRB.CreateFreeze(V: Rsrc, Name: I.getName() + ".rsrc");
2291 copyMetadata(Dest: RsrcRes, Src: &I);
2292 Value *OffRes = IRB.CreateFreeze(V: Off, Name: I.getName() + ".off");
2293 copyMetadata(Dest: OffRes, Src: &I);
2294 SplitUsers.insert(V: &I);
2295 return {RsrcRes, OffRes};
2296}
2297
2298PtrParts SplitPtrStructs::visitExtractElementInst(ExtractElementInst &I) {
2299 if (!isSplitFatPtr(Ty: I.getType()))
2300 return {nullptr, nullptr};
2301 IRB.SetInsertPoint(&I);
2302 Value *Vec = I.getVectorOperand();
2303 Value *Idx = I.getIndexOperand();
2304 auto [Rsrc, Off] = getPtrParts(V: Vec);
2305
2306 Value *RsrcRes = IRB.CreateExtractElement(Vec: Rsrc, Idx, Name: I.getName() + ".rsrc");
2307 copyMetadata(Dest: RsrcRes, Src: &I);
2308 Value *OffRes = IRB.CreateExtractElement(Vec: Off, Idx, Name: I.getName() + ".off");
2309 copyMetadata(Dest: OffRes, Src: &I);
2310 SplitUsers.insert(V: &I);
2311 return {RsrcRes, OffRes};
2312}
2313
2314PtrParts SplitPtrStructs::visitInsertElementInst(InsertElementInst &I) {
2315 // The mutated instructions temporarily don't return vectors, and so
2316 // we need the generic getType() here to avoid crashes.
2317 if (!isSplitFatPtr(Ty: cast<Instruction>(Val&: I).getType()))
2318 return {nullptr, nullptr};
2319 IRB.SetInsertPoint(&I);
2320 Value *Vec = I.getOperand(i_nocapture: 0);
2321 Value *Elem = I.getOperand(i_nocapture: 1);
2322 Value *Idx = I.getOperand(i_nocapture: 2);
2323 auto [VecRsrc, VecOff] = getPtrParts(V: Vec);
2324 auto [ElemRsrc, ElemOff] = getPtrParts(V: Elem);
2325
2326 Value *RsrcRes =
2327 IRB.CreateInsertElement(Vec: VecRsrc, NewElt: ElemRsrc, Idx, Name: I.getName() + ".rsrc");
2328 copyMetadata(Dest: RsrcRes, Src: &I);
2329 Value *OffRes =
2330 IRB.CreateInsertElement(Vec: VecOff, NewElt: ElemOff, Idx, Name: I.getName() + ".off");
2331 copyMetadata(Dest: OffRes, Src: &I);
2332 SplitUsers.insert(V: &I);
2333 return {RsrcRes, OffRes};
2334}
2335
2336PtrParts SplitPtrStructs::visitShuffleVectorInst(ShuffleVectorInst &I) {
2337 // Cast is needed for the same reason as insertelement's.
2338 if (!isSplitFatPtr(Ty: cast<Instruction>(Val&: I).getType()))
2339 return {nullptr, nullptr};
2340 IRB.SetInsertPoint(&I);
2341
2342 Value *V1 = I.getOperand(i_nocapture: 0);
2343 Value *V2 = I.getOperand(i_nocapture: 1);
2344 ArrayRef<int> Mask = I.getShuffleMask();
2345 auto [V1Rsrc, V1Off] = getPtrParts(V: V1);
2346 auto [V2Rsrc, V2Off] = getPtrParts(V: V2);
2347
2348 Value *RsrcRes =
2349 IRB.CreateShuffleVector(V1: V1Rsrc, V2: V2Rsrc, Mask, Name: I.getName() + ".rsrc");
2350 copyMetadata(Dest: RsrcRes, Src: &I);
2351 Value *OffRes =
2352 IRB.CreateShuffleVector(V1: V1Off, V2: V2Off, Mask, Name: I.getName() + ".off");
2353 copyMetadata(Dest: OffRes, Src: &I);
2354 SplitUsers.insert(V: &I);
2355 return {RsrcRes, OffRes};
2356}
2357
2358PtrParts SplitPtrStructs::visitPHINode(PHINode &PHI) {
2359 if (!isSplitFatPtr(Ty: PHI.getType()))
2360 return {nullptr, nullptr};
2361 IRB.SetInsertPoint(*PHI.getInsertionPointAfterDef());
2362 // Phi nodes will be handled in post-processing after we've visited every
2363 // instruction. However, instead of just returning {nullptr, nullptr},
2364 // we explicitly create the temporary extractvalue operations that are our
2365 // temporary results so that they end up at the beginning of the block with
2366 // the PHIs.
2367 Value *TmpRsrc = IRB.CreateExtractValue(Agg: &PHI, Idxs: 0, Name: PHI.getName() + ".rsrc");
2368 Value *TmpOff = IRB.CreateExtractValue(Agg: &PHI, Idxs: 1, Name: PHI.getName() + ".off");
2369 Conditionals.push_back(Elt: &PHI);
2370 SplitUsers.insert(V: &PHI);
2371 return {TmpRsrc, TmpOff};
2372}
2373
2374PtrParts SplitPtrStructs::visitSelectInst(SelectInst &SI) {
2375 if (!isSplitFatPtr(Ty: SI.getType()))
2376 return {nullptr, nullptr};
2377 IRB.SetInsertPoint(&SI);
2378
2379 Value *Cond = SI.getCondition();
2380 Value *True = SI.getTrueValue();
2381 Value *False = SI.getFalseValue();
2382 auto [TrueRsrc, TrueOff] = getPtrParts(V: True);
2383 auto [FalseRsrc, FalseOff] = getPtrParts(V: False);
2384
2385 Value *RsrcRes =
2386 IRB.CreateSelect(C: Cond, True: TrueRsrc, False: FalseRsrc, Name: SI.getName() + ".rsrc", MDFrom: &SI);
2387 copyMetadata(Dest: RsrcRes, Src: &SI);
2388 Conditionals.push_back(Elt: &SI);
2389 Value *OffRes =
2390 IRB.CreateSelect(C: Cond, True: TrueOff, False: FalseOff, Name: SI.getName() + ".off", MDFrom: &SI);
2391 copyMetadata(Dest: OffRes, Src: &SI);
2392 SplitUsers.insert(V: &SI);
2393 return {RsrcRes, OffRes};
2394}
2395
2396/// Returns true if this intrinsic needs to be removed when it is
2397/// applied to `ptr addrspace(7)` values. Calls to these intrinsics are
2398/// rewritten into calls to versions of that intrinsic on the resource
2399/// descriptor.
2400static bool isRemovablePointerIntrinsic(Intrinsic::ID IID) {
2401 switch (IID) {
2402 default:
2403 return false;
2404 case Intrinsic::amdgcn_make_buffer_rsrc:
2405 case Intrinsic::ptrmask:
2406 case Intrinsic::invariant_start:
2407 case Intrinsic::invariant_end:
2408 case Intrinsic::launder_invariant_group:
2409 case Intrinsic::memcpy:
2410 case Intrinsic::memcpy_inline:
2411 case Intrinsic::memmove:
2412 case Intrinsic::memset:
2413 case Intrinsic::memset_inline:
2414 case Intrinsic::experimental_memset_pattern:
2415 case Intrinsic::amdgcn_load_to_lds:
2416 case Intrinsic::amdgcn_load_async_to_lds:
2417 return true;
2418 }
2419}
2420
2421PtrParts SplitPtrStructs::visitIntrinsicInst(IntrinsicInst &I) {
2422 Intrinsic::ID IID = I.getIntrinsicID();
2423 switch (IID) {
2424 default:
2425 break;
2426 case Intrinsic::amdgcn_make_buffer_rsrc: {
2427 if (!isSplitFatPtr(Ty: I.getType()))
2428 return {nullptr, nullptr};
2429 Value *Base = I.getArgOperand(i: 0);
2430 Value *Stride = I.getArgOperand(i: 1);
2431 Value *NumRecords = I.getArgOperand(i: 2);
2432 Value *Flags = I.getArgOperand(i: 3);
2433 auto *SplitType = cast<StructType>(Val: I.getType());
2434 Type *RsrcType = SplitType->getElementType(N: 0);
2435 Type *OffType = SplitType->getElementType(N: 1);
2436 IRB.SetInsertPoint(&I);
2437 Value *Rsrc = IRB.CreateIntrinsic(
2438 ID: IID, OverloadTypes: {RsrcType, Base->getType(), NumRecords->getType()},
2439 Args: {Base, Stride, NumRecords, Flags});
2440 copyMetadata(Dest: Rsrc, Src: &I);
2441 Rsrc->takeName(V: &I);
2442 Value *Zero = Constant::getNullValue(Ty: OffType);
2443 SplitUsers.insert(V: &I);
2444 return {Rsrc, Zero};
2445 }
2446 case Intrinsic::ptrmask: {
2447 Value *Ptr = I.getArgOperand(i: 0);
2448 if (!isSplitFatPtr(Ty: Ptr->getType()))
2449 return {nullptr, nullptr};
2450 Value *Mask = I.getArgOperand(i: 1);
2451 IRB.SetInsertPoint(&I);
2452 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2453 if (Mask->getType() != Off->getType())
2454 reportFatalUsageError(reason: "offset width is not equal to index width of fat "
2455 "pointer (data layout not set up correctly?)");
2456 Value *OffRes = IRB.CreateAnd(LHS: Off, RHS: Mask, Name: I.getName() + ".off");
2457 copyMetadata(Dest: OffRes, Src: &I);
2458 SplitUsers.insert(V: &I);
2459 return {Rsrc, OffRes};
2460 }
2461 // Pointer annotation intrinsics that, given their object-wide nature
2462 // operate on the resource part.
2463 case Intrinsic::invariant_start: {
2464 Value *Ptr = I.getArgOperand(i: 1);
2465 if (!isSplitFatPtr(Ty: Ptr->getType()))
2466 return {nullptr, nullptr};
2467 IRB.SetInsertPoint(&I);
2468 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2469 Type *NewTy = PointerType::get(C&: I.getContext(), AddressSpace: AMDGPUAS::BUFFER_RESOURCE);
2470 auto *NewRsrc = IRB.CreateIntrinsic(ID: IID, OverloadTypes: {NewTy}, Args: {I.getOperand(i_nocapture: 0), Rsrc});
2471 copyMetadata(Dest: NewRsrc, Src: &I);
2472 NewRsrc->takeName(V: &I);
2473 SplitUsers.insert(V: &I);
2474 I.replaceAllUsesWith(V: NewRsrc);
2475 return {nullptr, nullptr};
2476 }
2477 case Intrinsic::invariant_end: {
2478 Value *RealPtr = I.getArgOperand(i: 2);
2479 if (!isSplitFatPtr(Ty: RealPtr->getType()))
2480 return {nullptr, nullptr};
2481 IRB.SetInsertPoint(&I);
2482 Value *RealRsrc = getPtrParts(V: RealPtr).first;
2483 Value *InvPtr = I.getArgOperand(i: 0);
2484 Value *Size = I.getArgOperand(i: 1);
2485 Value *NewRsrc = IRB.CreateIntrinsic(ID: IID, OverloadTypes: {RealRsrc->getType()},
2486 Args: {InvPtr, Size, RealRsrc});
2487 copyMetadata(Dest: NewRsrc, Src: &I);
2488 NewRsrc->takeName(V: &I);
2489 SplitUsers.insert(V: &I);
2490 I.replaceAllUsesWith(V: NewRsrc);
2491 return {nullptr, nullptr};
2492 }
2493 case Intrinsic::launder_invariant_group: {
2494 Value *Ptr = I.getArgOperand(i: 0);
2495 if (!isSplitFatPtr(Ty: Ptr->getType()))
2496 return {nullptr, nullptr};
2497 IRB.SetInsertPoint(&I);
2498 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2499 Value *NewRsrc = IRB.CreateIntrinsic(ID: IID, OverloadTypes: {Rsrc->getType()}, Args: {Rsrc});
2500 copyMetadata(Dest: NewRsrc, Src: &I);
2501 NewRsrc->takeName(V: &I);
2502 SplitUsers.insert(V: &I);
2503 return {NewRsrc, Off};
2504 }
2505 case Intrinsic::amdgcn_load_to_lds:
2506 case Intrinsic::amdgcn_load_async_to_lds: {
2507 Value *Ptr = I.getArgOperand(i: 0);
2508 if (!isSplitFatPtr(Ty: Ptr->getType()))
2509 return {nullptr, nullptr};
2510 IRB.SetInsertPoint(&I);
2511 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2512 Value *LDSPtr = I.getArgOperand(i: 1);
2513 Value *LoadSize = I.getArgOperand(i: 2);
2514 Value *ImmOff = I.getArgOperand(i: 3);
2515 Value *Aux = I.getArgOperand(i: 4);
2516 Value *SOffset = IRB.getInt32(C: 0);
2517 Intrinsic::ID NewIntr =
2518 IID == Intrinsic::amdgcn_load_to_lds
2519 ? Intrinsic::amdgcn_raw_ptr_buffer_load_lds
2520 : Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds;
2521 Instruction *NewLoad = IRB.CreateIntrinsicWithoutFolding(
2522 ID: NewIntr, OverloadTypes: {}, Args: {Rsrc, LDSPtr, LoadSize, Off, SOffset, ImmOff, Aux});
2523 copyMetadata(Dest: NewLoad, Src: &I);
2524 SplitUsers.insert(V: &I);
2525 I.replaceAllUsesWith(V: NewLoad);
2526 return {nullptr, nullptr};
2527 }
2528 case Intrinsic::prefetch: {
2529 Value *Ptr = I.getArgOperand(i: 0);
2530 if (!isSplitFatPtr(Ty: Ptr->getType()))
2531 return {nullptr, nullptr};
2532 // s_buffer_prefetch_data only takes an immediate offset, so only a data
2533 // prefetch at a constant offset can use it. Drop the others: a prefetch is
2534 // only a hint.
2535 if (ST->hasSafeSmemPrefetch() &&
2536 cast<ConstantInt>(Val: I.getArgOperand(i: 3))->isOne()) {
2537 IRB.SetInsertPoint(&I);
2538 auto [Rsrc, Off] = getPtrParts(V: Ptr);
2539 auto *ConstOff = dyn_cast<ConstantInt>(Val: Off);
2540 if (ConstOff && AMDGPU::isLegalSMRDEncodedSignedOffset(
2541 ST: *ST, EncodedOffset: ConstOff->getZExtValue(), /*IsBuffer=*/true)) {
2542 Value *NewPrefetch =
2543 IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_s_buffer_prefetch_data, OverloadTypes: {},
2544 Args: {Rsrc, ConstOff, IRB.getInt32(C: 0)});
2545 copyMetadata(Dest: NewPrefetch, Src: &I);
2546 }
2547 }
2548 SplitUsers.insert(V: &I);
2549 return {nullptr, nullptr};
2550 }
2551 case Intrinsic::amdgcn_s_prefetch_data:
2552 case Intrinsic::amdgcn_s_prefetch_inst:
2553 if (!isSplitFatPtr(Ty: I.getArgOperand(i: 0)->getType()))
2554 return {nullptr, nullptr};
2555 // Instruction selection drops these for every other non-global pointer.
2556 SplitUsers.insert(V: &I);
2557 return {nullptr, nullptr};
2558 }
2559 return {nullptr, nullptr};
2560}
2561
2562void SplitPtrStructs::processFunction(Function &F) {
2563 ST = &TM->getSubtarget<GCNSubtarget>(F);
2564 SmallVector<Instruction *, 0> Originals(
2565 llvm::make_pointer_range(Range: instructions(F)));
2566 LLVM_DEBUG(dbgs() << "Splitting pointer structs in function: " << F.getName()
2567 << "\n");
2568 for (Instruction *I : Originals) {
2569 // In some cases, instruction order doesn't reflect program order,
2570 // so the visit() call will have already visited coertain instructions
2571 // by the time this loop gets to them. Avoid re-visiting these so as to,
2572 // for example, avoid processing the same conditional twice.
2573 if (SplitUsers.contains(V: I))
2574 continue;
2575 auto [Rsrc, Off] = visit(I);
2576 assert(((Rsrc && Off) || (!Rsrc && !Off)) &&
2577 "Can't have a resource but no offset");
2578 if (Rsrc)
2579 RsrcParts[I] = Rsrc;
2580 if (Off)
2581 OffParts[I] = Off;
2582 }
2583 processConditionals();
2584 killAndReplaceSplitInstructions(Origs&: Originals);
2585
2586 // Clean up after ourselves to save on memory.
2587 RsrcParts.clear();
2588 OffParts.clear();
2589 SplitUsers.clear();
2590 Conditionals.clear();
2591 ConditionalTemps.clear();
2592}
2593
2594namespace {
2595class AMDGPULowerBufferFatPointers : public ModulePass {
2596public:
2597 static char ID;
2598
2599 AMDGPULowerBufferFatPointers() : ModulePass(ID) {}
2600
2601 bool run(Module &M, const TargetMachine &TM, GetTTIFn GetTTI, GetSEFn GetSE);
2602 bool runOnModule(Module &M) override;
2603
2604 void getAnalysisUsage(AnalysisUsage &AU) const override;
2605};
2606} // namespace
2607
2608/// Returns true if there are values that have a buffer fat pointer in them,
2609/// which means we'll need to perform rewrites on this function. As a side
2610/// effect, this will populate the type remapping cache.
2611static bool containsBufferFatPointers(const Function &F,
2612 BufferFatPtrToStructTypeMap *TypeMap) {
2613 bool HasFatPointers = false;
2614 for (const BasicBlock &BB : F)
2615 for (const Instruction &I : BB) {
2616 HasFatPointers |= (I.getType() != TypeMap->remapType(SrcTy: I.getType()));
2617 // Catch null pointer constants in loads, stores, etc.
2618 for (const Value *V : I.operand_values())
2619 HasFatPointers |= (V->getType() != TypeMap->remapType(SrcTy: V->getType()));
2620 }
2621 return HasFatPointers;
2622}
2623
2624static bool hasFatPointerInterface(const Function &F,
2625 BufferFatPtrToStructTypeMap *TypeMap) {
2626 Type *Ty = F.getFunctionType();
2627 return Ty != TypeMap->remapType(SrcTy: Ty);
2628}
2629
2630/// Move the body of `OldF` into a new function, returning it.
2631static Function *moveFunctionAdaptingType(Function *OldF, FunctionType *NewTy,
2632 ValueToValueMapTy &CloneMap) {
2633 bool IsIntrinsic = OldF->isIntrinsic();
2634 Function *NewF =
2635 Function::Create(Ty: NewTy, Linkage: OldF->getLinkage(), AddrSpace: OldF->getAddressSpace());
2636 NewF->copyAttributesFrom(Src: OldF);
2637 NewF->copyMetadata(Src: OldF, Offset: 0);
2638 NewF->takeName(V: OldF);
2639 NewF->updateAfterNameChange();
2640 NewF->setDLLStorageClass(OldF->getDLLStorageClass());
2641 OldF->getParent()->getFunctionList().insertAfter(where: OldF->getIterator(), New: NewF);
2642
2643 while (!OldF->empty()) {
2644 BasicBlock *BB = &OldF->front();
2645 BB->removeFromParent();
2646 BB->insertInto(Parent: NewF);
2647 CloneMap[BB] = BB;
2648 for (Instruction &I : *BB) {
2649 CloneMap[&I] = &I;
2650 }
2651 }
2652
2653 SmallVector<AttributeSet> ArgAttrs;
2654 AttributeList OldAttrs = OldF->getAttributes();
2655
2656 for (auto [I, OldArg, NewArg] : enumerate(First: OldF->args(), Rest: NewF->args())) {
2657 CloneMap[&NewArg] = &OldArg;
2658 NewArg.takeName(V: &OldArg);
2659 Type *OldArgTy = OldArg.getType(), *NewArgTy = NewArg.getType();
2660 // Temporarily mutate type of `NewArg` to allow RAUW to work.
2661 NewArg.mutateType(Ty: OldArgTy);
2662 OldArg.replaceAllUsesWith(V: &NewArg);
2663 NewArg.mutateType(Ty: NewArgTy);
2664
2665 AttributeSet ArgAttr = OldAttrs.getParamAttrs(ArgNo: I);
2666 // Intrinsics get their attributes fixed later.
2667 if (OldArgTy != NewArgTy && !IsIntrinsic)
2668 ArgAttr = ArgAttr.removeAttributes(
2669 C&: NewF->getContext(),
2670 AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewArgTy, AS: ArgAttr));
2671 ArgAttrs.push_back(Elt: ArgAttr);
2672 }
2673 AttributeSet RetAttrs = OldAttrs.getRetAttrs();
2674 if (OldF->getReturnType() != NewF->getReturnType() && !IsIntrinsic)
2675 RetAttrs = RetAttrs.removeAttributes(
2676 C&: NewF->getContext(),
2677 AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewF->getReturnType(), AS: RetAttrs));
2678 NewF->setAttributes(AttributeList::get(
2679 C&: NewF->getContext(), FnAttrs: OldAttrs.getFnAttrs(), RetAttrs, ArgAttrs));
2680 return NewF;
2681}
2682
2683static void makeCloneInPraceMap(Function *F, ValueToValueMapTy &CloneMap) {
2684 for (Argument &A : F->args())
2685 CloneMap[&A] = &A;
2686 for (BasicBlock &BB : *F) {
2687 CloneMap[&BB] = &BB;
2688 for (Instruction &I : BB)
2689 CloneMap[&I] = &I;
2690 }
2691}
2692
2693bool AMDGPULowerBufferFatPointers::run(Module &M, const TargetMachine &TM,
2694 GetTTIFn GetTTI, GetSEFn GetSE) {
2695 bool Changed = false;
2696 const DataLayout &DL = M.getDataLayout();
2697 // Record the functions which need to be remapped.
2698 // The second element of the pair indicates whether the function has to have
2699 // its arguments or return types adjusted.
2700 SmallVector<std::pair<Function *, bool>> NeedsRemap;
2701
2702 LLVMContext &Ctx = M.getContext();
2703
2704 BufferFatPtrToStructTypeMap StructTM(DL);
2705 BufferFatPtrToIntTypeMap IntTM(DL);
2706 for (GlobalVariable &GV : make_early_inc_range(Range: M.globals())) {
2707 if (GV.getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER) {
2708 // FIXME: Use DiagnosticInfo unsupported but it requires a Function
2709 Ctx.emitError(ErrorStr: "global variables with a buffer fat pointer address "
2710 "space (7) are not supported");
2711 GV.replaceAllUsesWith(V: PoisonValue::get(T: GV.getType()));
2712 GV.eraseFromParent();
2713 Changed = true;
2714 continue;
2715 }
2716
2717 Type *VT = GV.getValueType();
2718 if (VT != StructTM.remapType(SrcTy: VT)) {
2719 // FIXME: Use DiagnosticInfo unsupported but it requires a Function
2720 Ctx.emitError(ErrorStr: "global variables that contain buffer fat pointers "
2721 "(address space 7 pointers) are unsupported. Use "
2722 "buffer resource pointers (address space 8) instead");
2723 GV.replaceAllUsesWith(V: PoisonValue::get(T: GV.getType()));
2724 GV.eraseFromParent();
2725 Changed = true;
2726 continue;
2727 }
2728 }
2729
2730 {
2731 // Collect all constant exprs and aggregates referenced by any function.
2732 SmallVector<Constant *, 8> Worklist;
2733 for (Function &F : M.functions())
2734 for (Instruction &I : instructions(F))
2735 for (Value *Op : I.operands())
2736 if (isa<ConstantExpr, ConstantAggregate>(Val: Op))
2737 Worklist.push_back(Elt: cast<Constant>(Val: Op));
2738
2739 // Recursively look for any referenced buffer pointer constants.
2740 SmallPtrSet<Constant *, 8> Visited;
2741 SetVector<Constant *> BufferFatPtrConsts;
2742 while (!Worklist.empty()) {
2743 Constant *C = Worklist.pop_back_val();
2744 if (!Visited.insert(Ptr: C).second)
2745 continue;
2746 if (isBufferFatPtrOrVector(Ty: C->getType()))
2747 BufferFatPtrConsts.insert(X: C);
2748 for (Value *Op : C->operands())
2749 if (isa<ConstantExpr, ConstantAggregate>(Val: Op))
2750 Worklist.push_back(Elt: cast<Constant>(Val: Op));
2751 }
2752
2753 // Expand all constant expressions using fat buffer pointers to
2754 // instructions.
2755 Changed |= convertUsersOfConstantsToInstructions(
2756 Consts: BufferFatPtrConsts.getArrayRef(), /*RestrictToFunc=*/nullptr,
2757 /*RemoveDeadConstants=*/false, /*IncludeSelf=*/true);
2758 }
2759
2760 StoreFatPtrsAsIntsAndExpandMemcpyVisitor MemOpsRewrite(&IntTM, M);
2761 LegalizeBufferContentTypesVisitor BufferContentsTypeRewrite(M, &TM);
2762 for (Function &F : M.functions()) {
2763 bool InterfaceChange = hasFatPointerInterface(F, TypeMap: &StructTM);
2764 bool BodyChanges = containsBufferFatPointers(F, TypeMap: &StructTM);
2765 const TargetTransformInfo *TTI = GetTTI(F);
2766 ScalarEvolution *SE = GetSE(F);
2767 Changed |= MemOpsRewrite.processFunction(F, TTI, SE);
2768 if (InterfaceChange || BodyChanges) {
2769 NeedsRemap.push_back(Elt: std::make_pair(x: &F, y&: InterfaceChange));
2770 Changed |= BufferContentsTypeRewrite.processFunction(F, SE);
2771 }
2772 }
2773 if (NeedsRemap.empty())
2774 return Changed;
2775
2776 SmallVector<Function *> NeedsPostProcess;
2777 SmallVector<Function *> Intrinsics;
2778 // Keep one big map so as to memoize constants across functions.
2779 ValueToValueMapTy CloneMap;
2780 FatPtrConstMaterializer Materializer(&StructTM, CloneMap);
2781
2782 ValueMapper LowerInFuncs(CloneMap, RF_None, &StructTM, &Materializer);
2783 for (auto [F, InterfaceChange] : NeedsRemap) {
2784 Function *NewF = F;
2785 if (InterfaceChange)
2786 NewF = moveFunctionAdaptingType(
2787 OldF: F, NewTy: cast<FunctionType>(Val: StructTM.remapType(SrcTy: F->getFunctionType())),
2788 CloneMap);
2789 else
2790 makeCloneInPraceMap(F, CloneMap);
2791 LowerInFuncs.remapFunction(F&: *NewF);
2792 if (NewF->isIntrinsic())
2793 Intrinsics.push_back(Elt: NewF);
2794 else
2795 NeedsPostProcess.push_back(Elt: NewF);
2796 if (InterfaceChange) {
2797 F->replaceAllUsesWith(V: NewF);
2798 F->eraseFromParent();
2799 }
2800 Changed = true;
2801 }
2802 StructTM.clear();
2803 IntTM.clear();
2804 CloneMap.clear();
2805
2806 SplitPtrStructs Splitter(M, &TM);
2807 for (Function *F : NeedsPostProcess)
2808 Splitter.processFunction(F&: *F);
2809 for (Function *F : Intrinsics) {
2810 // use_empty() can also occur with cases like masked load, which will
2811 // have been rewritten out of the module by now but not erased.
2812 if (F->use_empty() || isRemovablePointerIntrinsic(IID: F->getIntrinsicID())) {
2813 F->eraseFromParent();
2814 } else {
2815 std::optional<Function *> NewF = Intrinsic::remangleIntrinsicFunction(F);
2816 if (NewF)
2817 F->replaceAllUsesWith(V: *NewF);
2818 }
2819 }
2820 return Changed;
2821}
2822
2823bool AMDGPULowerBufferFatPointers::runOnModule(Module &M) {
2824 TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
2825 const TargetMachine &TM = TPC.getTM<TargetMachine>();
2826 auto GetTTI = [&](Function &F) -> const TargetTransformInfo * {
2827 if (F.isDeclaration())
2828 return nullptr;
2829 return &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2830 };
2831 auto GetSE = [&](Function &F) -> ScalarEvolution * {
2832 if (F.isDeclaration())
2833 return nullptr;
2834 return &getAnalysis<ScalarEvolutionWrapperPass>(F).getSE();
2835 };
2836 return run(M, TM, GetTTI, GetSE);
2837}
2838
2839char AMDGPULowerBufferFatPointers::ID = 0;
2840
2841char &llvm::AMDGPULowerBufferFatPointersID = AMDGPULowerBufferFatPointers::ID;
2842
2843void AMDGPULowerBufferFatPointers::getAnalysisUsage(AnalysisUsage &AU) const {
2844 AU.addRequired<TargetPassConfig>();
2845 AU.addRequired<TargetTransformInfoWrapperPass>();
2846 AU.addRequired<ScalarEvolutionWrapperPass>();
2847}
2848
2849#define PASS_DESC "Lower buffer fat pointer operations to buffer resources"
2850INITIALIZE_PASS_BEGIN(AMDGPULowerBufferFatPointers, DEBUG_TYPE, PASS_DESC,
2851 false, false)
2852INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
2853INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
2854INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
2855INITIALIZE_PASS_END(AMDGPULowerBufferFatPointers, DEBUG_TYPE, PASS_DESC, false,
2856 false)
2857#undef PASS_DESC
2858
2859ModulePass *llvm::createAMDGPULowerBufferFatPointersPass() {
2860 return new AMDGPULowerBufferFatPointers();
2861}
2862
2863PreservedAnalyses
2864AMDGPULowerBufferFatPointersPass::run(Module &M, ModuleAnalysisManager &MA) {
2865 auto &FA = MA.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
2866 auto GetTTI = [&](Function &F) -> const TargetTransformInfo * {
2867 if (F.isDeclaration())
2868 return nullptr;
2869 return &FA.getResult<TargetIRAnalysis>(IR&: F);
2870 };
2871 auto GetSE = [&](Function &F) -> ScalarEvolution * {
2872 if (F.isDeclaration())
2873 return nullptr;
2874 return &FA.getResult<ScalarEvolutionAnalysis>(IR&: F);
2875 };
2876 return AMDGPULowerBufferFatPointers().run(M, TM, GetTTI, GetSE)
2877 ? PreservedAnalyses::none()
2878 : PreservedAnalyses::all();
2879}
2880