1//===- DXILResourceAccess.cpp - Resource access via load/store ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DirectX.h"
10#include "llvm/ADT/DenseMap.h"
11#include "llvm/ADT/MapVector.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/Analysis/DXILResource.h"
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/Analysis/VectorUtils.h"
16#include "llvm/Frontend/HLSL/HLSLResource.h"
17#include "llvm/IR/BasicBlock.h"
18#include "llvm/IR/Dominators.h"
19#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/Instruction.h"
21#include "llvm/IR/Instructions.h"
22#include "llvm/IR/IntrinsicInst.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/IntrinsicsDirectX.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/ProfDataUtils.h"
27#include "llvm/IR/User.h"
28#include "llvm/IR/ValueHandle.h"
29#include "llvm/InitializePasses.h"
30#include "llvm/Support/DXILABI.h"
31#include "llvm/Support/ErrorHandling.h"
32#include "llvm/Support/FormatVariadic.h"
33#include "llvm/Support/KnownBits.h"
34#include "llvm/Transforms/Utils/BasicBlockUtils.h"
35#include "llvm/Transforms/Utils/Local.h"
36#include "llvm/Transforms/Utils/ValueMapper.h"
37#include <optional>
38
39#define DEBUG_TYPE "dxil-resource-access"
40
41using namespace llvm;
42
43[[noreturn]] static void
44diagnoseNonUniqueResourceAccess(Instruction *I,
45 ArrayRef<IntrinsicInst *> Handles) {
46 LLVMContext &Context = I->getContext();
47 std::string InstStr;
48 raw_string_ostream InstOS(InstStr);
49 I->print(O&: InstOS);
50 Context.diagnose(
51 DI: DiagnosticInfoGeneric("At resource access:" + Twine(InstStr), DS_Note));
52
53 for (auto *Handle : Handles) {
54 std::string HandleStr;
55 raw_string_ostream HandleOS(HandleStr);
56 Handle->print(O&: HandleOS);
57 Context.diagnose(DI: DiagnosticInfoGeneric(
58 "Uses resource handle:" + Twine(HandleStr), DS_Note));
59 }
60 report_fatal_error(
61 reason: "Resource access is not guaranteed to map to a unique global resource",
62 /*gen_crash_diag=*/false);
63}
64
65static Value *traverseGEPOffsets(const DataLayout &DL, IRBuilder<> &Builder,
66 Value *Ptr, uint64_t AccessSize) {
67 Value *Offset = nullptr;
68
69 while (Ptr) {
70 if ([[maybe_unused]] auto *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
71 assert((II->getIntrinsicID() == Intrinsic::dx_resource_getpointer ||
72 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer) &&
73 "Resource access through unexpected intrinsic");
74 return Offset ? Offset : ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
75 }
76
77 auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
78 assert(GEP && "Resource access through unexpected instruction");
79
80 unsigned NumIndices = GEP->getNumIndices();
81 uint64_t IndexScale = DL.getTypeAllocSize(Ty: GEP->getSourceElementType());
82 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: GEP->getType()), 0);
83 Value *GEPOffset;
84 if (GEP->accumulateConstantOffset(DL, Offset&: ConstantOffset)) {
85 // We have a constant offset (in bytes).
86 GEPOffset =
87 ConstantInt::get(Ty: DL.getIndexType(PtrTy: GEP->getType()), V: ConstantOffset);
88 IndexScale = 1;
89 } else if (NumIndices == 1) {
90 // If we have a single index we're indexing into a top level array. This
91 // generally only happens with cbuffers.
92 GEPOffset = *GEP->idx_begin();
93 } else if (NumIndices == 2) {
94 // If we have two indices, this should be an access through a pointer.
95 auto *IndexIt = GEP->idx_begin();
96 assert(cast<ConstantInt>(IndexIt)->getZExtValue() == 0 &&
97 "GEP is not indexing through pointer");
98 GEPOffset = *(++IndexIt);
99 } else
100 llvm_unreachable("Unhandled GEP structure for resource access");
101
102 uint64_t ElemSize = AccessSize;
103 if (!(IndexScale % ElemSize)) {
104 // If our scale is an exact multiple of the access size, adjust the
105 // scaling to avoid an unnecessary division.
106 IndexScale /= ElemSize;
107 ElemSize = 1;
108 }
109 if (IndexScale != 1)
110 GEPOffset = Builder.CreateMul(
111 LHS: GEPOffset, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: IndexScale));
112 if (ElemSize != 1)
113 GEPOffset = Builder.CreateUDiv(
114 LHS: GEPOffset, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: ElemSize));
115
116 Offset = Offset ? Builder.CreateAdd(LHS: Offset, RHS: GEPOffset) : GEPOffset;
117 Ptr = GEP->getPointerOperand();
118 }
119
120 llvm_unreachable("GEP of null pointer?");
121}
122
123static void createTypedBufferStore(IntrinsicInst *II, StoreInst *SI,
124 dxil::ResourceTypeInfo &RTI) {
125 const DataLayout &DL = SI->getDataLayout();
126 IRBuilder<> Builder(SI);
127 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
128 Type *ScalarType = ContainedType->getScalarType();
129 Type *LoadType = StructType::get(elt1: ContainedType, elts: Builder.getInt1Ty());
130
131 Value *V = SI->getValueOperand();
132 if (V->getType() == ContainedType) {
133 // V is already the right type.
134 assert(SI->getPointerOperand() == II &&
135 "Store of whole element has mismatched address to store to");
136 } else if (V->getType() == ScalarType) {
137 // We're storing a scalar, so we need to load the current value and only
138 // replace the relevant part.
139 auto *Load = Builder.CreateIntrinsic(
140 RetTy: LoadType, ID: Intrinsic::dx_resource_load_typedbuffer,
141 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1)});
142 auto *Struct = Builder.CreateExtractValue(Agg: Load, Idxs: {0});
143
144 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
145 Value *Offset =
146 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
147 V = Builder.CreateInsertElement(Vec: Struct, NewElt: V, Idx: Offset);
148 } else {
149 llvm_unreachable("Store to typed resource has invalid type");
150 }
151
152 auto *Inst = Builder.CreateIntrinsic(
153 RetTy: Builder.getVoidTy(), ID: Intrinsic::dx_resource_store_typedbuffer,
154 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1), V});
155 SI->replaceAllUsesWith(V: Inst);
156}
157
158/// Build a zero-initialized offset operand matching the shape of the given
159/// coordinate operand. Accesses through `operator[]` never have offsets.
160static Value *getNullOffsetsFor(IRBuilder<> &Builder, Value *Coords) {
161 Type *CoordTy = Coords->getType();
162 Type *OffsetTy;
163 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: CoordTy))
164 OffsetTy =
165 FixedVectorType::get(ElementType: Builder.getInt32Ty(), NumElts: VecTy->getNumElements());
166 else
167 OffsetTy = Builder.getInt32Ty();
168 return Constant::getNullValue(Ty: OffsetTy);
169}
170
171static void createTextureStore(IntrinsicInst *II, StoreInst *SI,
172 dxil::ResourceTypeInfo &RTI) {
173 const DataLayout &DL = SI->getDataLayout();
174 IRBuilder<> Builder(SI);
175 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
176 Type *ScalarType = ContainedType->getScalarType();
177
178 Value *Handle = II->getOperand(i_nocapture: 0);
179 Value *Coords = II->getOperand(i_nocapture: 1);
180
181 Value *V = SI->getValueOperand();
182 if (V->getType() == ContainedType) {
183 // V is already the right type.
184 assert(SI->getPointerOperand() == II &&
185 "Store of whole element has mismatched address to store to");
186 } else if (V->getType() == ScalarType) {
187 // We're storing a scalar, so we need to load the current value and only
188 // replace the relevant part. For operator[] the mip level and the offsets
189 // are always zero; DXILOpLowering drops the mip level for UAVs.
190 Value *MipLevel = Builder.getInt32(C: 0);
191 Value *Offsets = getNullOffsetsFor(Builder, Coords);
192 auto *Load = Builder.CreateIntrinsic(RetTy: ContainedType,
193 ID: Intrinsic::dx_resource_load_level,
194 Args: {Handle, Coords, MipLevel, Offsets});
195
196 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
197 Value *Offset =
198 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
199 V = Builder.CreateInsertElement(Vec: Load, NewElt: V, Idx: Offset);
200 } else {
201 llvm_unreachable("Store to texture resource has invalid type");
202 }
203
204 auto *Inst = Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(),
205 ID: Intrinsic::dx_resource_store_texture,
206 Args: {Handle, Coords, V});
207 SI->replaceAllUsesWith(V: Inst);
208}
209
210static void emitRawStore(IRBuilder<> &Builder, Value *Buffer, Value *Index,
211 Value *Offset, Value *V, dxil::ResourceTypeInfo &RTI) {
212 // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
213 // entirely into the index.
214 if (!RTI.isStruct()) {
215 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
216 if (!ConstantOffset || !ConstantOffset->isZero())
217 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
218 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
219 }
220
221 Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(),
222 ID: Intrinsic::dx_resource_store_rawbuffer,
223 Args: {Buffer, Index, Offset, V});
224}
225
226static void createRawStores(IntrinsicInst *II, StoreInst *SI,
227 dxil::ResourceTypeInfo &RTI) {
228 const DataLayout &DL = SI->getDataLayout();
229 IRBuilder<> Builder(SI);
230
231 Value *V = SI->getValueOperand();
232 assert(!V->getType()->isAggregateType() &&
233 "Resource store should be scalar or vector type");
234
235 Value *Index = II->getOperand(i_nocapture: 1);
236 // The offset for the rawbuffer load and store ops is always in bytes.
237 uint64_t AccessSize = 1;
238 Value *Offset =
239 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
240
241 auto *VT = dyn_cast<FixedVectorType>(Val: V->getType());
242 if (VT && VT->getNumElements() > 4) {
243 // Split into stores of at most 4 elements.
244 Type *EltTy = VT->getElementType();
245 Value *Stride = ConstantInt::get(Ty: Builder.getInt32Ty(),
246 V: 4 * (DL.getTypeSizeInBits(Ty: EltTy) / 8));
247
248 SmallVector<int, 4> Indices;
249 for (unsigned int I = 0, N = VT->getNumElements(); I < N; I += 4) {
250 if (I > 0)
251 Offset = Builder.CreateAdd(LHS: Offset, RHS: Stride);
252
253 for (unsigned int J = I, E = std::min(a: N, b: J + 4); J < E; ++J)
254 Indices.push_back(Elt: J);
255 Value *Part = Builder.CreateShuffleVector(V, Mask: Indices);
256 emitRawStore(Builder, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, V: Part, RTI);
257
258 Indices.clear();
259 }
260 } else
261 emitRawStore(Builder, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, V, RTI);
262}
263
264static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI,
265 dxil::ResourceTypeInfo &RTI) {
266 switch (RTI.getResourceKind()) {
267 case dxil::ResourceKind::TypedBuffer:
268 return createTypedBufferStore(II, SI, RTI);
269 case dxil::ResourceKind::RawBuffer:
270 case dxil::ResourceKind::StructuredBuffer:
271 return createRawStores(II, SI, RTI);
272 case dxil::ResourceKind::Texture1D:
273 case dxil::ResourceKind::Texture2D:
274 case dxil::ResourceKind::Texture3D:
275 case dxil::ResourceKind::Texture1DArray:
276 case dxil::ResourceKind::Texture2DArray:
277 return createTextureStore(II, SI, RTI);
278 case dxil::ResourceKind::Texture2DMS:
279 case dxil::ResourceKind::Texture2DMSArray:
280 case dxil::ResourceKind::TextureCube:
281 case dxil::ResourceKind::TextureCubeArray:
282 case dxil::ResourceKind::FeedbackTexture2D:
283 case dxil::ResourceKind::FeedbackTexture2DArray:
284 reportFatalUsageError(
285 reason: "DXIL Store not implemented for this texture resource kind");
286 return;
287 case dxil::ResourceKind::CBuffer:
288 case dxil::ResourceKind::Sampler:
289 case dxil::ResourceKind::TBuffer:
290 case dxil::ResourceKind::RTAccelerationStructure:
291 case dxil::ResourceKind::Invalid:
292 case dxil::ResourceKind::NumEntries:
293 llvm_unreachable("Invalid resource kind for store");
294 }
295 llvm_unreachable("Unhandled case in switch");
296}
297
298static std::optional<dxil::AtomicBinOpCode>
299getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) {
300 switch (BinOp) {
301 case AtomicRMWInst::Add:
302 return dxil::AtomicBinOpCode::Add;
303 case AtomicRMWInst::And:
304 return dxil::AtomicBinOpCode::And;
305 case AtomicRMWInst::Or:
306 return dxil::AtomicBinOpCode::Or;
307 case AtomicRMWInst::Xor:
308 return dxil::AtomicBinOpCode::Xor;
309 case AtomicRMWInst::Min:
310 return dxil::AtomicBinOpCode::IMin;
311 case AtomicRMWInst::Max:
312 return dxil::AtomicBinOpCode::IMax;
313 case AtomicRMWInst::UMin:
314 return dxil::AtomicBinOpCode::UMin;
315 case AtomicRMWInst::UMax:
316 return dxil::AtomicBinOpCode::UMax;
317 case AtomicRMWInst::Xchg:
318 return dxil::AtomicBinOpCode::Exchange;
319 case AtomicRMWInst::Sub:
320 case AtomicRMWInst::Nand:
321 case AtomicRMWInst::FAdd:
322 case AtomicRMWInst::FSub:
323 case AtomicRMWInst::FMax:
324 case AtomicRMWInst::FMin:
325 case AtomicRMWInst::FMaximum:
326 case AtomicRMWInst::FMinimum:
327 case AtomicRMWInst::FMaximumNum:
328 case AtomicRMWInst::FMinimumNum:
329 case AtomicRMWInst::UIncWrap:
330 case AtomicRMWInst::UDecWrap:
331 case AtomicRMWInst::USubCond:
332 case AtomicRMWInst::USubSat:
333 case AtomicRMWInst::BAD_BINOP:
334 return std::nullopt;
335 }
336 llvm_unreachable("Unhandled atomicrmw operation");
337}
338
339// Compute the (coord0, coord1) pair for a buffer resource atomic operation.
340// Non-struct buffers (RawBuffer or TypedBuffer) fold the byte offset into the
341// index and leave coord1 poison. Only StructuredBuffer atomics use both a
342// struct index and a byte offset.
343static std::pair<Value *, Value *>
344getAtomicResourceCoords(IntrinsicInst *II, Value *PointerOperand,
345 dxil::ResourceTypeInfo &RTI, IRBuilder<> &Builder,
346 const DataLayout &DL) {
347 Value *Index = II->getOperand(i_nocapture: 1);
348
349 // The offset for the rawbuffer load/store/atomic ops is always in bytes.
350 uint64_t AccessSize = 1;
351 Value *Offset = traverseGEPOffsets(DL, Builder, Ptr: PointerOperand, AccessSize);
352
353 if (!RTI.isStruct()) {
354 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
355 if (!ConstantOffset || !ConstantOffset->isZero())
356 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
357 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
358 }
359
360 return {Index, Offset};
361}
362
363// The coordinates of a texture access are a scalar or a vector with one element
364// per texture dimension, including the array slice if there is one. These map
365// directly onto the coordinate operands of the atomic ops.
366static SmallVector<Value *, 3> getTextureAtomicCoords(IntrinsicInst *II,
367 IRBuilder<> &Builder) {
368 Value *Coords = II->getOperand(i_nocapture: 1);
369 SmallVector<Value *, 3> CoordArgs;
370 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Coords->getType())) {
371 assert(VecTy->getNumElements() <= 3 && "Too many texture coordinates");
372 for (unsigned I = 0, E = VecTy->getNumElements(); I != E; ++I)
373 CoordArgs.push_back(Elt: Builder.CreateExtractElement(Vec: Coords, Idx: I));
374 } else {
375 CoordArgs.push_back(Elt: Coords);
376 }
377 return CoordArgs;
378}
379
380static void emitAtomicBinOp(IRBuilder<> &Builder, AtomicRMWInst *AI,
381 Value *Handle, ArrayRef<Value *> Coords) {
382 assert(!Coords.empty() && Coords.size() <= 3 &&
383 "Atomic operations take between one and three coordinates");
384
385 std::optional<dxil::AtomicBinOpCode> BinOpCode =
386 getAtomicBinOpCode(BinOp: AI->getOperation());
387 if (!BinOpCode) {
388 reportFatalUsageError(reason: "DXIL resource atomicrmw operation not implemented");
389 return;
390 }
391
392 // DXIL has no floating-point atomic op. A float exchange only moves the bit
393 // pattern, so cast the value to an integer of the same width, exchange, and
394 // cast the result back. This matches what DXC emits.
395 Value *Val = AI->getValOperand();
396 Type *ValTy = Val->getType();
397 Type *OpTy = ValTy;
398 if (ValTy->isFloatingPointTy()) {
399 OpTy = Builder.getIntNTy(N: ValTy->getPrimitiveSizeInBits());
400 Val = Builder.CreateBitCast(V: Val, DestTy: OpTy);
401 }
402
403 SmallVector<Value *, 6> Args{
404 Handle, Builder.getInt32(C: static_cast<uint32_t>(*BinOpCode))};
405 append_range(C&: Args, R&: Coords);
406 Args.append(NumInputs: 3 - Coords.size(), Elt: PoisonValue::get(T: Builder.getInt32Ty()));
407 Args.push_back(Elt: Val);
408
409 // Emit the target-independent intrinsic; DXILOpLowering lowers it to the
410 // DXIL `AtomicBinOp` op and handles the target-ext-typed handle cast via
411 // its `createTmpHandleCast` bookkeeping.
412 Value *Result =
413 Builder.CreateIntrinsic(RetTy: OpTy, ID: Intrinsic::dx_resource_atomic_binop, Args);
414
415 if (OpTy != ValTy)
416 Result = Builder.CreateBitCast(V: Result, DestTy: ValTy);
417
418 AI->replaceAllUsesWith(V: Result);
419}
420
421static void createBufferAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
422 dxil::ResourceTypeInfo &RTI) {
423 const DataLayout &DL = AI->getDataLayout();
424 IRBuilder<> Builder(AI);
425 auto [Index, Offset] =
426 getAtomicResourceCoords(II, PointerOperand: AI->getPointerOperand(), RTI, Builder, DL);
427
428 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: {Index, Offset});
429}
430
431static void createTextureAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
432 dxil::ResourceTypeInfo &RTI) {
433 // A texture atomic operates on a whole texel, so a multi-component texel has
434 // no single addressable component. A scalar float texel is allowed, because
435 // emitAtomicBinOp exchanges its bit pattern as an integer.
436 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
437 if (!ContainedType->isIntegerTy() && !ContainedType->isFloatingPointTy()) {
438 reportFatalUsageError(reason: "DXIL atomicrmw requires a texture resource with a "
439 "scalar element type");
440 return;
441 }
442
443 IRBuilder<> Builder(AI);
444
445 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0),
446 Coords: getTextureAtomicCoords(II, Builder));
447}
448
449static void emitAtomicCompareExchange(IRBuilder<> &Builder,
450 AtomicCmpXchgInst *AI, Value *Handle,
451 ArrayRef<Value *> Coords) {
452 assert(!Coords.empty() && Coords.size() <= 3 &&
453 "Atomic operations take between one and three coordinates");
454
455 Value *Compare = AI->getCompareOperand();
456 Value *NewValue = AI->getNewValOperand();
457
458 SmallVector<Value *, 6> Args{Handle};
459 append_range(C&: Args, R&: Coords);
460 Args.append(NumInputs: 3 - Coords.size(), Elt: PoisonValue::get(T: Builder.getInt32Ty()));
461 Args.push_back(Elt: Compare);
462 Args.push_back(Elt: NewValue);
463
464 Value *Original = Builder.CreateIntrinsic(
465 RetTy: NewValue->getType(), ID: Intrinsic::dx_resource_atomic_compare_exchange,
466 Args);
467
468 // `cmpxchg` yields a { original, success } pair, but the DXIL op returns
469 // only the original value. DXIL has no way to express the success flag, and
470 // no HLSL builtin reads it, so replace the users of the pair directly
471 // instead of building it again. No pass after this one removes dead code.
472 SmallVector<ExtractValueInst *> Extracts;
473 for (User *U : AI->users()) {
474 auto *EV = dyn_cast<ExtractValueInst>(Val: U);
475 if (!EV || EV->getIndices()[0] != 0)
476 reportFatalUsageError(reason: "DXIL cmpxchg provides only the original value");
477 Extracts.push_back(Elt: EV);
478 }
479
480 for (ExtractValueInst *EV : Extracts) {
481 EV->replaceAllUsesWith(V: Original);
482 EV->eraseFromParent();
483 }
484}
485
486static void createBufferAtomicCompareExchange(IntrinsicInst *II,
487 AtomicCmpXchgInst *AI,
488 dxil::ResourceTypeInfo &RTI) {
489 const DataLayout &DL = AI->getDataLayout();
490 IRBuilder<> Builder(AI);
491 auto [Index, Offset] =
492 getAtomicResourceCoords(II, PointerOperand: AI->getPointerOperand(), RTI, Builder, DL);
493
494 emitAtomicCompareExchange(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: {Index, Offset});
495}
496
497// `cmpxchg` operands are always integers, so unlike atomicrmw there is no
498// float element type to convert here. The element type must be scalar.
499static void createTextureAtomicCompareExchange(IntrinsicInst *II,
500 AtomicCmpXchgInst *AI,
501 dxil::ResourceTypeInfo &RTI) {
502 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
503 if (!ContainedType->isIntegerTy() && !ContainedType->isFloatingPointTy()) {
504 reportFatalUsageError(reason: "DXIL cmpxchg requires a texture resource with a "
505 "scalar element type");
506 return;
507 }
508
509 IRBuilder<> Builder(AI);
510
511 emitAtomicCompareExchange(Builder, AI, Handle: II->getOperand(i_nocapture: 0),
512 Coords: getTextureAtomicCoords(II, Builder));
513}
514
515static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
516 dxil::ResourceTypeInfo &RTI) {
517 switch (RTI.getResourceKind()) {
518 case dxil::ResourceKind::TypedBuffer:
519 case dxil::ResourceKind::RawBuffer:
520 case dxil::ResourceKind::StructuredBuffer:
521 return createBufferAtomicBinOp(II, AI, RTI);
522 case dxil::ResourceKind::Texture1D:
523 case dxil::ResourceKind::Texture2D:
524 case dxil::ResourceKind::Texture3D:
525 case dxil::ResourceKind::Texture1DArray:
526 case dxil::ResourceKind::Texture2DArray:
527 return createTextureAtomicBinOp(II, AI, RTI);
528 case dxil::ResourceKind::Texture2DMS:
529 case dxil::ResourceKind::Texture2DMSArray:
530 case dxil::ResourceKind::TextureCube:
531 case dxil::ResourceKind::TextureCubeArray:
532 case dxil::ResourceKind::FeedbackTexture2D:
533 case dxil::ResourceKind::FeedbackTexture2DArray:
534 reportFatalUsageError(
535 reason: "DXIL atomicrmw not implemented for this texture resource kind");
536 return;
537 case dxil::ResourceKind::CBuffer:
538 case dxil::ResourceKind::Sampler:
539 case dxil::ResourceKind::TBuffer:
540 reportFatalUsageError(
541 reason: "DXIL atomicrmw not implemented for this resource type");
542 return;
543 case dxil::ResourceKind::RTAccelerationStructure:
544 case dxil::ResourceKind::Invalid:
545 case dxil::ResourceKind::NumEntries:
546 llvm_unreachable("Invalid resource kind for atomicrmw");
547 }
548 llvm_unreachable("Unhandled case in switch");
549}
550
551static void createAtomicCompareExchangeIntrinsic(IntrinsicInst *II,
552 AtomicCmpXchgInst *AI,
553 dxil::ResourceTypeInfo &RTI) {
554 switch (RTI.getResourceKind()) {
555 case dxil::ResourceKind::TypedBuffer:
556 case dxil::ResourceKind::RawBuffer:
557 case dxil::ResourceKind::StructuredBuffer:
558 return createBufferAtomicCompareExchange(II, AI, RTI);
559 case dxil::ResourceKind::Texture1D:
560 case dxil::ResourceKind::Texture2D:
561 case dxil::ResourceKind::Texture3D:
562 case dxil::ResourceKind::Texture1DArray:
563 case dxil::ResourceKind::Texture2DArray:
564 return createTextureAtomicCompareExchange(II, AI, RTI);
565 case dxil::ResourceKind::Texture2DMS:
566 case dxil::ResourceKind::Texture2DMSArray:
567 case dxil::ResourceKind::TextureCube:
568 case dxil::ResourceKind::TextureCubeArray:
569 case dxil::ResourceKind::FeedbackTexture2D:
570 case dxil::ResourceKind::FeedbackTexture2DArray:
571 reportFatalUsageError(
572 reason: "DXIL cmpxchg not implemented for this texture resource kind");
573 return;
574 case dxil::ResourceKind::CBuffer:
575 case dxil::ResourceKind::Sampler:
576 case dxil::ResourceKind::TBuffer:
577 reportFatalUsageError(
578 reason: "DXIL cmpxchg not implemented for this resource type");
579 return;
580 case dxil::ResourceKind::RTAccelerationStructure:
581 case dxil::ResourceKind::Invalid:
582 case dxil::ResourceKind::NumEntries:
583 llvm_unreachable("Invalid resource kind for cmpxchg");
584 }
585 llvm_unreachable("Unhandled case in switch");
586}
587
588static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI,
589 dxil::ResourceTypeInfo &RTI) {
590 const DataLayout &DL = LI->getDataLayout();
591 IRBuilder<> Builder(LI);
592 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
593 Type *LoadType = StructType::get(elt1: ContainedType, elts: Builder.getInt1Ty());
594
595 Value *V =
596 Builder.CreateIntrinsic(RetTy: LoadType, ID: Intrinsic::dx_resource_load_typedbuffer,
597 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1)});
598 V = Builder.CreateExtractValue(Agg: V, Idxs: {0});
599
600 Type *ScalarType = ContainedType->getScalarType();
601 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
602 Value *Offset =
603 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
604 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
605 if (!ConstantOffset || !ConstantOffset->isZero())
606 V = Builder.CreateExtractElement(Vec: V, Idx: Offset);
607
608 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a
609 // shufflevector), then make sure we're maintaining the resulting type.
610 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType()))
611 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(Val: V->getType()))
612 V = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: V,
613 Idx: Builder.getInt32(C: 0));
614
615 LI->replaceAllUsesWith(V);
616}
617
618static void createTextureLoad(IntrinsicInst *II, LoadInst *LI,
619 dxil::ResourceTypeInfo &RTI) {
620 const DataLayout &DL = LI->getDataLayout();
621 IRBuilder<> Builder(LI);
622 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
623
624 Value *Handle = II->getOperand(i_nocapture: 0);
625 Value *Coords = II->getOperand(i_nocapture: 1);
626
627 // For operator[], mip level is 0.
628 Value *MipLevel = Builder.getInt32(C: 0);
629
630 // For operator[], offsets are zero.
631 Value *Offsets = getNullOffsetsFor(Builder, Coords);
632
633 Value *V =
634 Builder.CreateIntrinsic(RetTy: ContainedType, ID: Intrinsic::dx_resource_load_level,
635 Args: {Handle, Coords, MipLevel, Offsets});
636
637 Type *ScalarType = ContainedType->getScalarType();
638 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
639 Value *Offset =
640 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
641 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
642 if (!ConstantOffset || !ConstantOffset->isZero())
643 V = Builder.CreateExtractElement(Vec: V, Idx: Offset);
644
645 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a
646 // shufflevector), then make sure we're maintaining the resulting type.
647 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType()))
648 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(Val: V->getType()))
649 V = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: V,
650 Idx: Builder.getInt32(C: 0));
651
652 LI->replaceAllUsesWith(V);
653}
654
655static Value *emitRawLoad(IRBuilder<> &Builder, Type *Ty, Value *Buffer,
656 Value *Index, Value *Offset,
657 dxil::ResourceTypeInfo &RTI) {
658 // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
659 // entirely into the index.
660 if (!RTI.isStruct()) {
661 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
662 if (!ConstantOffset || !ConstantOffset->isZero())
663 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
664 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
665 }
666
667 // The load intrinsic includes the bit for CheckAccessFullyMapped, so we need
668 // to add that to the return type.
669 Type *TypeWithCheck = StructType::get(elt1: Ty, elts: Builder.getInt1Ty());
670 Value *V = Builder.CreateIntrinsic(RetTy: TypeWithCheck,
671 ID: Intrinsic::dx_resource_load_rawbuffer,
672 Args: {Buffer, Index, Offset});
673 return Builder.CreateExtractValue(Agg: V, Idxs: {0});
674}
675
676static void createRawLoads(IntrinsicInst *II, LoadInst *LI,
677 dxil::ResourceTypeInfo &RTI) {
678 const DataLayout &DL = LI->getDataLayout();
679 IRBuilder<> Builder(LI);
680
681 Value *Index = II->getOperand(i_nocapture: 1);
682 // The offset for the rawbuffer load and store ops is always in bytes.
683 uint64_t AccessSize = 1;
684 Value *Offset =
685 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
686
687 // TODO: We could make this handle aggregates by walking the structure and
688 // handling each field individually, but we don't ever generate code that
689 // would hit that so it seems superfluous.
690 assert(!LI->getType()->isAggregateType() &&
691 "Resource load should be scalar or vector type");
692
693 Value *V;
694 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType())) {
695 // Split into loads of at most 4 elements.
696 Type *EltTy = VT->getElementType();
697 Value *Stride = ConstantInt::get(Ty: Builder.getInt32Ty(),
698 V: 4 * (DL.getTypeSizeInBits(Ty: EltTy) / 8));
699
700 SmallVector<Value *> Parts;
701 for (unsigned int I = 0, N = VT->getNumElements(); I < N; I += 4) {
702 Type *Ty = FixedVectorType::get(ElementType: EltTy, NumElts: N - I < 4 ? N - I : 4);
703 if (I > 0)
704 Offset = Builder.CreateAdd(LHS: Offset, RHS: Stride);
705 Parts.push_back(
706 Elt: emitRawLoad(Builder, Ty, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, RTI));
707 }
708
709 V = Parts.size() > 1 ? concatenateVectors(Builder, Vecs: Parts) : Parts[0];
710 } else
711 V = emitRawLoad(Builder, Ty: LI->getType(), Buffer: II->getOperand(i_nocapture: 0), Index, Offset,
712 RTI);
713
714 LI->replaceAllUsesWith(V);
715}
716
717namespace {
718/// Helper for building a `load.cbufferrow` intrinsic given a simple type.
719struct CBufferRowIntrin {
720 Intrinsic::ID IID;
721 Type *RetTy;
722 unsigned int EltSize;
723 unsigned int NumElts;
724
725 CBufferRowIntrin(const DataLayout &DL, Type *Ty) {
726 assert(Ty == Ty->getScalarType() && "Expected scalar type");
727
728 switch (DL.getTypeSizeInBits(Ty)) {
729 case 16:
730 IID = Intrinsic::dx_resource_load_cbufferrow_8;
731 RetTy = StructType::get(elt1: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty);
732 EltSize = 2;
733 NumElts = 8;
734 break;
735 case 32:
736 IID = Intrinsic::dx_resource_load_cbufferrow_4;
737 RetTy = StructType::get(elt1: Ty, elts: Ty, elts: Ty, elts: Ty);
738 EltSize = 4;
739 NumElts = 4;
740 break;
741 case 64:
742 IID = Intrinsic::dx_resource_load_cbufferrow_2;
743 RetTy = StructType::get(elt1: Ty, elts: Ty);
744 EltSize = 8;
745 NumElts = 2;
746 break;
747 default:
748 llvm_unreachable("Only 16, 32, and 64 bit types supported");
749 }
750 }
751};
752} // namespace
753
754static Value *loadCBufferRowIntoWindow(IRBuilder<> &Builder,
755 const CBufferRowIntrin &Intrin,
756 Value *Handle, Value *FirstRow,
757 Value *Window, unsigned RowNumber,
758 StringRef Name) {
759 Value *RowIndex =
760 RowNumber == 0 ? FirstRow
761 : Builder.CreateAdd(LHS: FirstRow, RHS: Builder.getInt32(C: RowNumber));
762 Value *RowLoad = Builder.CreateIntrinsic(
763 RetTy: Intrin.RetTy, ID: Intrin.IID, Args: {Handle, RowIndex}, FMFSource: nullptr, Name: Name + ".load");
764 for (unsigned J = 0; J != Intrin.NumElts; ++J) {
765 Value *Element = Builder.CreateExtractValue(Agg: RowLoad, Idxs: {J});
766 Window = Builder.CreateInsertElement(
767 Vec: Window, NewElt: Element, Idx: Builder.getInt32(C: RowNumber * Intrin.NumElts + J));
768 }
769 return Window;
770}
771
772static Value *createDynamicCBufferLoad(IntrinsicInst *II, LoadInst *LI,
773 const CBufferRowIntrin &Intrin,
774 Value *BaseRow, Value *ByteOffset,
775 const KnownBits &WithinRow) {
776 IRBuilder<> Builder(LI);
777 Type *Ty = LI->getType();
778 auto *VT = dyn_cast<FixedVectorType>(Val: Ty);
779 unsigned NumElements = VT ? VT->getNumElements() : 1;
780 StringRef Name = LI->getName();
781 Value *Handle = II->getOperand(i_nocapture: 0);
782 Value *RowSize = Builder.getInt32(C: hlsl::CBufferRowSizeInBytes);
783 Value *FirstRow =
784 Builder.CreateAdd(LHS: BaseRow, RHS: Builder.CreateUDiv(LHS: ByteOffset, RHS: RowSize));
785 Value *Component = Builder.CreateUDiv(LHS: Builder.CreateURem(LHS: ByteOffset, RHS: RowSize),
786 RHS: Builder.getInt32(C: Intrin.EltSize));
787 unsigned MinComponent =
788 WithinRow.getMinValue().getZExtValue() / Intrin.EltSize;
789 unsigned MaxComponent =
790 WithinRow.getMaxValue().getZExtValue() / Intrin.EltSize;
791 unsigned MinRows = 1 + (MinComponent + NumElements - 1) / Intrin.NumElts;
792 unsigned MaxRows = 1 + (MaxComponent + NumElements - 1) / Intrin.NumElts;
793 auto *WindowTy =
794 FixedVectorType::get(ElementType: Ty->getScalarType(), NumElts: MaxRows * Intrin.NumElts);
795 Value *Window = PoisonValue::get(T: WindowTy);
796 for (unsigned I = 0; I != MinRows; ++I)
797 Window = loadCBufferRowIntoWindow(Builder, Intrin, Handle, FirstRow, Window,
798 RowNumber: I, Name);
799
800 // The unknown starting component can require at most one extra row.
801 // Load it only when needed, rather than accessing beyond the load.
802 if (MaxRows != MinRows) {
803 BasicBlock *Head = LI->getParent();
804 Value *LastComponent =
805 Builder.CreateAdd(LHS: Component, RHS: Builder.getInt32(C: NumElements - 1));
806 Value *CrossesRow = Builder.CreateICmpUGE(
807 LHS: LastComponent, RHS: Builder.getInt32(C: MinRows * Intrin.NumElts));
808 Instruction *ThenTerm = SplitBlockAndInsertIfThen(Cond: CrossesRow, SplitBefore: LI, Unreachable: false);
809 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *Head->getTerminator(),
810 DEBUG_TYPE);
811 Builder.SetInsertPoint(ThenTerm);
812 Value *NextWindow = loadCBufferRowIntoWindow(
813 Builder, Intrin, Handle, FirstRow, Window, RowNumber: MinRows, Name);
814 Builder.SetInsertPoint(LI);
815 auto *Rows = Builder.CreatePHI(Ty: WindowTy, NumReservedValues: 2, Name: Name + ".rows");
816 Rows->addIncoming(V: Window, BB: Head);
817 Rows->addIncoming(V: NextWindow, BB: ThenTerm->getParent());
818 Window = Rows;
819 }
820
821 Value *Result = PoisonValue::get(T: Ty);
822 for (unsigned I = 0; I != NumElements; ++I) {
823 Value *Index =
824 I == 0 ? Component : Builder.CreateAdd(LHS: Component, RHS: Builder.getInt32(C: I));
825 Value *Element =
826 Builder.CreateExtractElement(Vec: Window, Idx: Index, Name: Name + ".extract");
827 Result =
828 VT ? Builder.CreateInsertElement(Vec: Result, NewElt: Element, Idx: Builder.getInt32(C: I))
829 : Element;
830 }
831 return Result;
832}
833
834static KnownBits computeCBufferOffsetKnownBits(Value *Offset,
835 const DataLayout &DL,
836 const Instruction *CxtI) {
837 KnownBits Known = computeKnownBits(V: Offset, DL, AC: nullptr, CtxI: CxtI);
838 auto *Phi = dyn_cast<PHINode>(Val: Offset);
839 if (!Phi || Phi->getNumIncomingValues() == 0)
840 return Known;
841
842 // PHI queries limit recursion into each input, which can hide row alignment.
843 KnownBits IncomingKnown(Known.getBitWidth());
844 IncomingKnown.setAllConflict();
845 for (unsigned I = 0; I != Phi->getNumIncomingValues(); ++I)
846 IncomingKnown = IncomingKnown.intersectWith(
847 RHS: computeKnownBits(V: Phi->getIncomingValue(i: I), DL, AC: nullptr,
848 CtxI: Phi->getIncomingBlock(i: I)->getTerminator()));
849 return Known.unionWith(RHS: IncomingKnown);
850}
851
852static void createCBufferLoad(IntrinsicInst *II, LoadInst *LI,
853 dxil::ResourceTypeInfo &RTI) {
854 const DataLayout &DL = LI->getDataLayout();
855
856 Type *Ty = LI->getType();
857 assert(!isa<StructType>(Ty) && "Structs not handled yet");
858 CBufferRowIntrin Intrin(DL, Ty->getScalarType());
859
860 StringRef Name = LI->getName();
861 Value *Handle = II->getOperand(i_nocapture: 0);
862
863 IRBuilder<> Builder(LI);
864
865 ConstantInt *GlobalOffset =
866 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer
867 ? ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)
868 : dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1));
869 assert(GlobalOffset && "CBuffer getpointer index must be constant");
870
871 uint64_t GlobalOffsetVal = GlobalOffset->getZExtValue();
872 Value *CurrentRow = ConstantInt::get(
873 Ty: Builder.getInt32Ty(), V: GlobalOffsetVal / hlsl::CBufferRowSizeInBytes);
874 unsigned int CurrentIndex =
875 (GlobalOffsetVal % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
876
877 // Every object in a cbuffer either fits in a row or is aligned to a row. This
878 // means that only the very last pointer access can point into a row.
879 auto *LastGEP = dyn_cast<GEPOperator>(Val: LI->getPointerOperand());
880 if (!LastGEP) {
881 // If we don't have a GEP at all we're just accessing the resource through
882 // the result of getpointer directly.
883 assert(LI->getPointerOperand() == II &&
884 "Unexpected indirect access to resource without GEP");
885 } else {
886 Value *GEPOffset = traverseGEPOffsets(
887 DL, Builder, Ptr: LastGEP->getPointerOperand(), AccessSize: hlsl::CBufferRowSizeInBytes);
888 CurrentRow = Builder.CreateAdd(LHS: GEPOffset, RHS: CurrentRow);
889
890 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: LastGEP->getType()), 0);
891 if (LastGEP->accumulateConstantOffset(DL, Offset&: ConstantOffset)) {
892 APInt Remainder(DL.getIndexTypeSizeInBits(Ty: LastGEP->getType()),
893 hlsl::CBufferRowSizeInBytes);
894 APInt::udivrem(LHS: ConstantOffset, RHS: Remainder, Quotient&: ConstantOffset, Remainder);
895 CurrentRow = Builder.CreateAdd(
896 LHS: CurrentRow, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: ConstantOffset));
897 CurrentIndex += Remainder.udiv(RHS: Intrin.EltSize).getZExtValue();
898 } else {
899 assert(LastGEP->getNumIndices() == 1 &&
900 "Last GEP of cbuffer access is not array or struct access");
901 // Aggregate strides are row-aligned, but merged byte offsets can select
902 // different components within a row.
903 Value *RowIndex = Builder.CreateSExtOrTrunc(V: *LastGEP->idx_begin(),
904 DestTy: Builder.getInt32Ty());
905 uint64_t IndexScale =
906 DL.getTypeAllocSize(Ty: LastGEP->getSourceElementType());
907 if (IndexScale < hlsl::CBufferRowSizeInBytes) {
908 assert(hlsl::CBufferRowSizeInBytes % IndexScale == 0 &&
909 "CBuffer GEP index is not row aligned");
910 Value *ByteOffset =
911 IndexScale == 1
912 ? RowIndex
913 : Builder.CreateMul(LHS: RowIndex, RHS: Builder.getInt32(C: IndexScale));
914 uint64_t BaseWithinRow = GlobalOffsetVal % hlsl::CBufferRowSizeInBytes;
915 if (BaseWithinRow != 0)
916 ByteOffset =
917 Builder.CreateAdd(LHS: ByteOffset, RHS: Builder.getInt32(C: BaseWithinRow));
918 Value *RowSize = Builder.getInt32(C: hlsl::CBufferRowSizeInBytes);
919 KnownBits WithinRow =
920 computeCBufferOffsetKnownBits(Offset: ByteOffset, DL, CxtI: LI) &
921 KnownBits::makeConstant(C: APInt(32, hlsl::CBufferRowSizeInBytes - 1));
922 if (!WithinRow.isConstant()) {
923 LI->replaceAllUsesWith(V: createDynamicCBufferLoad(
924 II, LI, Intrin, BaseRow: CurrentRow, ByteOffset, WithinRow));
925 return;
926 }
927 RowIndex = Builder.CreateUDiv(LHS: ByteOffset, RHS: RowSize, Name: "",
928 isExact: WithinRow.getConstant().isZero());
929 CurrentIndex = WithinRow.getConstant().getZExtValue() / Intrin.EltSize;
930 } else if (IndexScale > hlsl::CBufferRowSizeInBytes) {
931 assert(IndexScale % hlsl::CBufferRowSizeInBytes == 0 &&
932 "CBuffer GEP index is not row aligned");
933 RowIndex = Builder.CreateMul(
934 LHS: RowIndex,
935 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(),
936 V: IndexScale / hlsl::CBufferRowSizeInBytes));
937 }
938 if (auto *BaseRow = dyn_cast<ConstantInt>(Val: CurrentRow);
939 BaseRow && BaseRow->isZero())
940 CurrentRow = RowIndex;
941 else
942 CurrentRow = Builder.CreateAdd(LHS: CurrentRow, RHS: RowIndex);
943 }
944 }
945
946 auto *CBufLoad = Builder.CreateIntrinsic(
947 RetTy: Intrin.RetTy, ID: Intrin.IID, Args: {Handle, CurrentRow}, FMFSource: nullptr, Name: Name + ".load");
948 auto *Elt =
949 Builder.CreateExtractValue(Agg: CBufLoad, Idxs: {CurrentIndex++}, Name: Name + ".extract");
950
951 // At this point we've loaded the first scalar of our result, but our original
952 // type may have been a vector.
953 unsigned int Remaining =
954 ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
955 if (Remaining == 0) {
956 // We only have a single element, so we're done.
957 Value *Result = Elt;
958
959 // However, if we loaded a <1 x T>, then we need to adjust the type.
960 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
961 assert(VT->getNumElements() == 1 && "Can't have multiple elements here");
962 Result = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: Result,
963 Idx: Builder.getInt32(C: 0), Name);
964 }
965 LI->replaceAllUsesWith(V: Result);
966 return;
967 }
968
969 // Walk each element and extract it, wrapping to new rows as needed.
970 SmallVector<Value *> Extracts{Elt};
971 while (Remaining--) {
972 CurrentIndex %= Intrin.NumElts;
973
974 if (CurrentIndex == 0) {
975 CurrentRow = Builder.CreateAdd(LHS: CurrentRow,
976 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
977 CBufLoad = Builder.CreateIntrinsic(RetTy: Intrin.RetTy, ID: Intrin.IID,
978 Args: {Handle, CurrentRow}, FMFSource: nullptr,
979 Name: Name + ".load");
980 }
981
982 Extracts.push_back(Elt: Builder.CreateExtractValue(Agg: CBufLoad, Idxs: {CurrentIndex++},
983 Name: Name + ".extract"));
984 }
985
986 // Finally, we build up the original loaded value.
987 Value *Result = PoisonValue::get(T: Ty);
988 for (int I = 0, E = Extracts.size(); I < E; ++I)
989 Result = Builder.CreateInsertElement(
990 Vec: Result, NewElt: Extracts[I], Idx: Builder.getInt32(C: I), Name: Name + formatv(Fmt: ".upto{}", Vals&: I));
991 LI->replaceAllUsesWith(V: Result);
992}
993
994static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI,
995 dxil::ResourceTypeInfo &RTI) {
996 switch (RTI.getResourceKind()) {
997 case dxil::ResourceKind::TypedBuffer:
998 return createTypedBufferLoad(II, LI, RTI);
999 case dxil::ResourceKind::RawBuffer:
1000 case dxil::ResourceKind::StructuredBuffer:
1001 return createRawLoads(II, LI, RTI);
1002 case dxil::ResourceKind::CBuffer:
1003 return createCBufferLoad(II, LI, RTI);
1004 case dxil::ResourceKind::Texture1D:
1005 case dxil::ResourceKind::Texture2D:
1006 case dxil::ResourceKind::Texture2DMS:
1007 case dxil::ResourceKind::Texture3D:
1008 case dxil::ResourceKind::TextureCube:
1009 case dxil::ResourceKind::Texture1DArray:
1010 case dxil::ResourceKind::Texture2DArray:
1011 case dxil::ResourceKind::Texture2DMSArray:
1012 case dxil::ResourceKind::TextureCubeArray:
1013 return createTextureLoad(II, LI, RTI);
1014 case dxil::ResourceKind::FeedbackTexture2D:
1015 case dxil::ResourceKind::FeedbackTexture2DArray:
1016 case dxil::ResourceKind::TBuffer:
1017 reportFatalUsageError(reason: "Load not yet implemented for resource type");
1018 return;
1019 case dxil::ResourceKind::Sampler:
1020 case dxil::ResourceKind::RTAccelerationStructure:
1021 case dxil::ResourceKind::Invalid:
1022 case dxil::ResourceKind::NumEntries:
1023 llvm_unreachable("Invalid resource kind for load");
1024 }
1025 llvm_unreachable("Unhandled case in switch");
1026}
1027
1028static Instruction *getHandleOperand(Instruction *AI) {
1029 if (auto *LI = dyn_cast<LoadInst>(Val: AI))
1030 return dyn_cast<Instruction>(Val: LI->getPointerOperand());
1031 if (auto *SI = dyn_cast<StoreInst>(Val: AI))
1032 return dyn_cast<Instruction>(Val: SI->getPointerOperand());
1033 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: AI))
1034 return dyn_cast<Instruction>(Val: RMWI->getPointerOperand());
1035 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: AI))
1036 return dyn_cast<Instruction>(Val: CXI->getPointerOperand());
1037 if (auto *II = dyn_cast<IntrinsicInst>(Val: AI))
1038 if (II->getIntrinsicID() == Intrinsic::dx_resource_updatecounter)
1039 return dyn_cast<Instruction>(Val: II->getArgOperand(i: 0));
1040
1041 return nullptr;
1042}
1043
1044static Instruction *getHandleRoot(Instruction *I) {
1045 while (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1046 auto *PointerI = dyn_cast<Instruction>(Val: GEP->getPointerOperand());
1047 if (!PointerI)
1048 break;
1049 I = PointerI;
1050 }
1051 return I;
1052}
1053
1054static const std::array<Intrinsic::ID, 2> HandleIntrins = {
1055 Intrinsic::dx_resource_handlefrombinding,
1056 Intrinsic::dx_resource_handlefromimplicitbinding,
1057};
1058
1059static SmallVector<IntrinsicInst *> collectUsedHandles(Value *Ptr) {
1060 SmallVector<Value *> Worklist = {Ptr};
1061 SmallVector<IntrinsicInst *> Handles;
1062 SmallPtrSet<Value *, 4> VisitedPhis;
1063
1064 while (!Worklist.empty()) {
1065 Value *X = Worklist.pop_back_val();
1066
1067 if (!X->getType()->isPointerTy() && !X->getType()->isTargetExtTy())
1068 return {}; // Early exit on store/load into non-resource
1069
1070 if (auto *Phi = dyn_cast<PHINode>(Val: X)) {
1071 if (VisitedPhis.contains(Ptr: X))
1072 continue;
1073 for (Use &V : Phi->incoming_values())
1074 Worklist.push_back(Elt: V.get());
1075 VisitedPhis.insert(Ptr: Phi);
1076 } else if (auto *Select = dyn_cast<SelectInst>(Val: X)) {
1077 Worklist.push_back(Elt: Select->getTrueValue());
1078 Worklist.push_back(Elt: Select->getFalseValue());
1079 } else if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: X)) {
1080 Worklist.push_back(Elt: GEP->getPointerOperand());
1081 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: X)) {
1082 Intrinsic::ID IID = II->getIntrinsicID();
1083
1084 if (IID == Intrinsic::dx_resource_getpointer ||
1085 IID == Intrinsic::dx_resource_getbasepointer)
1086 Worklist.push_back(Elt: II->getArgOperand(/*Handle=*/i: 0));
1087
1088 if (llvm::is_contained(Range: HandleIntrins, Element: IID))
1089 Handles.push_back(Elt: II);
1090 }
1091 }
1092
1093 return Handles;
1094}
1095
1096static hlsl::Binding getHandleIntrinsicBinding(IntrinsicInst *Handle,
1097 DXILResourceTypeMap &DRTM) {
1098 assert(llvm::is_contained(HandleIntrins, Handle->getIntrinsicID()) &&
1099 "Only expects a Handle as determined from collectUsedHandles.");
1100
1101 auto *HandleTy = cast<TargetExtType>(Val: Handle->getType());
1102 dxil::ResourceClass Class = DRTM[HandleTy].getResourceClass();
1103 uint32_t Space = cast<ConstantInt>(Val: Handle->getArgOperand(i: 0))->getZExtValue();
1104 uint32_t LowerBound =
1105 cast<ConstantInt>(Val: Handle->getArgOperand(i: 1))->getZExtValue();
1106 uint32_t Size = cast<ConstantInt>(Val: Handle->getArgOperand(i: 2))->getZExtValue();
1107 uint32_t UpperBound = Size == UINT32_MAX ? UINT32_MAX : LowerBound + Size - 1;
1108
1109 return hlsl::Binding(Class, Space, LowerBound, UpperBound, nullptr);
1110}
1111
1112namespace {
1113/// Helper for propagating the current handle and ptr indices.
1114struct AccessIndices {
1115 Value *GetPtrIdx;
1116 Value *HandleIdx;
1117 Value *OffsetIdx;
1118
1119 bool hasGetPtrIdx() { return GetPtrIdx != nullptr; }
1120 bool hasHandleIdx() { return HandleIdx != nullptr; }
1121 bool hasOffsetIdx() { return OffsetIdx != nullptr; }
1122};
1123
1124/// AccessIndices cached per phi. The values may be placeholder phis that are
1125/// later folded away (via replaceAllUsesWith) and freed, so track them with
1126/// value handles rather than raw pointers.
1127struct TrackedAccessIndices {
1128 WeakTrackingVH GetPtrIdx;
1129 WeakTrackingVH HandleIdx;
1130 WeakTrackingVH OffsetIdx;
1131
1132 TrackedAccessIndices() = default;
1133 TrackedAccessIndices(const AccessIndices &AI)
1134 : GetPtrIdx(AI.GetPtrIdx), HandleIdx(AI.HandleIdx),
1135 OffsetIdx(AI.OffsetIdx) {}
1136 operator AccessIndices() const { return {.GetPtrIdx: GetPtrIdx, .HandleIdx: HandleIdx, .OffsetIdx: OffsetIdx}; }
1137};
1138
1139using PhiAccessIndicesMap = SmallDenseMap<PHINode *, TrackedAccessIndices>;
1140} // namespace
1141
1142// Compute the total byte offset described by a GEP as a single i32 value,
1143// handling both constant and variable indices.
1144static Value *accumulateGEPOffset(GetElementPtrInst *GEP,
1145 IRBuilder<> &Builder) {
1146 const DataLayout &DL = GEP->getDataLayout();
1147 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1148 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1149 APInt ConstantOffset(BitWidth, 0);
1150 bool Success =
1151 GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset);
1152 assert(Success && "Unhandled GEP structure for resource access");
1153 (void)Success;
1154
1155 Type *I32 = Builder.getInt32Ty();
1156 Value *Offset = ConstantInt::get(Ty: I32, V: ConstantOffset.getSExtValue());
1157 for (auto &[V, Scale] : VariableOffsets) {
1158 Value *Index = Builder.CreateSExtOrTrunc(V, DestTy: I32);
1159 if (!Scale.isOne())
1160 Index =
1161 Builder.CreateMul(LHS: Index, RHS: ConstantInt::get(Ty: I32, V: Scale.getSExtValue()));
1162 Offset = Builder.CreateAdd(LHS: Offset, RHS: Index);
1163 }
1164 return Offset;
1165}
1166
1167// getAccessIndices traverses up the control flow that a ptr came from and
1168// propagates back the indicies used to access the resource (AccessIndices):
1169//
1170// - GetPtrIdx is the index of dx.resource.getpointer
1171// - HandleIdx is the index of dx.resource.handlefrom.*
1172// - OffsetIdx is the accumulated byte offset of any GEPs in the ptr chain
1173static AccessIndices
1174getAccessIndices(Instruction *I, SmallSetVector<Instruction *, 16> &DeadInsts,
1175 PhiAccessIndicesMap &VisitedPhis) {
1176 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
1177 if (llvm::is_contained(Range: HandleIntrins, Element: II->getIntrinsicID())) {
1178 DeadInsts.insert(X: II);
1179 return {.GetPtrIdx: nullptr, .HandleIdx: II->getArgOperand(/*Index=*/i: 3), .OffsetIdx: nullptr};
1180 }
1181
1182 Intrinsic::ID IID = II->getIntrinsicID();
1183 if (IID == Intrinsic::dx_resource_getpointer ||
1184 IID == Intrinsic::dx_resource_getbasepointer) {
1185 auto *V = dyn_cast<Instruction>(Val: II->getArgOperand(/*Handle=*/i: 0));
1186 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
1187 assert(!AccessIdx.hasGetPtrIdx() &&
1188 "Encountered multiple dx.resource.getpointers in ptr chain?");
1189 IRBuilder<> Builder(II);
1190 AccessIdx.GetPtrIdx = ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
1191 if (IID == Intrinsic::dx_resource_getpointer)
1192 AccessIdx.GetPtrIdx = II->getArgOperand(i: 1);
1193
1194 DeadInsts.insert(X: II);
1195 return AccessIdx;
1196 }
1197 }
1198
1199 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1200 auto *V = dyn_cast<Instruction>(Val: GEP->getPointerOperand());
1201 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
1202
1203 IRBuilder<> Builder(GEP);
1204 Value *GEPOffset = accumulateGEPOffset(GEP, Builder);
1205 AccessIdx.OffsetIdx =
1206 AccessIdx.hasOffsetIdx()
1207 ? Builder.CreateAdd(LHS: AccessIdx.OffsetIdx, RHS: GEPOffset)
1208 : GEPOffset;
1209
1210 DeadInsts.insert(X: GEP);
1211 return AccessIdx;
1212 }
1213
1214 if (auto *Phi = dyn_cast<PHINode>(Val: I)) {
1215 // If we're already building indices for this phi, return a ref to the phi
1216 if (auto It = VisitedPhis.find(Val: Phi); It != VisitedPhis.end())
1217 return It->second;
1218
1219 unsigned NumEdges = Phi->getNumIncomingValues();
1220 assert(NumEdges != 0 && "Malformed Phi Node");
1221
1222 IRBuilder<> Builder(Phi);
1223 std::unique_ptr<PHINode> GetPtrPhi(
1224 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
1225 std::unique_ptr<PHINode> HandlePhi(
1226 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
1227 std::unique_ptr<PHINode> OffsetPhi(
1228 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
1229 bool HasOffset = false;
1230
1231 // Register a ref to this phi for a recursive phi. If a placeholder phi
1232 // below folds to a single value, replaceAllUsesWith updates any cached
1233 // entries that captured it.
1234 if (Phi->getType()->isTargetExtTy())
1235 VisitedPhis[Phi] = AccessIndices{/*GetPtrIdx=*/nullptr, .HandleIdx: HandlePhi.get(),
1236 /*OffsetIdx=*/nullptr};
1237 else if (Phi->getType()->isPointerTy())
1238 VisitedPhis[Phi] =
1239 AccessIndices{.GetPtrIdx: GetPtrPhi.get(), .HandleIdx: HandlePhi.get(), .OffsetIdx: OffsetPhi.get()};
1240
1241 for (unsigned Idx = 0; Idx < NumEdges; Idx++) {
1242 auto *BB = Phi->getIncomingBlock(i: Idx);
1243 auto *V = dyn_cast<Instruction>(Val: Phi->getIncomingValue(i: Idx));
1244 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
1245 if (AccessIdx.hasGetPtrIdx())
1246 GetPtrPhi->addIncoming(V: AccessIdx.GetPtrIdx, BB);
1247 HandlePhi->addIncoming(V: AccessIdx.HandleIdx, BB);
1248 if (AccessIdx.hasOffsetIdx()) {
1249 OffsetPhi->addIncoming(V: AccessIdx.OffsetIdx, BB);
1250 HasOffset = true;
1251 } else {
1252 OffsetPhi->addIncoming(V: ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0), BB);
1253 }
1254 }
1255
1256 Value *GetPtrIdx;
1257 if (GetPtrPhi->getNumIncomingValues() == 0) {
1258 GetPtrIdx = nullptr;
1259 } else if (Value *ConstantGetPtr = GetPtrPhi->hasConstantValue()) {
1260 GetPtrIdx = ConstantGetPtr;
1261 GetPtrPhi->replaceAllUsesWith(V: GetPtrIdx);
1262 } else {
1263 GetPtrIdx = GetPtrPhi.release();
1264 Builder.Insert(V: GetPtrIdx);
1265 }
1266
1267 Value *HandleIdx;
1268 if (Value *ConstantHandle = HandlePhi->hasConstantValue()) {
1269 HandleIdx = ConstantHandle;
1270 HandlePhi->replaceAllUsesWith(V: HandleIdx);
1271 } else {
1272 HandleIdx = HandlePhi.release();
1273 Builder.Insert(V: HandleIdx);
1274 }
1275
1276 Value *OffsetIdx = nullptr;
1277 if (HasOffset) {
1278 if (Value *ConstantIdx = OffsetPhi->hasConstantValue()) {
1279 OffsetIdx = ConstantIdx;
1280 OffsetPhi->replaceAllUsesWith(V: OffsetIdx);
1281 } else {
1282 OffsetIdx = OffsetPhi.release();
1283 Builder.Insert(V: OffsetIdx);
1284 }
1285 }
1286
1287 // Replace the placeholder entry with the final indices.
1288 AccessIndices Result = {.GetPtrIdx: GetPtrIdx, .HandleIdx: HandleIdx, .OffsetIdx: OffsetIdx};
1289 VisitedPhis[Phi] = Result;
1290 DeadInsts.insert(X: Phi);
1291 return Result;
1292 }
1293
1294 if (auto *Select = dyn_cast<SelectInst>(Val: I)) {
1295 auto *TrueV = dyn_cast<Instruction>(Val: Select->getTrueValue());
1296 auto TrueAccessIdx = getAccessIndices(I: TrueV, DeadInsts, VisitedPhis);
1297
1298 auto *FalseV = dyn_cast<Instruction>(Val: Select->getFalseValue());
1299 auto FalseAccessIdx = getAccessIndices(I: FalseV, DeadInsts, VisitedPhis);
1300
1301 IRBuilder<> Builder(Select);
1302 Value *GetPtrSelect = nullptr;
1303 Value *OffsetSelect = nullptr;
1304
1305 if (TrueAccessIdx.hasGetPtrIdx() && FalseAccessIdx.hasGetPtrIdx())
1306 GetPtrSelect =
1307 Builder.CreateSelect(C: Select->getCondition(), True: TrueAccessIdx.GetPtrIdx,
1308 False: FalseAccessIdx.GetPtrIdx, Name: "", MDFrom: Select);
1309
1310 auto *HandleSelect =
1311 Builder.CreateSelect(C: Select->getCondition(), True: TrueAccessIdx.HandleIdx,
1312 False: FalseAccessIdx.HandleIdx, Name: "", MDFrom: Select);
1313
1314 if (TrueAccessIdx.hasOffsetIdx() || FalseAccessIdx.hasOffsetIdx()) {
1315 Value *Zero = ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
1316 Value *TrueOffset =
1317 TrueAccessIdx.hasOffsetIdx() ? TrueAccessIdx.OffsetIdx : Zero;
1318 Value *FalseOffset =
1319 FalseAccessIdx.hasOffsetIdx() ? FalseAccessIdx.OffsetIdx : Zero;
1320 OffsetSelect = Builder.CreateSelect(C: Select->getCondition(), True: TrueOffset,
1321 False: FalseOffset, Name: "", MDFrom: Select);
1322 }
1323
1324 DeadInsts.insert(X: Select);
1325 return {.GetPtrIdx: GetPtrSelect, .HandleIdx: HandleSelect, .OffsetIdx: OffsetSelect};
1326 }
1327
1328 llvm_unreachable("collectUsedHandles should assure this does not occur");
1329}
1330
1331static void
1332replaceHandleWithIndices(Instruction *Ptr, IntrinsicInst *OldHandle,
1333 SmallSetVector<Instruction *, 16> &DeadInsts,
1334 PhiAccessIndicesMap &VisitedPhis) {
1335 auto AccessIdx = getAccessIndices(I: Ptr, DeadInsts, VisitedPhis);
1336 assert(AccessIdx.hasHandleIdx() &&
1337 "Couldn't retrieve handle index. This is guaranteed by "
1338 "getAccessIndices");
1339
1340 IRBuilder<> Builder(Ptr);
1341 if (isa<PHINode>(Val: Ptr))
1342 Builder.SetInsertPoint(Ptr->getParent()->getFirstNonPHIIt());
1343 IntrinsicInst *Handle = cast<IntrinsicInst>(Val: OldHandle->clone());
1344 Handle->setArgOperand(/*Index=*/i: 3, v: AccessIdx.HandleIdx);
1345 Builder.Insert(I: Handle);
1346
1347 if (Ptr->getType()->isPointerTy()) {
1348 assert(AccessIdx.hasGetPtrIdx() &&
1349 "Couldn't retrieve getpointer index. This is guaranteed by "
1350 "getAccessIndices");
1351 auto *GetPtr = Builder.CreateIntrinsic(RetTy: Ptr->getType(),
1352 ID: Intrinsic::dx_resource_getpointer,
1353 Args: {Handle, AccessIdx.GetPtrIdx});
1354
1355 // Reapply any byte offset that came from GEPs in the original ptr chain so
1356 // that later access lowering still sees it.
1357 Value *Result = GetPtr;
1358 if (AccessIdx.hasOffsetIdx())
1359 Result = Builder.CreatePtrAdd(Ptr: GetPtr, Offset: AccessIdx.OffsetIdx);
1360 Ptr->replaceAllUsesWith(V: Result);
1361 } else {
1362 assert(Ptr->getType()->isTargetExtTy() && !AccessIdx.hasGetPtrIdx() &&
1363 "Unexpected resource access operand type");
1364 Ptr->replaceAllUsesWith(V: Handle);
1365 }
1366
1367 DeadInsts.insert(X: Ptr);
1368}
1369
1370// Try to legalize dx.resource.handlefrom.*.binding and dx.resource.getpointer
1371// calls with their respective index values and propagate the index values to
1372// be used at resource access.
1373//
1374// If it can't be transformed to be legal then:
1375//
1376// Reports an error if a resource access is not guaranteed into a unique global
1377// resource.
1378//
1379// Returns true if any changes are made.
1380static bool legalizeResourceHandles(Function &F, DXILResourceTypeMap &DRTM) {
1381 SmallSetVector<Instruction *, 16> DeadInsts;
1382 PhiAccessIndicesMap VisitedPhis;
1383
1384 for (BasicBlock &BB : make_early_inc_range(Range&: F)) {
1385 for (Instruction &I : BB) {
1386 if (auto *HandleOp = getHandleOperand(AI: &I)) {
1387 SmallVector<IntrinsicInst *> Handles = collectUsedHandles(Ptr: HandleOp);
1388 unsigned NumHandles = Handles.size();
1389 if (NumHandles <= 1)
1390 continue; // Legal, no-replacement required
1391
1392 bool SameGlobalBinding = true;
1393 hlsl::Binding B = getHandleIntrinsicBinding(Handle: Handles[0], DRTM);
1394 for (unsigned Idx = 1; Idx < NumHandles; Idx++)
1395 SameGlobalBinding &=
1396 (B == getHandleIntrinsicBinding(Handle: Handles[Idx], DRTM));
1397
1398 if (!SameGlobalBinding)
1399 diagnoseNonUniqueResourceAccess(I: &I, Handles);
1400
1401 replaceHandleWithIndices(Ptr: getHandleRoot(I: HandleOp), OldHandle: Handles[0], DeadInsts,
1402 VisitedPhis);
1403 }
1404 }
1405 }
1406
1407 bool MadeChanges = false;
1408
1409 // Set up the phis to track if they are erased below
1410 SmallVector<WeakTrackingVH> ResourcePhis;
1411 for (const auto &HandleToIndex : VisitedPhis)
1412 ResourcePhis.push_back(Elt: HandleToIndex.first);
1413
1414 for (auto *I : llvm::reverse(C&: DeadInsts))
1415 if (I->hasNUses(N: 0)) { // Handle can still be used outside of replaced path
1416 I->eraseFromParent();
1417 MadeChanges = true;
1418 }
1419
1420 // Any remaining phi nodes are now looped with another phi node and have no
1421 // other uses
1422 for (WeakTrackingVH &VH : ResourcePhis)
1423 if (VH) // True if not removed above or already in this loop
1424 MadeChanges |= RecursivelyDeleteDeadPHINode(PN: cast<PHINode>(Val&: VH));
1425
1426 return MadeChanges;
1427}
1428
1429static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
1430 SmallVector<User *> Worklist;
1431 for (User *U : II->users())
1432 Worklist.push_back(Elt: U);
1433
1434 SmallVector<Instruction *> DeadInsts;
1435 while (!Worklist.empty()) {
1436 User *U = Worklist.back();
1437 Worklist.pop_back();
1438
1439 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: U)) {
1440 for (User *U : GEP->users())
1441 Worklist.push_back(Elt: U);
1442 DeadInsts.push_back(Elt: GEP);
1443
1444 } else if (auto *SI = dyn_cast<StoreInst>(Val: U)) {
1445 assert(SI->getValueOperand() != II && "Pointer escaped!");
1446 createStoreIntrinsic(II, SI, RTI);
1447 DeadInsts.push_back(Elt: SI);
1448
1449 } else if (auto *LI = dyn_cast<LoadInst>(Val: U)) {
1450 createLoadIntrinsic(II, LI, RTI);
1451 DeadInsts.push_back(Elt: LI);
1452 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: U)) {
1453 createAtomicBinOpIntrinsic(II, AI, RTI);
1454 DeadInsts.push_back(Elt: AI);
1455 } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: U)) {
1456 createAtomicCompareExchangeIntrinsic(II, AI: CXI, RTI);
1457 DeadInsts.push_back(Elt: CXI);
1458 } else
1459 llvm_unreachable("Unhandled instruction - pointer escaped?");
1460 }
1461
1462 // Traverse the now-dead instructions in RPO and remove them.
1463 for (Instruction *Dead : llvm::reverse(C&: DeadInsts))
1464 Dead->eraseFromParent();
1465 II->eraseFromParent();
1466}
1467
1468static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) {
1469 SmallVector<std::pair<IntrinsicInst *, dxil::ResourceTypeInfo>> Resources;
1470 for (BasicBlock &BB : make_early_inc_range(Range&: F))
1471 for (Instruction &I : BB)
1472 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1473 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer ||
1474 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer) {
1475 auto *HandleTy = cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType());
1476 assert(
1477 (DRTM[HandleTy].isCBuffer() ||
1478 II->getIntrinsicID() != Intrinsic::dx_resource_getbasepointer) &&
1479 "dx_resource_getbasepointer should only be used by cbuffers");
1480 Resources.emplace_back(Args&: II, Args&: DRTM[HandleTy]);
1481 }
1482
1483 for (auto &[II, RI] : Resources)
1484 replaceAccess(II, RTI&: RI);
1485
1486 return !Resources.empty();
1487}
1488
1489PreservedAnalyses DXILResourceAccessPass::run(Function &F,
1490 FunctionAnalysisManager &FAM) {
1491 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1492 DXILResourceTypeMap *DRTM =
1493 MAMProxy.getCachedResult<DXILResourceTypeAnalysis>(IR&: *F.getParent());
1494 assert(DRTM && "DXILResourceTypeAnalysis must be available");
1495
1496 bool MadeHandleChanges = legalizeResourceHandles(F, DRTM&: *DRTM);
1497 bool MadeResourceChanges = transformResourcePointers(F, DRTM&: *DRTM);
1498 if (!(MadeHandleChanges || MadeResourceChanges))
1499 return PreservedAnalyses::all();
1500
1501 PreservedAnalyses PA;
1502 PA.preserve<DXILResourceTypeAnalysis>();
1503 return PA;
1504}
1505
1506namespace {
1507class DXILResourceAccessLegacy : public FunctionPass {
1508public:
1509 bool runOnFunction(Function &F) override {
1510 DXILResourceTypeMap &DRTM =
1511 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
1512 bool MadeHandleChanges = legalizeResourceHandles(F, DRTM);
1513 bool MadeResourceChanges = transformResourcePointers(F, DRTM);
1514 return MadeHandleChanges || MadeResourceChanges;
1515 }
1516 StringRef getPassName() const override { return "DXIL Resource Access"; }
1517 DXILResourceAccessLegacy() : FunctionPass(ID) {}
1518
1519 static char ID; // Pass identification.
1520 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
1521 AU.addRequired<DXILResourceTypeWrapperPass>();
1522 }
1523};
1524char DXILResourceAccessLegacy::ID = 0;
1525} // end anonymous namespace
1526
1527INITIALIZE_PASS_BEGIN(DXILResourceAccessLegacy, DEBUG_TYPE,
1528 "DXIL Resource Access", false, false)
1529INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
1530INITIALIZE_PASS_END(DXILResourceAccessLegacy, DEBUG_TYPE,
1531 "DXIL Resource Access", false, false)
1532
1533FunctionPass *llvm::createDXILResourceAccessLegacyPass() {
1534 return new DXILResourceAccessLegacy();
1535}
1536