1//===- DXILResourceAccess.cpp - Resource access via load/store ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DXILResourceAccess.h"
10#include "DirectX.h"
11#include "llvm/ADT/DenseMap.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/ADT/SmallSet.h"
14#include "llvm/Analysis/DXILResource.h"
15#include "llvm/Analysis/VectorUtils.h"
16#include "llvm/Frontend/HLSL/HLSLResource.h"
17#include "llvm/IR/BasicBlock.h"
18#include "llvm/IR/Dominators.h"
19#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/Instruction.h"
21#include "llvm/IR/Instructions.h"
22#include "llvm/IR/IntrinsicInst.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/IntrinsicsDirectX.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/User.h"
27#include "llvm/IR/ValueHandle.h"
28#include "llvm/InitializePasses.h"
29#include "llvm/Support/DXILABI.h"
30#include "llvm/Support/FormatVariadic.h"
31#include "llvm/Transforms/Utils/Local.h"
32#include "llvm/Transforms/Utils/ValueMapper.h"
33#include <optional>
34
35#define DEBUG_TYPE "dxil-resource-access"
36
37using namespace llvm;
38
39static void diagnoseNonUniqueResourceAccess(Instruction *I,
40 ArrayRef<IntrinsicInst *> Handles) {
41 LLVMContext &Context = I->getContext();
42 std::string InstStr;
43 raw_string_ostream InstOS(InstStr);
44 I->print(O&: InstOS);
45 Context.diagnose(
46 DI: DiagnosticInfoGeneric("At resource access:" + Twine(InstStr), DS_Note));
47
48 for (auto *Handle : Handles) {
49 std::string HandleStr;
50 raw_string_ostream HandleOS(HandleStr);
51 Handle->print(O&: HandleOS);
52 Context.diagnose(DI: DiagnosticInfoGeneric(
53 "Uses resource handle:" + Twine(HandleStr), DS_Note));
54 }
55 Context.diagnose(DI: DiagnosticInfoGeneric(
56 "Resource access is not guaranteed to map to a unique global resource"));
57}
58
59static Value *traverseGEPOffsets(const DataLayout &DL, IRBuilder<> &Builder,
60 Value *Ptr, uint64_t AccessSize) {
61 Value *Offset = nullptr;
62
63 while (Ptr) {
64 if ([[maybe_unused]] auto *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
65 assert((II->getIntrinsicID() == Intrinsic::dx_resource_getpointer ||
66 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer) &&
67 "Resource access through unexpected intrinsic");
68 return Offset ? Offset : ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
69 }
70
71 auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
72 assert(GEP && "Resource access through unexpected instruction");
73
74 unsigned NumIndices = GEP->getNumIndices();
75 uint64_t IndexScale = DL.getTypeAllocSize(Ty: GEP->getSourceElementType());
76 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: GEP->getType()), 0);
77 Value *GEPOffset;
78 if (GEP->accumulateConstantOffset(DL, Offset&: ConstantOffset)) {
79 // We have a constant offset (in bytes).
80 GEPOffset =
81 ConstantInt::get(Ty: DL.getIndexType(PtrTy: GEP->getType()), V: ConstantOffset);
82 IndexScale = 1;
83 } else if (NumIndices == 1) {
84 // If we have a single index we're indexing into a top level array. This
85 // generally only happens with cbuffers.
86 GEPOffset = *GEP->idx_begin();
87 } else if (NumIndices == 2) {
88 // If we have two indices, this should be an access through a pointer.
89 auto *IndexIt = GEP->idx_begin();
90 assert(cast<ConstantInt>(IndexIt)->getZExtValue() == 0 &&
91 "GEP is not indexing through pointer");
92 GEPOffset = *(++IndexIt);
93 } else
94 llvm_unreachable("Unhandled GEP structure for resource access");
95
96 uint64_t ElemSize = AccessSize;
97 if (!(IndexScale % ElemSize)) {
98 // If our scale is an exact multiple of the access size, adjust the
99 // scaling to avoid an unnecessary division.
100 IndexScale /= ElemSize;
101 ElemSize = 1;
102 }
103 if (IndexScale != 1)
104 GEPOffset = Builder.CreateMul(
105 LHS: GEPOffset, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: IndexScale));
106 if (ElemSize != 1)
107 GEPOffset = Builder.CreateUDiv(
108 LHS: GEPOffset, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: ElemSize));
109
110 Offset = Offset ? Builder.CreateAdd(LHS: Offset, RHS: GEPOffset) : GEPOffset;
111 Ptr = GEP->getPointerOperand();
112 }
113
114 llvm_unreachable("GEP of null pointer?");
115}
116
117static void createTypedBufferStore(IntrinsicInst *II, StoreInst *SI,
118 dxil::ResourceTypeInfo &RTI) {
119 const DataLayout &DL = SI->getDataLayout();
120 IRBuilder<> Builder(SI);
121 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
122 Type *ScalarType = ContainedType->getScalarType();
123 Type *LoadType = StructType::get(elt1: ContainedType, elts: Builder.getInt1Ty());
124
125 Value *V = SI->getValueOperand();
126 if (V->getType() == ContainedType) {
127 // V is already the right type.
128 assert(SI->getPointerOperand() == II &&
129 "Store of whole element has mismatched address to store to");
130 } else if (V->getType() == ScalarType) {
131 // We're storing a scalar, so we need to load the current value and only
132 // replace the relevant part.
133 auto *Load = Builder.CreateIntrinsic(
134 RetTy: LoadType, ID: Intrinsic::dx_resource_load_typedbuffer,
135 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1)});
136 auto *Struct = Builder.CreateExtractValue(Agg: Load, Idxs: {0});
137
138 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
139 Value *Offset =
140 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
141 V = Builder.CreateInsertElement(Vec: Struct, NewElt: V, Idx: Offset);
142 } else {
143 llvm_unreachable("Store to typed resource has invalid type");
144 }
145
146 auto *Inst = Builder.CreateIntrinsic(
147 RetTy: Builder.getVoidTy(), ID: Intrinsic::dx_resource_store_typedbuffer,
148 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1), V});
149 SI->replaceAllUsesWith(V: Inst);
150}
151
152/// Build a zero-initialized offset operand matching the shape of the given
153/// coordinate operand. Accesses through `operator[]` never have offsets.
154static Value *getNullOffsetsFor(IRBuilder<> &Builder, Value *Coords) {
155 Type *CoordTy = Coords->getType();
156 Type *OffsetTy;
157 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: CoordTy))
158 OffsetTy =
159 FixedVectorType::get(ElementType: Builder.getInt32Ty(), NumElts: VecTy->getNumElements());
160 else
161 OffsetTy = Builder.getInt32Ty();
162 return Constant::getNullValue(Ty: OffsetTy);
163}
164
165static void createTextureStore(IntrinsicInst *II, StoreInst *SI,
166 dxil::ResourceTypeInfo &RTI) {
167 const DataLayout &DL = SI->getDataLayout();
168 IRBuilder<> Builder(SI);
169 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
170 Type *ScalarType = ContainedType->getScalarType();
171
172 Value *Handle = II->getOperand(i_nocapture: 0);
173 Value *Coords = II->getOperand(i_nocapture: 1);
174
175 Value *V = SI->getValueOperand();
176 if (V->getType() == ContainedType) {
177 // V is already the right type.
178 assert(SI->getPointerOperand() == II &&
179 "Store of whole element has mismatched address to store to");
180 } else if (V->getType() == ScalarType) {
181 // We're storing a scalar, so we need to load the current value and only
182 // replace the relevant part. For operator[] the mip level and the offsets
183 // are always zero; DXILOpLowering drops the mip level for UAVs.
184 Value *MipLevel = Builder.getInt32(C: 0);
185 Value *Offsets = getNullOffsetsFor(Builder, Coords);
186 auto *Load = Builder.CreateIntrinsic(RetTy: ContainedType,
187 ID: Intrinsic::dx_resource_load_level,
188 Args: {Handle, Coords, MipLevel, Offsets});
189
190 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
191 Value *Offset =
192 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
193 V = Builder.CreateInsertElement(Vec: Load, NewElt: V, Idx: Offset);
194 } else {
195 llvm_unreachable("Store to texture resource has invalid type");
196 }
197
198 auto *Inst = Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(),
199 ID: Intrinsic::dx_resource_store_texture,
200 Args: {Handle, Coords, V});
201 SI->replaceAllUsesWith(V: Inst);
202}
203
204static void emitRawStore(IRBuilder<> &Builder, Value *Buffer, Value *Index,
205 Value *Offset, Value *V, dxil::ResourceTypeInfo &RTI) {
206 // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
207 // entirely into the index.
208 if (!RTI.isStruct()) {
209 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
210 if (!ConstantOffset || !ConstantOffset->isZero())
211 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
212 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
213 }
214
215 Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(),
216 ID: Intrinsic::dx_resource_store_rawbuffer,
217 Args: {Buffer, Index, Offset, V});
218}
219
220static void createRawStores(IntrinsicInst *II, StoreInst *SI,
221 dxil::ResourceTypeInfo &RTI) {
222 const DataLayout &DL = SI->getDataLayout();
223 IRBuilder<> Builder(SI);
224
225 Value *V = SI->getValueOperand();
226 assert(!V->getType()->isAggregateType() &&
227 "Resource store should be scalar or vector type");
228
229 Value *Index = II->getOperand(i_nocapture: 1);
230 // The offset for the rawbuffer load and store ops is always in bytes.
231 uint64_t AccessSize = 1;
232 Value *Offset =
233 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
234
235 auto *VT = dyn_cast<FixedVectorType>(Val: V->getType());
236 if (VT && VT->getNumElements() > 4) {
237 // Split into stores of at most 4 elements.
238 Type *EltTy = VT->getElementType();
239 Value *Stride = ConstantInt::get(Ty: Builder.getInt32Ty(),
240 V: 4 * (DL.getTypeSizeInBits(Ty: EltTy) / 8));
241
242 SmallVector<int, 4> Indices;
243 for (unsigned int I = 0, N = VT->getNumElements(); I < N; I += 4) {
244 if (I > 0)
245 Offset = Builder.CreateAdd(LHS: Offset, RHS: Stride);
246
247 for (unsigned int J = I, E = std::min(a: N, b: J + 4); J < E; ++J)
248 Indices.push_back(Elt: J);
249 Value *Part = Builder.CreateShuffleVector(V, Mask: Indices);
250 emitRawStore(Builder, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, V: Part, RTI);
251
252 Indices.clear();
253 }
254 } else
255 emitRawStore(Builder, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, V, RTI);
256}
257
258static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI,
259 dxil::ResourceTypeInfo &RTI) {
260 switch (RTI.getResourceKind()) {
261 case dxil::ResourceKind::TypedBuffer:
262 return createTypedBufferStore(II, SI, RTI);
263 case dxil::ResourceKind::RawBuffer:
264 case dxil::ResourceKind::StructuredBuffer:
265 return createRawStores(II, SI, RTI);
266 case dxil::ResourceKind::Texture1D:
267 case dxil::ResourceKind::Texture2D:
268 case dxil::ResourceKind::Texture3D:
269 case dxil::ResourceKind::Texture1DArray:
270 case dxil::ResourceKind::Texture2DArray:
271 return createTextureStore(II, SI, RTI);
272 case dxil::ResourceKind::Texture2DMS:
273 case dxil::ResourceKind::Texture2DMSArray:
274 case dxil::ResourceKind::TextureCube:
275 case dxil::ResourceKind::TextureCubeArray:
276 case dxil::ResourceKind::FeedbackTexture2D:
277 case dxil::ResourceKind::FeedbackTexture2DArray:
278 reportFatalUsageError(
279 reason: "DXIL Store not implemented for this texture resource kind");
280 return;
281 case dxil::ResourceKind::CBuffer:
282 case dxil::ResourceKind::Sampler:
283 case dxil::ResourceKind::TBuffer:
284 case dxil::ResourceKind::RTAccelerationStructure:
285 case dxil::ResourceKind::Invalid:
286 case dxil::ResourceKind::NumEntries:
287 llvm_unreachable("Invalid resource kind for store");
288 }
289 llvm_unreachable("Unhandled case in switch");
290}
291
292static std::optional<dxil::AtomicBinOpCode>
293getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) {
294 switch (BinOp) {
295 case AtomicRMWInst::Add:
296 return dxil::AtomicBinOpCode::Add;
297 case AtomicRMWInst::And:
298 return dxil::AtomicBinOpCode::And;
299 case AtomicRMWInst::Or:
300 return dxil::AtomicBinOpCode::Or;
301 case AtomicRMWInst::Xor:
302 return dxil::AtomicBinOpCode::Xor;
303 case AtomicRMWInst::Min:
304 return dxil::AtomicBinOpCode::IMin;
305 case AtomicRMWInst::Max:
306 return dxil::AtomicBinOpCode::IMax;
307 case AtomicRMWInst::UMin:
308 return dxil::AtomicBinOpCode::UMin;
309 case AtomicRMWInst::UMax:
310 return dxil::AtomicBinOpCode::UMax;
311 case AtomicRMWInst::Xchg:
312 return dxil::AtomicBinOpCode::Exchange;
313 case AtomicRMWInst::Sub:
314 case AtomicRMWInst::Nand:
315 case AtomicRMWInst::FAdd:
316 case AtomicRMWInst::FSub:
317 case AtomicRMWInst::FMax:
318 case AtomicRMWInst::FMin:
319 case AtomicRMWInst::FMaximum:
320 case AtomicRMWInst::FMinimum:
321 case AtomicRMWInst::FMaximumNum:
322 case AtomicRMWInst::FMinimumNum:
323 case AtomicRMWInst::UIncWrap:
324 case AtomicRMWInst::UDecWrap:
325 case AtomicRMWInst::USubCond:
326 case AtomicRMWInst::USubSat:
327 case AtomicRMWInst::BAD_BINOP:
328 return std::nullopt;
329 }
330 llvm_unreachable("Unhandled atomicrmw operation");
331}
332
333static void emitAtomicBinOp(IRBuilder<> &Builder, AtomicRMWInst *AI,
334 Value *Handle, ArrayRef<Value *> Coords) {
335 assert(!Coords.empty() && Coords.size() <= 3 &&
336 "Atomic operations take between one and three coordinates");
337
338 std::optional<dxil::AtomicBinOpCode> BinOpCode =
339 getAtomicBinOpCode(BinOp: AI->getOperation());
340 if (!BinOpCode) {
341 reportFatalUsageError(reason: "DXIL resource atomicrmw operation not implemented");
342 return;
343 }
344
345 SmallVector<Value *, 6> Args{
346 Handle, Builder.getInt32(C: static_cast<uint32_t>(*BinOpCode))};
347 append_range(C&: Args, R&: Coords);
348 Args.append(NumInputs: 3 - Coords.size(), Elt: PoisonValue::get(T: Builder.getInt32Ty()));
349 Args.push_back(Elt: AI->getValOperand());
350
351 // Emit the target-independent intrinsic; DXILOpLowering lowers it to the
352 // DXIL `AtomicBinOp` op and handles the target-ext-typed handle cast via
353 // its `createTmpHandleCast` bookkeeping.
354 Value *Result = Builder.CreateIntrinsic(
355 RetTy: AI->getType(), ID: Intrinsic::dx_resource_atomic_binop, Args);
356
357 AI->replaceAllUsesWith(V: Result);
358}
359
360static void createBufferAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
361 dxil::ResourceTypeInfo &RTI) {
362 const DataLayout &DL = AI->getDataLayout();
363 IRBuilder<> Builder(AI);
364 Value *Index = II->getOperand(i_nocapture: 1);
365
366 // The offset for the rawbuffer load/store/atomic ops is always in bytes.
367 uint64_t AccessSize = 1;
368 Value *Offset =
369 traverseGEPOffsets(DL, Builder, Ptr: AI->getPointerOperand(), AccessSize);
370
371 // For non-struct buffers (RawBuffer or TypedBuffer), fold the byte offset
372 // into the index and only pass a single coordinate — only StructuredBuffer
373 // atomics use both a struct index and a byte offset.
374 if (!RTI.isStruct()) {
375 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
376 if (!ConstantOffset || !ConstantOffset->isZero())
377 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
378
379 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: {Index});
380 return;
381 }
382
383 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: {Index, Offset});
384}
385
386static void createTextureAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
387 dxil::ResourceTypeInfo &RTI) {
388 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
389 if (!ContainedType->isIntegerTy()) {
390 reportFatalUsageError(reason: "DXIL atomicrmw requires a texture resource with a "
391 "scalar integer element type");
392 return;
393 }
394
395 IRBuilder<> Builder(AI);
396
397 // The coordinates of a texture access are a scalar or a vector with one
398 // element per texture dimension, including the array slice if there is one.
399 // These map directly onto the coordinate operands of the atomic op.
400 Value *Coords = II->getOperand(i_nocapture: 1);
401 SmallVector<Value *, 3> CoordArgs;
402 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Coords->getType())) {
403 assert(VecTy->getNumElements() <= 3 && "Too many texture coordinates");
404 for (unsigned I = 0, E = VecTy->getNumElements(); I != E; ++I)
405 CoordArgs.push_back(Elt: Builder.CreateExtractElement(Vec: Coords, Idx: I));
406 } else {
407 CoordArgs.push_back(Elt: Coords);
408 }
409
410 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: CoordArgs);
411}
412
413static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
414 dxil::ResourceTypeInfo &RTI) {
415 switch (RTI.getResourceKind()) {
416 case dxil::ResourceKind::TypedBuffer:
417 case dxil::ResourceKind::RawBuffer:
418 case dxil::ResourceKind::StructuredBuffer:
419 return createBufferAtomicBinOp(II, AI, RTI);
420 case dxil::ResourceKind::Texture1D:
421 case dxil::ResourceKind::Texture2D:
422 case dxil::ResourceKind::Texture3D:
423 case dxil::ResourceKind::Texture1DArray:
424 case dxil::ResourceKind::Texture2DArray:
425 return createTextureAtomicBinOp(II, AI, RTI);
426 case dxil::ResourceKind::Texture2DMS:
427 case dxil::ResourceKind::Texture2DMSArray:
428 case dxil::ResourceKind::TextureCube:
429 case dxil::ResourceKind::TextureCubeArray:
430 case dxil::ResourceKind::FeedbackTexture2D:
431 case dxil::ResourceKind::FeedbackTexture2DArray:
432 reportFatalUsageError(
433 reason: "DXIL atomicrmw not implemented for this texture resource kind");
434 return;
435 case dxil::ResourceKind::CBuffer:
436 case dxil::ResourceKind::Sampler:
437 case dxil::ResourceKind::TBuffer:
438 reportFatalUsageError(
439 reason: "DXIL atomicrmw not implemented for this resource type");
440 return;
441 case dxil::ResourceKind::RTAccelerationStructure:
442 case dxil::ResourceKind::Invalid:
443 case dxil::ResourceKind::NumEntries:
444 llvm_unreachable("Invalid resource kind for atomicrmw");
445 }
446 llvm_unreachable("Unhandled case in switch");
447}
448
449static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI,
450 dxil::ResourceTypeInfo &RTI) {
451 const DataLayout &DL = LI->getDataLayout();
452 IRBuilder<> Builder(LI);
453 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
454 Type *LoadType = StructType::get(elt1: ContainedType, elts: Builder.getInt1Ty());
455
456 Value *V =
457 Builder.CreateIntrinsic(RetTy: LoadType, ID: Intrinsic::dx_resource_load_typedbuffer,
458 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1)});
459 V = Builder.CreateExtractValue(Agg: V, Idxs: {0});
460
461 Type *ScalarType = ContainedType->getScalarType();
462 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
463 Value *Offset =
464 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
465 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
466 if (!ConstantOffset || !ConstantOffset->isZero())
467 V = Builder.CreateExtractElement(Vec: V, Idx: Offset);
468
469 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a
470 // shufflevector), then make sure we're maintaining the resulting type.
471 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType()))
472 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(Val: V->getType()))
473 V = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: V,
474 Idx: Builder.getInt32(C: 0));
475
476 LI->replaceAllUsesWith(V);
477}
478
479static void createTextureLoad(IntrinsicInst *II, LoadInst *LI,
480 dxil::ResourceTypeInfo &RTI) {
481 const DataLayout &DL = LI->getDataLayout();
482 IRBuilder<> Builder(LI);
483 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
484
485 Value *Handle = II->getOperand(i_nocapture: 0);
486 Value *Coords = II->getOperand(i_nocapture: 1);
487
488 // For operator[], mip level is 0.
489 Value *MipLevel = Builder.getInt32(C: 0);
490
491 // For operator[], offsets are zero.
492 Value *Offsets = getNullOffsetsFor(Builder, Coords);
493
494 Value *V =
495 Builder.CreateIntrinsic(RetTy: ContainedType, ID: Intrinsic::dx_resource_load_level,
496 Args: {Handle, Coords, MipLevel, Offsets});
497
498 Type *ScalarType = ContainedType->getScalarType();
499 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
500 Value *Offset =
501 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
502 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
503 if (!ConstantOffset || !ConstantOffset->isZero())
504 V = Builder.CreateExtractElement(Vec: V, Idx: Offset);
505
506 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a
507 // shufflevector), then make sure we're maintaining the resulting type.
508 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType()))
509 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(Val: V->getType()))
510 V = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: V,
511 Idx: Builder.getInt32(C: 0));
512
513 LI->replaceAllUsesWith(V);
514}
515
516static Value *emitRawLoad(IRBuilder<> &Builder, Type *Ty, Value *Buffer,
517 Value *Index, Value *Offset,
518 dxil::ResourceTypeInfo &RTI) {
519 // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
520 // entirely into the index.
521 if (!RTI.isStruct()) {
522 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
523 if (!ConstantOffset || !ConstantOffset->isZero())
524 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
525 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
526 }
527
528 // The load intrinsic includes the bit for CheckAccessFullyMapped, so we need
529 // to add that to the return type.
530 Type *TypeWithCheck = StructType::get(elt1: Ty, elts: Builder.getInt1Ty());
531 Value *V = Builder.CreateIntrinsic(RetTy: TypeWithCheck,
532 ID: Intrinsic::dx_resource_load_rawbuffer,
533 Args: {Buffer, Index, Offset});
534 return Builder.CreateExtractValue(Agg: V, Idxs: {0});
535}
536
537static void createRawLoads(IntrinsicInst *II, LoadInst *LI,
538 dxil::ResourceTypeInfo &RTI) {
539 const DataLayout &DL = LI->getDataLayout();
540 IRBuilder<> Builder(LI);
541
542 Value *Index = II->getOperand(i_nocapture: 1);
543 // The offset for the rawbuffer load and store ops is always in bytes.
544 uint64_t AccessSize = 1;
545 Value *Offset =
546 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
547
548 // TODO: We could make this handle aggregates by walking the structure and
549 // handling each field individually, but we don't ever generate code that
550 // would hit that so it seems superfluous.
551 assert(!LI->getType()->isAggregateType() &&
552 "Resource load should be scalar or vector type");
553
554 Value *V;
555 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType())) {
556 // Split into loads of at most 4 elements.
557 Type *EltTy = VT->getElementType();
558 Value *Stride = ConstantInt::get(Ty: Builder.getInt32Ty(),
559 V: 4 * (DL.getTypeSizeInBits(Ty: EltTy) / 8));
560
561 SmallVector<Value *> Parts;
562 for (unsigned int I = 0, N = VT->getNumElements(); I < N; I += 4) {
563 Type *Ty = FixedVectorType::get(ElementType: EltTy, NumElts: N - I < 4 ? N - I : 4);
564 if (I > 0)
565 Offset = Builder.CreateAdd(LHS: Offset, RHS: Stride);
566 Parts.push_back(
567 Elt: emitRawLoad(Builder, Ty, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, RTI));
568 }
569
570 V = Parts.size() > 1 ? concatenateVectors(Builder, Vecs: Parts) : Parts[0];
571 } else
572 V = emitRawLoad(Builder, Ty: LI->getType(), Buffer: II->getOperand(i_nocapture: 0), Index, Offset,
573 RTI);
574
575 LI->replaceAllUsesWith(V);
576}
577
578namespace {
579/// Helper for building a `load.cbufferrow` intrinsic given a simple type.
580struct CBufferRowIntrin {
581 Intrinsic::ID IID;
582 Type *RetTy;
583 unsigned int EltSize;
584 unsigned int NumElts;
585
586 CBufferRowIntrin(const DataLayout &DL, Type *Ty) {
587 assert(Ty == Ty->getScalarType() && "Expected scalar type");
588
589 switch (DL.getTypeSizeInBits(Ty)) {
590 case 16:
591 IID = Intrinsic::dx_resource_load_cbufferrow_8;
592 RetTy = StructType::get(elt1: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty);
593 EltSize = 2;
594 NumElts = 8;
595 break;
596 case 32:
597 IID = Intrinsic::dx_resource_load_cbufferrow_4;
598 RetTy = StructType::get(elt1: Ty, elts: Ty, elts: Ty, elts: Ty);
599 EltSize = 4;
600 NumElts = 4;
601 break;
602 case 64:
603 IID = Intrinsic::dx_resource_load_cbufferrow_2;
604 RetTy = StructType::get(elt1: Ty, elts: Ty);
605 EltSize = 8;
606 NumElts = 2;
607 break;
608 default:
609 llvm_unreachable("Only 16, 32, and 64 bit types supported");
610 }
611 }
612};
613} // namespace
614
615static void createCBufferLoad(IntrinsicInst *II, LoadInst *LI,
616 dxil::ResourceTypeInfo &RTI) {
617 const DataLayout &DL = LI->getDataLayout();
618
619 Type *Ty = LI->getType();
620 assert(!isa<StructType>(Ty) && "Structs not handled yet");
621 CBufferRowIntrin Intrin(DL, Ty->getScalarType());
622
623 StringRef Name = LI->getName();
624 Value *Handle = II->getOperand(i_nocapture: 0);
625
626 IRBuilder<> Builder(LI);
627
628 ConstantInt *GlobalOffset =
629 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer
630 ? ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)
631 : dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1));
632 assert(GlobalOffset && "CBuffer getpointer index must be constant");
633
634 uint64_t GlobalOffsetVal = GlobalOffset->getZExtValue();
635 Value *CurrentRow = ConstantInt::get(
636 Ty: Builder.getInt32Ty(), V: GlobalOffsetVal / hlsl::CBufferRowSizeInBytes);
637 unsigned int CurrentIndex =
638 (GlobalOffsetVal % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
639
640 // Every object in a cbuffer either fits in a row or is aligned to a row. This
641 // means that only the very last pointer access can point into a row.
642 auto *LastGEP = dyn_cast<GEPOperator>(Val: LI->getPointerOperand());
643 if (!LastGEP) {
644 // If we don't have a GEP at all we're just accessing the resource through
645 // the result of getpointer directly.
646 assert(LI->getPointerOperand() == II &&
647 "Unexpected indirect access to resource without GEP");
648 } else {
649 Value *GEPOffset = traverseGEPOffsets(
650 DL, Builder, Ptr: LastGEP->getPointerOperand(), AccessSize: hlsl::CBufferRowSizeInBytes);
651 CurrentRow = Builder.CreateAdd(LHS: GEPOffset, RHS: CurrentRow);
652
653 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: LastGEP->getType()), 0);
654 if (LastGEP->accumulateConstantOffset(DL, Offset&: ConstantOffset)) {
655 APInt Remainder(DL.getIndexTypeSizeInBits(Ty: LastGEP->getType()),
656 hlsl::CBufferRowSizeInBytes);
657 APInt::udivrem(LHS: ConstantOffset, RHS: Remainder, Quotient&: ConstantOffset, Remainder);
658 CurrentRow = Builder.CreateAdd(
659 LHS: CurrentRow, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: ConstantOffset));
660 CurrentIndex += Remainder.udiv(RHS: Intrin.EltSize).getZExtValue();
661 } else {
662 assert(LastGEP->getNumIndices() == 1 &&
663 "Last GEP of cbuffer access is not array or struct access");
664 // We assume a non-constant access will be row-aligned. This is safe
665 // because arrays and structs are always row aligned, and accesses to
666 // vector elements will show up as a load of the vector followed by an
667 // extractelement.
668 CurrentRow = cast<ConstantInt>(Val: CurrentRow)->isZero()
669 ? *LastGEP->idx_begin()
670 : Builder.CreateAdd(LHS: CurrentRow, RHS: *LastGEP->idx_begin());
671 CurrentIndex = 0;
672 }
673 }
674
675 auto *CBufLoad = Builder.CreateIntrinsic(
676 RetTy: Intrin.RetTy, ID: Intrin.IID, Args: {Handle, CurrentRow}, FMFSource: nullptr, Name: Name + ".load");
677 auto *Elt =
678 Builder.CreateExtractValue(Agg: CBufLoad, Idxs: {CurrentIndex++}, Name: Name + ".extract");
679
680 // At this point we've loaded the first scalar of our result, but our original
681 // type may have been a vector.
682 unsigned int Remaining =
683 ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
684 if (Remaining == 0) {
685 // We only have a single element, so we're done.
686 Value *Result = Elt;
687
688 // However, if we loaded a <1 x T>, then we need to adjust the type.
689 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
690 assert(VT->getNumElements() == 1 && "Can't have multiple elements here");
691 Result = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: Result,
692 Idx: Builder.getInt32(C: 0), Name);
693 }
694 LI->replaceAllUsesWith(V: Result);
695 return;
696 }
697
698 // Walk each element and extract it, wrapping to new rows as needed.
699 SmallVector<Value *> Extracts{Elt};
700 while (Remaining--) {
701 CurrentIndex %= Intrin.NumElts;
702
703 if (CurrentIndex == 0) {
704 CurrentRow = Builder.CreateAdd(LHS: CurrentRow,
705 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
706 CBufLoad = Builder.CreateIntrinsic(RetTy: Intrin.RetTy, ID: Intrin.IID,
707 Args: {Handle, CurrentRow}, FMFSource: nullptr,
708 Name: Name + ".load");
709 }
710
711 Extracts.push_back(Elt: Builder.CreateExtractValue(Agg: CBufLoad, Idxs: {CurrentIndex++},
712 Name: Name + ".extract"));
713 }
714
715 // Finally, we build up the original loaded value.
716 Value *Result = PoisonValue::get(T: Ty);
717 for (int I = 0, E = Extracts.size(); I < E; ++I)
718 Result = Builder.CreateInsertElement(
719 Vec: Result, NewElt: Extracts[I], Idx: Builder.getInt32(C: I), Name: Name + formatv(Fmt: ".upto{}", Vals&: I));
720 LI->replaceAllUsesWith(V: Result);
721}
722
723static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI,
724 dxil::ResourceTypeInfo &RTI) {
725 switch (RTI.getResourceKind()) {
726 case dxil::ResourceKind::TypedBuffer:
727 return createTypedBufferLoad(II, LI, RTI);
728 case dxil::ResourceKind::RawBuffer:
729 case dxil::ResourceKind::StructuredBuffer:
730 return createRawLoads(II, LI, RTI);
731 case dxil::ResourceKind::CBuffer:
732 return createCBufferLoad(II, LI, RTI);
733 case dxil::ResourceKind::Texture1D:
734 case dxil::ResourceKind::Texture2D:
735 case dxil::ResourceKind::Texture2DMS:
736 case dxil::ResourceKind::Texture3D:
737 case dxil::ResourceKind::TextureCube:
738 case dxil::ResourceKind::Texture1DArray:
739 case dxil::ResourceKind::Texture2DArray:
740 case dxil::ResourceKind::Texture2DMSArray:
741 case dxil::ResourceKind::TextureCubeArray:
742 return createTextureLoad(II, LI, RTI);
743 case dxil::ResourceKind::FeedbackTexture2D:
744 case dxil::ResourceKind::FeedbackTexture2DArray:
745 case dxil::ResourceKind::TBuffer:
746 reportFatalUsageError(reason: "Load not yet implemented for resource type");
747 return;
748 case dxil::ResourceKind::Sampler:
749 case dxil::ResourceKind::RTAccelerationStructure:
750 case dxil::ResourceKind::Invalid:
751 case dxil::ResourceKind::NumEntries:
752 llvm_unreachable("Invalid resource kind for load");
753 }
754 llvm_unreachable("Unhandled case in switch");
755}
756
757static Instruction *getHandleOperand(Instruction *AI) {
758 if (auto *LI = dyn_cast<LoadInst>(Val: AI))
759 return dyn_cast<Instruction>(Val: LI->getPointerOperand());
760 if (auto *SI = dyn_cast<StoreInst>(Val: AI))
761 return dyn_cast<Instruction>(Val: SI->getPointerOperand());
762 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: AI))
763 return dyn_cast<Instruction>(Val: RMWI->getPointerOperand());
764 if (auto *II = dyn_cast<IntrinsicInst>(Val: AI))
765 if (II->getIntrinsicID() == Intrinsic::dx_resource_updatecounter)
766 return dyn_cast<Instruction>(Val: II->getArgOperand(i: 0));
767
768 return nullptr;
769}
770
771static const std::array<Intrinsic::ID, 2> HandleIntrins = {
772 Intrinsic::dx_resource_handlefrombinding,
773 Intrinsic::dx_resource_handlefromimplicitbinding,
774};
775
776static SmallVector<IntrinsicInst *> collectUsedHandles(Value *Ptr) {
777 SmallVector<Value *> Worklist = {Ptr};
778 SmallVector<IntrinsicInst *> Handles;
779 SmallSet<Value *, 4> VisitedPhis;
780
781 while (!Worklist.empty()) {
782 Value *X = Worklist.pop_back_val();
783
784 if (!X->getType()->isPointerTy() && !X->getType()->isTargetExtTy())
785 return {}; // Early exit on store/load into non-resource
786
787 if (auto *Phi = dyn_cast<PHINode>(Val: X)) {
788 if (VisitedPhis.contains(Ptr: X))
789 continue;
790 for (Use &V : Phi->incoming_values())
791 Worklist.push_back(Elt: V.get());
792 VisitedPhis.insert(Ptr: Phi);
793 } else if (auto *Select = dyn_cast<SelectInst>(Val: X))
794 for (Value *V : {Select->getTrueValue(), Select->getFalseValue()})
795 Worklist.push_back(Elt: V);
796 else if (auto *II = dyn_cast<IntrinsicInst>(Val: X)) {
797 Intrinsic::ID IID = II->getIntrinsicID();
798
799 if (IID == Intrinsic::dx_resource_getpointer)
800 Worklist.push_back(Elt: II->getArgOperand(/*Handle=*/i: 0));
801
802 if (llvm::is_contained(Range: HandleIntrins, Element: IID))
803 Handles.push_back(Elt: II);
804 }
805 }
806
807 return Handles;
808}
809
810static hlsl::Binding getHandleIntrinsicBinding(IntrinsicInst *Handle,
811 DXILResourceTypeMap &DRTM) {
812 assert(llvm::is_contained(HandleIntrins, Handle->getIntrinsicID()) &&
813 "Only expects a Handle as determined from collectUsedHandles.");
814
815 auto *HandleTy = cast<TargetExtType>(Val: Handle->getType());
816 dxil::ResourceClass Class = DRTM[HandleTy].getResourceClass();
817 uint32_t Space = cast<ConstantInt>(Val: Handle->getArgOperand(i: 0))->getZExtValue();
818 uint32_t LowerBound =
819 cast<ConstantInt>(Val: Handle->getArgOperand(i: 1))->getZExtValue();
820 uint32_t Size = cast<ConstantInt>(Val: Handle->getArgOperand(i: 2))->getZExtValue();
821 uint32_t UpperBound = Size == UINT32_MAX ? UINT32_MAX : LowerBound + Size - 1;
822
823 return hlsl::Binding(Class, Space, LowerBound, UpperBound, nullptr);
824}
825
826namespace {
827/// Helper for propagating the current handle and ptr indices.
828struct AccessIndices {
829 Value *GetPtrIdx;
830 Value *HandleIdx;
831
832 bool hasGetPtrIdx() { return GetPtrIdx != nullptr; }
833 bool hasHandleIdx() { return HandleIdx != nullptr; }
834};
835} // namespace
836
837// getAccessIndices traverses up the control flow that a ptr came from and
838// propagates back the indicies used to access the resource (AccessIndices):
839//
840// - GetPtrIdx is the index of dx.resource.getpointer
841// - HandleIdx is the index of dx.resource.handlefrom.*
842static AccessIndices
843getAccessIndices(Instruction *I, SmallSetVector<Instruction *, 16> &DeadInsts,
844 SmallDenseMap<PHINode *, PHINode *> &VisitedPhis) {
845 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
846 if (llvm::is_contained(Range: HandleIntrins, Element: II->getIntrinsicID())) {
847 DeadInsts.insert(X: II);
848 return {.GetPtrIdx: nullptr, .HandleIdx: II->getArgOperand(/*Index=*/i: 3)};
849 }
850
851 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer) {
852 auto *V = dyn_cast<Instruction>(Val: II->getArgOperand(/*Handle=*/i: 0));
853 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
854 assert(!AccessIdx.hasGetPtrIdx() &&
855 "Encountered multiple dx.resource.getpointers in ptr chain?");
856 AccessIdx.GetPtrIdx = II->getArgOperand(i: 1);
857
858 DeadInsts.insert(X: II);
859 return AccessIdx;
860 }
861 }
862
863 if (auto *Phi = dyn_cast<PHINode>(Val: I)) {
864 // If we're already building indices for this phi, return a ref to the phi
865 if (auto It = VisitedPhis.find(Val: Phi); It != VisitedPhis.end())
866 return {.GetPtrIdx: nullptr, .HandleIdx: It->second};
867
868 unsigned NumEdges = Phi->getNumIncomingValues();
869 assert(NumEdges != 0 && "Malformed Phi Node");
870
871 IRBuilder<> Builder(Phi);
872 std::unique_ptr<PHINode> GetPtrPhi(
873 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
874 std::unique_ptr<PHINode> HandlePhi(
875 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
876
877 // Register a ref to this phi for a recursive phi. This is safe to add to
878 // the map even if we end up deleting newly created phi below since we can't
879 // possibly have a constant value if we recursed.
880 if (Phi->getType()->isTargetExtTy())
881 VisitedPhis[Phi] = HandlePhi.get();
882
883 for (unsigned Idx = 0; Idx < NumEdges; Idx++) {
884 auto *BB = Phi->getIncomingBlock(i: Idx);
885 auto *V = dyn_cast<Instruction>(Val: Phi->getIncomingValue(i: Idx));
886 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
887 if (AccessIdx.hasGetPtrIdx())
888 GetPtrPhi->addIncoming(V: AccessIdx.GetPtrIdx, BB);
889 HandlePhi->addIncoming(V: AccessIdx.HandleIdx, BB);
890 }
891
892 Value *GetPtrIdx;
893 if (GetPtrPhi->getNumIncomingValues() == 0)
894 GetPtrIdx = nullptr;
895 else if (Value *ConstantGetPtr = GetPtrPhi->hasConstantValue())
896 GetPtrIdx = ConstantGetPtr;
897 else {
898 GetPtrIdx = GetPtrPhi.release();
899 Builder.Insert(V: GetPtrIdx);
900 }
901
902 Value *HandleIdx;
903 if (Value *ConstantHandle = HandlePhi->hasConstantValue())
904 HandleIdx = ConstantHandle;
905 else {
906 HandleIdx = HandlePhi.release();
907 Builder.Insert(V: HandleIdx);
908 }
909
910 DeadInsts.insert(X: Phi);
911 return {.GetPtrIdx: GetPtrIdx, .HandleIdx: HandleIdx};
912 }
913
914 if (auto *Select = dyn_cast<SelectInst>(Val: I)) {
915 auto *TrueV = dyn_cast<Instruction>(Val: Select->getTrueValue());
916 auto TrueAccessIdx = getAccessIndices(I: TrueV, DeadInsts, VisitedPhis);
917
918 auto *FalseV = dyn_cast<Instruction>(Val: Select->getFalseValue());
919 auto FalseAccessIdx = getAccessIndices(I: FalseV, DeadInsts, VisitedPhis);
920
921 IRBuilder<> Builder(Select);
922 Value *GetPtrSelect = nullptr;
923
924 if (TrueAccessIdx.hasGetPtrIdx() && FalseAccessIdx.hasGetPtrIdx())
925 GetPtrSelect =
926 Builder.CreateSelect(C: Select->getCondition(), True: TrueAccessIdx.GetPtrIdx,
927 False: FalseAccessIdx.GetPtrIdx);
928
929 auto *HandleSelect =
930 Builder.CreateSelect(C: Select->getCondition(), True: TrueAccessIdx.HandleIdx,
931 False: FalseAccessIdx.HandleIdx);
932 DeadInsts.insert(X: Select);
933 return {.GetPtrIdx: GetPtrSelect, .HandleIdx: HandleSelect};
934 }
935
936 llvm_unreachable("collectUsedHandles should assure this does not occur");
937}
938
939static void
940replaceHandleWithIndices(Instruction *Ptr, IntrinsicInst *OldHandle,
941 SmallSetVector<Instruction *, 16> &DeadInsts,
942 SmallDenseMap<PHINode *, PHINode *> &VisitedPhis) {
943 auto AccessIdx = getAccessIndices(I: Ptr, DeadInsts, VisitedPhis);
944 assert(AccessIdx.hasHandleIdx() &&
945 "Couldn't retrieve handle index. This is guaranteed by "
946 "getAccessIndices");
947
948 IRBuilder<> Builder(Ptr);
949 if (isa<PHINode>(Val: Ptr))
950 Builder.SetInsertPoint(Ptr->getParent()->getFirstNonPHIIt());
951 IntrinsicInst *Handle = cast<IntrinsicInst>(Val: OldHandle->clone());
952 Handle->setArgOperand(/*Index=*/i: 3, v: AccessIdx.HandleIdx);
953 Builder.Insert(I: Handle);
954
955 if (Ptr->getType()->isPointerTy()) {
956 assert(AccessIdx.hasGetPtrIdx() &&
957 "Couldn't retrieve getpointer index. This is guaranteed by "
958 "getAccessIndices");
959 auto *GetPtr = Builder.CreateIntrinsic(RetTy: Ptr->getType(),
960 ID: Intrinsic::dx_resource_getpointer,
961 Args: {Handle, AccessIdx.GetPtrIdx});
962 Ptr->replaceAllUsesWith(V: GetPtr);
963 } else {
964 assert(Ptr->getType()->isTargetExtTy() && !AccessIdx.hasGetPtrIdx() &&
965 "Unexpected resource access operand type");
966 Ptr->replaceAllUsesWith(V: Handle);
967 }
968
969 DeadInsts.insert(X: Ptr);
970}
971
972// Try to legalize dx.resource.handlefrom.*.binding and dx.resource.getpointer
973// calls with their respective index values and propagate the index values to
974// be used at resource access.
975//
976// If it can't be transformed to be legal then:
977//
978// Reports an error if a resource access is not guaranteed into a unique global
979// resource.
980//
981// Returns true if any changes are made.
982static bool legalizeResourceHandles(Function &F, DXILResourceTypeMap &DRTM) {
983 SmallSetVector<Instruction *, 16> DeadInsts;
984 SmallDenseMap<PHINode *, PHINode *> VisitedPhis;
985
986 for (BasicBlock &BB : make_early_inc_range(Range&: F)) {
987 for (Instruction &I : BB) {
988 if (auto *HandleOp = getHandleOperand(AI: &I)) {
989 SmallVector<IntrinsicInst *> Handles = collectUsedHandles(Ptr: HandleOp);
990 unsigned NumHandles = Handles.size();
991 if (NumHandles <= 1)
992 continue; // Legal, no-replacement required
993
994 bool SameGlobalBinding = true;
995 hlsl::Binding B = getHandleIntrinsicBinding(Handle: Handles[0], DRTM);
996 for (unsigned Idx = 1; Idx < NumHandles; Idx++)
997 SameGlobalBinding &=
998 (B == getHandleIntrinsicBinding(Handle: Handles[Idx], DRTM));
999
1000 if (!SameGlobalBinding) {
1001 diagnoseNonUniqueResourceAccess(I: &I, Handles);
1002 continue;
1003 }
1004
1005 replaceHandleWithIndices(Ptr: HandleOp, OldHandle: Handles[0], DeadInsts, VisitedPhis);
1006 }
1007 }
1008 }
1009
1010 bool MadeChanges = false;
1011
1012 // Set up the phis to track if they are erased below
1013 SmallVector<WeakTrackingVH> ResourcePhis;
1014 for (const auto &HandleToIndex : VisitedPhis)
1015 ResourcePhis.push_back(Elt: HandleToIndex.first);
1016
1017 for (auto *I : llvm::reverse(C&: DeadInsts))
1018 if (I->hasNUses(N: 0)) { // Handle can still be used outside of replaced path
1019 I->eraseFromParent();
1020 MadeChanges = true;
1021 }
1022
1023 // Any remaining phi nodes are now looped with another phi node and have no
1024 // other uses
1025 for (WeakTrackingVH &VH : ResourcePhis)
1026 if (VH) // True if not removed above or already in this loop
1027 MadeChanges |= RecursivelyDeleteDeadPHINode(PN: cast<PHINode>(Val&: VH));
1028
1029 return MadeChanges;
1030}
1031
1032static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
1033 SmallVector<User *> Worklist;
1034 for (User *U : II->users())
1035 Worklist.push_back(Elt: U);
1036
1037 SmallVector<Instruction *> DeadInsts;
1038 while (!Worklist.empty()) {
1039 User *U = Worklist.back();
1040 Worklist.pop_back();
1041
1042 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: U)) {
1043 for (User *U : GEP->users())
1044 Worklist.push_back(Elt: U);
1045 DeadInsts.push_back(Elt: GEP);
1046
1047 } else if (auto *SI = dyn_cast<StoreInst>(Val: U)) {
1048 assert(SI->getValueOperand() != II && "Pointer escaped!");
1049 createStoreIntrinsic(II, SI, RTI);
1050 DeadInsts.push_back(Elt: SI);
1051
1052 } else if (auto *LI = dyn_cast<LoadInst>(Val: U)) {
1053 createLoadIntrinsic(II, LI, RTI);
1054 DeadInsts.push_back(Elt: LI);
1055 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: U)) {
1056 createAtomicBinOpIntrinsic(II, AI, RTI);
1057 DeadInsts.push_back(Elt: AI);
1058 } else
1059 llvm_unreachable("Unhandled instruction - pointer escaped?");
1060 }
1061
1062 // Traverse the now-dead instructions in RPO and remove them.
1063 for (Instruction *Dead : llvm::reverse(C&: DeadInsts))
1064 Dead->eraseFromParent();
1065 II->eraseFromParent();
1066}
1067
1068static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) {
1069 SmallVector<std::pair<IntrinsicInst *, dxil::ResourceTypeInfo>> Resources;
1070 for (BasicBlock &BB : make_early_inc_range(Range&: F))
1071 for (Instruction &I : BB)
1072 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1073 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer ||
1074 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer) {
1075 auto *HandleTy = cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType());
1076 assert(
1077 (DRTM[HandleTy].isCBuffer() ||
1078 II->getIntrinsicID() != Intrinsic::dx_resource_getbasepointer) &&
1079 "dx_resource_getbasepointer should only be used by cbuffers");
1080 Resources.emplace_back(Args&: II, Args&: DRTM[HandleTy]);
1081 }
1082
1083 for (auto &[II, RI] : Resources)
1084 replaceAccess(II, RTI&: RI);
1085
1086 return !Resources.empty();
1087}
1088
1089PreservedAnalyses DXILResourceAccess::run(Function &F,
1090 FunctionAnalysisManager &FAM) {
1091 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1092 DXILResourceTypeMap *DRTM =
1093 MAMProxy.getCachedResult<DXILResourceTypeAnalysis>(IR&: *F.getParent());
1094 assert(DRTM && "DXILResourceTypeAnalysis must be available");
1095
1096 bool MadeHandleChanges = legalizeResourceHandles(F, DRTM&: *DRTM);
1097 bool MadeResourceChanges = transformResourcePointers(F, DRTM&: *DRTM);
1098 if (!(MadeHandleChanges || MadeResourceChanges))
1099 return PreservedAnalyses::all();
1100
1101 PreservedAnalyses PA;
1102 PA.preserve<DXILResourceTypeAnalysis>();
1103 PA.preserve<DominatorTreeAnalysis>();
1104 return PA;
1105}
1106
1107namespace {
1108class DXILResourceAccessLegacy : public FunctionPass {
1109public:
1110 bool runOnFunction(Function &F) override {
1111 DXILResourceTypeMap &DRTM =
1112 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
1113 bool MadeHandleChanges = legalizeResourceHandles(F, DRTM);
1114 bool MadeResourceChanges = transformResourcePointers(F, DRTM);
1115 return MadeHandleChanges || MadeResourceChanges;
1116 }
1117 StringRef getPassName() const override { return "DXIL Resource Access"; }
1118 DXILResourceAccessLegacy() : FunctionPass(ID) {}
1119
1120 static char ID; // Pass identification.
1121 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
1122 AU.addRequired<DXILResourceTypeWrapperPass>();
1123 AU.addPreserved<DominatorTreeWrapperPass>();
1124 }
1125};
1126char DXILResourceAccessLegacy::ID = 0;
1127} // end anonymous namespace
1128
1129INITIALIZE_PASS_BEGIN(DXILResourceAccessLegacy, DEBUG_TYPE,
1130 "DXIL Resource Access", false, false)
1131INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
1132INITIALIZE_PASS_END(DXILResourceAccessLegacy, DEBUG_TYPE,
1133 "DXIL Resource Access", false, false)
1134
1135FunctionPass *llvm::createDXILResourceAccessLegacyPass() {
1136 return new DXILResourceAccessLegacy();
1137}
1138