1//===- DXILResourceAccess.cpp - Resource access via load/store ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DXILResourceAccess.h"
10#include "DirectX.h"
11#include "llvm/ADT/DenseMap.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/Analysis/DXILResource.h"
14#include "llvm/Analysis/VectorUtils.h"
15#include "llvm/Frontend/HLSL/HLSLResource.h"
16#include "llvm/IR/BasicBlock.h"
17#include "llvm/IR/Dominators.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/Instruction.h"
20#include "llvm/IR/Instructions.h"
21#include "llvm/IR/IntrinsicInst.h"
22#include "llvm/IR/Intrinsics.h"
23#include "llvm/IR/IntrinsicsDirectX.h"
24#include "llvm/IR/LLVMContext.h"
25#include "llvm/IR/User.h"
26#include "llvm/IR/ValueHandle.h"
27#include "llvm/InitializePasses.h"
28#include "llvm/Support/DXILABI.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/FormatVariadic.h"
31#include "llvm/Transforms/Utils/Local.h"
32#include "llvm/Transforms/Utils/ValueMapper.h"
33#include <optional>
34
35#define DEBUG_TYPE "dxil-resource-access"
36
37using namespace llvm;
38
39[[noreturn]] static void
40diagnoseNonUniqueResourceAccess(Instruction *I,
41 ArrayRef<IntrinsicInst *> Handles) {
42 LLVMContext &Context = I->getContext();
43 std::string InstStr;
44 raw_string_ostream InstOS(InstStr);
45 I->print(O&: InstOS);
46 Context.diagnose(
47 DI: DiagnosticInfoGeneric("At resource access:" + Twine(InstStr), DS_Note));
48
49 for (auto *Handle : Handles) {
50 std::string HandleStr;
51 raw_string_ostream HandleOS(HandleStr);
52 Handle->print(O&: HandleOS);
53 Context.diagnose(DI: DiagnosticInfoGeneric(
54 "Uses resource handle:" + Twine(HandleStr), DS_Note));
55 }
56 report_fatal_error(
57 reason: "Resource access is not guaranteed to map to a unique global resource",
58 /*gen_crash_diag=*/false);
59}
60
61static Value *traverseGEPOffsets(const DataLayout &DL, IRBuilder<> &Builder,
62 Value *Ptr, uint64_t AccessSize) {
63 Value *Offset = nullptr;
64
65 while (Ptr) {
66 if ([[maybe_unused]] auto *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
67 assert((II->getIntrinsicID() == Intrinsic::dx_resource_getpointer ||
68 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer) &&
69 "Resource access through unexpected intrinsic");
70 return Offset ? Offset : ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
71 }
72
73 auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
74 assert(GEP && "Resource access through unexpected instruction");
75
76 unsigned NumIndices = GEP->getNumIndices();
77 uint64_t IndexScale = DL.getTypeAllocSize(Ty: GEP->getSourceElementType());
78 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: GEP->getType()), 0);
79 Value *GEPOffset;
80 if (GEP->accumulateConstantOffset(DL, Offset&: ConstantOffset)) {
81 // We have a constant offset (in bytes).
82 GEPOffset =
83 ConstantInt::get(Ty: DL.getIndexType(PtrTy: GEP->getType()), V: ConstantOffset);
84 IndexScale = 1;
85 } else if (NumIndices == 1) {
86 // If we have a single index we're indexing into a top level array. This
87 // generally only happens with cbuffers.
88 GEPOffset = *GEP->idx_begin();
89 } else if (NumIndices == 2) {
90 // If we have two indices, this should be an access through a pointer.
91 auto *IndexIt = GEP->idx_begin();
92 assert(cast<ConstantInt>(IndexIt)->getZExtValue() == 0 &&
93 "GEP is not indexing through pointer");
94 GEPOffset = *(++IndexIt);
95 } else
96 llvm_unreachable("Unhandled GEP structure for resource access");
97
98 uint64_t ElemSize = AccessSize;
99 if (!(IndexScale % ElemSize)) {
100 // If our scale is an exact multiple of the access size, adjust the
101 // scaling to avoid an unnecessary division.
102 IndexScale /= ElemSize;
103 ElemSize = 1;
104 }
105 if (IndexScale != 1)
106 GEPOffset = Builder.CreateMul(
107 LHS: GEPOffset, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: IndexScale));
108 if (ElemSize != 1)
109 GEPOffset = Builder.CreateUDiv(
110 LHS: GEPOffset, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: ElemSize));
111
112 Offset = Offset ? Builder.CreateAdd(LHS: Offset, RHS: GEPOffset) : GEPOffset;
113 Ptr = GEP->getPointerOperand();
114 }
115
116 llvm_unreachable("GEP of null pointer?");
117}
118
119static void createTypedBufferStore(IntrinsicInst *II, StoreInst *SI,
120 dxil::ResourceTypeInfo &RTI) {
121 const DataLayout &DL = SI->getDataLayout();
122 IRBuilder<> Builder(SI);
123 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
124 Type *ScalarType = ContainedType->getScalarType();
125 Type *LoadType = StructType::get(elt1: ContainedType, elts: Builder.getInt1Ty());
126
127 Value *V = SI->getValueOperand();
128 if (V->getType() == ContainedType) {
129 // V is already the right type.
130 assert(SI->getPointerOperand() == II &&
131 "Store of whole element has mismatched address to store to");
132 } else if (V->getType() == ScalarType) {
133 // We're storing a scalar, so we need to load the current value and only
134 // replace the relevant part.
135 auto *Load = Builder.CreateIntrinsic(
136 RetTy: LoadType, ID: Intrinsic::dx_resource_load_typedbuffer,
137 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1)});
138 auto *Struct = Builder.CreateExtractValue(Agg: Load, Idxs: {0});
139
140 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
141 Value *Offset =
142 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
143 V = Builder.CreateInsertElement(Vec: Struct, NewElt: V, Idx: Offset);
144 } else {
145 llvm_unreachable("Store to typed resource has invalid type");
146 }
147
148 auto *Inst = Builder.CreateIntrinsic(
149 RetTy: Builder.getVoidTy(), ID: Intrinsic::dx_resource_store_typedbuffer,
150 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1), V});
151 SI->replaceAllUsesWith(V: Inst);
152}
153
154/// Build a zero-initialized offset operand matching the shape of the given
155/// coordinate operand. Accesses through `operator[]` never have offsets.
156static Value *getNullOffsetsFor(IRBuilder<> &Builder, Value *Coords) {
157 Type *CoordTy = Coords->getType();
158 Type *OffsetTy;
159 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: CoordTy))
160 OffsetTy =
161 FixedVectorType::get(ElementType: Builder.getInt32Ty(), NumElts: VecTy->getNumElements());
162 else
163 OffsetTy = Builder.getInt32Ty();
164 return Constant::getNullValue(Ty: OffsetTy);
165}
166
167static void createTextureStore(IntrinsicInst *II, StoreInst *SI,
168 dxil::ResourceTypeInfo &RTI) {
169 const DataLayout &DL = SI->getDataLayout();
170 IRBuilder<> Builder(SI);
171 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
172 Type *ScalarType = ContainedType->getScalarType();
173
174 Value *Handle = II->getOperand(i_nocapture: 0);
175 Value *Coords = II->getOperand(i_nocapture: 1);
176
177 Value *V = SI->getValueOperand();
178 if (V->getType() == ContainedType) {
179 // V is already the right type.
180 assert(SI->getPointerOperand() == II &&
181 "Store of whole element has mismatched address to store to");
182 } else if (V->getType() == ScalarType) {
183 // We're storing a scalar, so we need to load the current value and only
184 // replace the relevant part. For operator[] the mip level and the offsets
185 // are always zero; DXILOpLowering drops the mip level for UAVs.
186 Value *MipLevel = Builder.getInt32(C: 0);
187 Value *Offsets = getNullOffsetsFor(Builder, Coords);
188 auto *Load = Builder.CreateIntrinsic(RetTy: ContainedType,
189 ID: Intrinsic::dx_resource_load_level,
190 Args: {Handle, Coords, MipLevel, Offsets});
191
192 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
193 Value *Offset =
194 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
195 V = Builder.CreateInsertElement(Vec: Load, NewElt: V, Idx: Offset);
196 } else {
197 llvm_unreachable("Store to texture resource has invalid type");
198 }
199
200 auto *Inst = Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(),
201 ID: Intrinsic::dx_resource_store_texture,
202 Args: {Handle, Coords, V});
203 SI->replaceAllUsesWith(V: Inst);
204}
205
206static void emitRawStore(IRBuilder<> &Builder, Value *Buffer, Value *Index,
207 Value *Offset, Value *V, dxil::ResourceTypeInfo &RTI) {
208 // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
209 // entirely into the index.
210 if (!RTI.isStruct()) {
211 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
212 if (!ConstantOffset || !ConstantOffset->isZero())
213 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
214 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
215 }
216
217 Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(),
218 ID: Intrinsic::dx_resource_store_rawbuffer,
219 Args: {Buffer, Index, Offset, V});
220}
221
222static void createRawStores(IntrinsicInst *II, StoreInst *SI,
223 dxil::ResourceTypeInfo &RTI) {
224 const DataLayout &DL = SI->getDataLayout();
225 IRBuilder<> Builder(SI);
226
227 Value *V = SI->getValueOperand();
228 assert(!V->getType()->isAggregateType() &&
229 "Resource store should be scalar or vector type");
230
231 Value *Index = II->getOperand(i_nocapture: 1);
232 // The offset for the rawbuffer load and store ops is always in bytes.
233 uint64_t AccessSize = 1;
234 Value *Offset =
235 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
236
237 auto *VT = dyn_cast<FixedVectorType>(Val: V->getType());
238 if (VT && VT->getNumElements() > 4) {
239 // Split into stores of at most 4 elements.
240 Type *EltTy = VT->getElementType();
241 Value *Stride = ConstantInt::get(Ty: Builder.getInt32Ty(),
242 V: 4 * (DL.getTypeSizeInBits(Ty: EltTy) / 8));
243
244 SmallVector<int, 4> Indices;
245 for (unsigned int I = 0, N = VT->getNumElements(); I < N; I += 4) {
246 if (I > 0)
247 Offset = Builder.CreateAdd(LHS: Offset, RHS: Stride);
248
249 for (unsigned int J = I, E = std::min(a: N, b: J + 4); J < E; ++J)
250 Indices.push_back(Elt: J);
251 Value *Part = Builder.CreateShuffleVector(V, Mask: Indices);
252 emitRawStore(Builder, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, V: Part, RTI);
253
254 Indices.clear();
255 }
256 } else
257 emitRawStore(Builder, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, V, RTI);
258}
259
260static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI,
261 dxil::ResourceTypeInfo &RTI) {
262 switch (RTI.getResourceKind()) {
263 case dxil::ResourceKind::TypedBuffer:
264 return createTypedBufferStore(II, SI, RTI);
265 case dxil::ResourceKind::RawBuffer:
266 case dxil::ResourceKind::StructuredBuffer:
267 return createRawStores(II, SI, RTI);
268 case dxil::ResourceKind::Texture1D:
269 case dxil::ResourceKind::Texture2D:
270 case dxil::ResourceKind::Texture3D:
271 case dxil::ResourceKind::Texture1DArray:
272 case dxil::ResourceKind::Texture2DArray:
273 return createTextureStore(II, SI, RTI);
274 case dxil::ResourceKind::Texture2DMS:
275 case dxil::ResourceKind::Texture2DMSArray:
276 case dxil::ResourceKind::TextureCube:
277 case dxil::ResourceKind::TextureCubeArray:
278 case dxil::ResourceKind::FeedbackTexture2D:
279 case dxil::ResourceKind::FeedbackTexture2DArray:
280 reportFatalUsageError(
281 reason: "DXIL Store not implemented for this texture resource kind");
282 return;
283 case dxil::ResourceKind::CBuffer:
284 case dxil::ResourceKind::Sampler:
285 case dxil::ResourceKind::TBuffer:
286 case dxil::ResourceKind::RTAccelerationStructure:
287 case dxil::ResourceKind::Invalid:
288 case dxil::ResourceKind::NumEntries:
289 llvm_unreachable("Invalid resource kind for store");
290 }
291 llvm_unreachable("Unhandled case in switch");
292}
293
294static std::optional<dxil::AtomicBinOpCode>
295getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) {
296 switch (BinOp) {
297 case AtomicRMWInst::Add:
298 return dxil::AtomicBinOpCode::Add;
299 case AtomicRMWInst::And:
300 return dxil::AtomicBinOpCode::And;
301 case AtomicRMWInst::Or:
302 return dxil::AtomicBinOpCode::Or;
303 case AtomicRMWInst::Xor:
304 return dxil::AtomicBinOpCode::Xor;
305 case AtomicRMWInst::Min:
306 return dxil::AtomicBinOpCode::IMin;
307 case AtomicRMWInst::Max:
308 return dxil::AtomicBinOpCode::IMax;
309 case AtomicRMWInst::UMin:
310 return dxil::AtomicBinOpCode::UMin;
311 case AtomicRMWInst::UMax:
312 return dxil::AtomicBinOpCode::UMax;
313 case AtomicRMWInst::Xchg:
314 return dxil::AtomicBinOpCode::Exchange;
315 case AtomicRMWInst::Sub:
316 case AtomicRMWInst::Nand:
317 case AtomicRMWInst::FAdd:
318 case AtomicRMWInst::FSub:
319 case AtomicRMWInst::FMax:
320 case AtomicRMWInst::FMin:
321 case AtomicRMWInst::FMaximum:
322 case AtomicRMWInst::FMinimum:
323 case AtomicRMWInst::FMaximumNum:
324 case AtomicRMWInst::FMinimumNum:
325 case AtomicRMWInst::UIncWrap:
326 case AtomicRMWInst::UDecWrap:
327 case AtomicRMWInst::USubCond:
328 case AtomicRMWInst::USubSat:
329 case AtomicRMWInst::BAD_BINOP:
330 return std::nullopt;
331 }
332 llvm_unreachable("Unhandled atomicrmw operation");
333}
334
335// Compute the (coord0, coord1) pair for a buffer resource atomic operation.
336// Non-struct buffers (RawBuffer or TypedBuffer) fold the byte offset into the
337// index and leave coord1 poison. Only StructuredBuffer atomics use both a
338// struct index and a byte offset.
339static std::pair<Value *, Value *>
340getAtomicResourceCoords(IntrinsicInst *II, Value *PointerOperand,
341 dxil::ResourceTypeInfo &RTI, IRBuilder<> &Builder,
342 const DataLayout &DL) {
343 Value *Index = II->getOperand(i_nocapture: 1);
344
345 // The offset for the rawbuffer load/store/atomic ops is always in bytes.
346 uint64_t AccessSize = 1;
347 Value *Offset = traverseGEPOffsets(DL, Builder, Ptr: PointerOperand, AccessSize);
348
349 if (!RTI.isStruct()) {
350 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
351 if (!ConstantOffset || !ConstantOffset->isZero())
352 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
353 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
354 }
355
356 return {Index, Offset};
357}
358
359// The coordinates of a texture access are a scalar or a vector with one element
360// per texture dimension, including the array slice if there is one. These map
361// directly onto the coordinate operands of the atomic ops.
362static SmallVector<Value *, 3> getTextureAtomicCoords(IntrinsicInst *II,
363 IRBuilder<> &Builder) {
364 Value *Coords = II->getOperand(i_nocapture: 1);
365 SmallVector<Value *, 3> CoordArgs;
366 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Coords->getType())) {
367 assert(VecTy->getNumElements() <= 3 && "Too many texture coordinates");
368 for (unsigned I = 0, E = VecTy->getNumElements(); I != E; ++I)
369 CoordArgs.push_back(Elt: Builder.CreateExtractElement(Vec: Coords, Idx: I));
370 } else {
371 CoordArgs.push_back(Elt: Coords);
372 }
373 return CoordArgs;
374}
375
376static void emitAtomicBinOp(IRBuilder<> &Builder, AtomicRMWInst *AI,
377 Value *Handle, ArrayRef<Value *> Coords) {
378 assert(!Coords.empty() && Coords.size() <= 3 &&
379 "Atomic operations take between one and three coordinates");
380
381 std::optional<dxil::AtomicBinOpCode> BinOpCode =
382 getAtomicBinOpCode(BinOp: AI->getOperation());
383 if (!BinOpCode) {
384 reportFatalUsageError(reason: "DXIL resource atomicrmw operation not implemented");
385 return;
386 }
387
388 // DXIL has no floating-point atomic op. A float exchange only moves the bit
389 // pattern, so cast the value to an integer of the same width, exchange, and
390 // cast the result back. This matches what DXC emits.
391 Value *Val = AI->getValOperand();
392 Type *ValTy = Val->getType();
393 Type *OpTy = ValTy;
394 if (ValTy->isFloatingPointTy()) {
395 OpTy = Builder.getIntNTy(N: ValTy->getPrimitiveSizeInBits());
396 Val = Builder.CreateBitCast(V: Val, DestTy: OpTy);
397 }
398
399 SmallVector<Value *, 6> Args{
400 Handle, Builder.getInt32(C: static_cast<uint32_t>(*BinOpCode))};
401 append_range(C&: Args, R&: Coords);
402 Args.append(NumInputs: 3 - Coords.size(), Elt: PoisonValue::get(T: Builder.getInt32Ty()));
403 Args.push_back(Elt: Val);
404
405 // Emit the target-independent intrinsic; DXILOpLowering lowers it to the
406 // DXIL `AtomicBinOp` op and handles the target-ext-typed handle cast via
407 // its `createTmpHandleCast` bookkeeping.
408 Value *Result =
409 Builder.CreateIntrinsic(RetTy: OpTy, ID: Intrinsic::dx_resource_atomic_binop, Args);
410
411 if (OpTy != ValTy)
412 Result = Builder.CreateBitCast(V: Result, DestTy: ValTy);
413
414 AI->replaceAllUsesWith(V: Result);
415}
416
417static void createBufferAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
418 dxil::ResourceTypeInfo &RTI) {
419 const DataLayout &DL = AI->getDataLayout();
420 IRBuilder<> Builder(AI);
421 auto [Index, Offset] =
422 getAtomicResourceCoords(II, PointerOperand: AI->getPointerOperand(), RTI, Builder, DL);
423
424 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: {Index, Offset});
425}
426
427static void createTextureAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
428 dxil::ResourceTypeInfo &RTI) {
429 // A texture atomic operates on a whole texel, so a multi-component texel has
430 // no single addressable component. A scalar float texel is allowed, because
431 // emitAtomicBinOp exchanges its bit pattern as an integer.
432 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
433 if (!ContainedType->isIntegerTy() && !ContainedType->isFloatingPointTy()) {
434 reportFatalUsageError(reason: "DXIL atomicrmw requires a texture resource with a "
435 "scalar element type");
436 return;
437 }
438
439 IRBuilder<> Builder(AI);
440
441 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0),
442 Coords: getTextureAtomicCoords(II, Builder));
443}
444
445static void emitAtomicCompareExchange(IRBuilder<> &Builder,
446 AtomicCmpXchgInst *AI, Value *Handle,
447 ArrayRef<Value *> Coords) {
448 assert(!Coords.empty() && Coords.size() <= 3 &&
449 "Atomic operations take between one and three coordinates");
450
451 Value *Compare = AI->getCompareOperand();
452 Value *NewValue = AI->getNewValOperand();
453
454 SmallVector<Value *, 6> Args{Handle};
455 append_range(C&: Args, R&: Coords);
456 Args.append(NumInputs: 3 - Coords.size(), Elt: PoisonValue::get(T: Builder.getInt32Ty()));
457 Args.push_back(Elt: Compare);
458 Args.push_back(Elt: NewValue);
459
460 Value *Original = Builder.CreateIntrinsic(
461 RetTy: NewValue->getType(), ID: Intrinsic::dx_resource_atomic_compare_exchange,
462 Args);
463
464 // `cmpxchg` yields a { original, success } pair, but the DXIL op returns
465 // only the original value. DXIL has no way to express the success flag, and
466 // no HLSL builtin reads it, so replace the users of the pair directly
467 // instead of building it again. No pass after this one removes dead code.
468 SmallVector<ExtractValueInst *> Extracts;
469 for (User *U : AI->users()) {
470 auto *EV = dyn_cast<ExtractValueInst>(Val: U);
471 if (!EV || EV->getIndices()[0] != 0)
472 reportFatalUsageError(reason: "DXIL cmpxchg provides only the original value");
473 Extracts.push_back(Elt: EV);
474 }
475
476 for (ExtractValueInst *EV : Extracts) {
477 EV->replaceAllUsesWith(V: Original);
478 EV->eraseFromParent();
479 }
480}
481
482static void createBufferAtomicCompareExchange(IntrinsicInst *II,
483 AtomicCmpXchgInst *AI,
484 dxil::ResourceTypeInfo &RTI) {
485 const DataLayout &DL = AI->getDataLayout();
486 IRBuilder<> Builder(AI);
487 auto [Index, Offset] =
488 getAtomicResourceCoords(II, PointerOperand: AI->getPointerOperand(), RTI, Builder, DL);
489
490 emitAtomicCompareExchange(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: {Index, Offset});
491}
492
493// `cmpxchg` operands are always integers, so unlike atomicrmw there is no
494// float element type to convert here. The element type must be scalar.
495static void createTextureAtomicCompareExchange(IntrinsicInst *II,
496 AtomicCmpXchgInst *AI,
497 dxil::ResourceTypeInfo &RTI) {
498 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
499 if (!ContainedType->isIntegerTy() && !ContainedType->isFloatingPointTy()) {
500 reportFatalUsageError(reason: "DXIL cmpxchg requires a texture resource with a "
501 "scalar element type");
502 return;
503 }
504
505 IRBuilder<> Builder(AI);
506
507 emitAtomicCompareExchange(Builder, AI, Handle: II->getOperand(i_nocapture: 0),
508 Coords: getTextureAtomicCoords(II, Builder));
509}
510
511static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
512 dxil::ResourceTypeInfo &RTI) {
513 switch (RTI.getResourceKind()) {
514 case dxil::ResourceKind::TypedBuffer:
515 case dxil::ResourceKind::RawBuffer:
516 case dxil::ResourceKind::StructuredBuffer:
517 return createBufferAtomicBinOp(II, AI, RTI);
518 case dxil::ResourceKind::Texture1D:
519 case dxil::ResourceKind::Texture2D:
520 case dxil::ResourceKind::Texture3D:
521 case dxil::ResourceKind::Texture1DArray:
522 case dxil::ResourceKind::Texture2DArray:
523 return createTextureAtomicBinOp(II, AI, RTI);
524 case dxil::ResourceKind::Texture2DMS:
525 case dxil::ResourceKind::Texture2DMSArray:
526 case dxil::ResourceKind::TextureCube:
527 case dxil::ResourceKind::TextureCubeArray:
528 case dxil::ResourceKind::FeedbackTexture2D:
529 case dxil::ResourceKind::FeedbackTexture2DArray:
530 reportFatalUsageError(
531 reason: "DXIL atomicrmw not implemented for this texture resource kind");
532 return;
533 case dxil::ResourceKind::CBuffer:
534 case dxil::ResourceKind::Sampler:
535 case dxil::ResourceKind::TBuffer:
536 reportFatalUsageError(
537 reason: "DXIL atomicrmw not implemented for this resource type");
538 return;
539 case dxil::ResourceKind::RTAccelerationStructure:
540 case dxil::ResourceKind::Invalid:
541 case dxil::ResourceKind::NumEntries:
542 llvm_unreachable("Invalid resource kind for atomicrmw");
543 }
544 llvm_unreachable("Unhandled case in switch");
545}
546
547static void createAtomicCompareExchangeIntrinsic(IntrinsicInst *II,
548 AtomicCmpXchgInst *AI,
549 dxil::ResourceTypeInfo &RTI) {
550 switch (RTI.getResourceKind()) {
551 case dxil::ResourceKind::TypedBuffer:
552 case dxil::ResourceKind::RawBuffer:
553 case dxil::ResourceKind::StructuredBuffer:
554 return createBufferAtomicCompareExchange(II, AI, RTI);
555 case dxil::ResourceKind::Texture1D:
556 case dxil::ResourceKind::Texture2D:
557 case dxil::ResourceKind::Texture3D:
558 case dxil::ResourceKind::Texture1DArray:
559 case dxil::ResourceKind::Texture2DArray:
560 return createTextureAtomicCompareExchange(II, AI, RTI);
561 case dxil::ResourceKind::Texture2DMS:
562 case dxil::ResourceKind::Texture2DMSArray:
563 case dxil::ResourceKind::TextureCube:
564 case dxil::ResourceKind::TextureCubeArray:
565 case dxil::ResourceKind::FeedbackTexture2D:
566 case dxil::ResourceKind::FeedbackTexture2DArray:
567 reportFatalUsageError(
568 reason: "DXIL cmpxchg not implemented for this texture resource kind");
569 return;
570 case dxil::ResourceKind::CBuffer:
571 case dxil::ResourceKind::Sampler:
572 case dxil::ResourceKind::TBuffer:
573 reportFatalUsageError(
574 reason: "DXIL cmpxchg not implemented for this resource type");
575 return;
576 case dxil::ResourceKind::RTAccelerationStructure:
577 case dxil::ResourceKind::Invalid:
578 case dxil::ResourceKind::NumEntries:
579 llvm_unreachable("Invalid resource kind for cmpxchg");
580 }
581 llvm_unreachable("Unhandled case in switch");
582}
583
584static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI,
585 dxil::ResourceTypeInfo &RTI) {
586 const DataLayout &DL = LI->getDataLayout();
587 IRBuilder<> Builder(LI);
588 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
589 Type *LoadType = StructType::get(elt1: ContainedType, elts: Builder.getInt1Ty());
590
591 Value *V =
592 Builder.CreateIntrinsic(RetTy: LoadType, ID: Intrinsic::dx_resource_load_typedbuffer,
593 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1)});
594 V = Builder.CreateExtractValue(Agg: V, Idxs: {0});
595
596 Type *ScalarType = ContainedType->getScalarType();
597 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
598 Value *Offset =
599 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
600 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
601 if (!ConstantOffset || !ConstantOffset->isZero())
602 V = Builder.CreateExtractElement(Vec: V, Idx: Offset);
603
604 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a
605 // shufflevector), then make sure we're maintaining the resulting type.
606 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType()))
607 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(Val: V->getType()))
608 V = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: V,
609 Idx: Builder.getInt32(C: 0));
610
611 LI->replaceAllUsesWith(V);
612}
613
614static void createTextureLoad(IntrinsicInst *II, LoadInst *LI,
615 dxil::ResourceTypeInfo &RTI) {
616 const DataLayout &DL = LI->getDataLayout();
617 IRBuilder<> Builder(LI);
618 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
619
620 Value *Handle = II->getOperand(i_nocapture: 0);
621 Value *Coords = II->getOperand(i_nocapture: 1);
622
623 // For operator[], mip level is 0.
624 Value *MipLevel = Builder.getInt32(C: 0);
625
626 // For operator[], offsets are zero.
627 Value *Offsets = getNullOffsetsFor(Builder, Coords);
628
629 Value *V =
630 Builder.CreateIntrinsic(RetTy: ContainedType, ID: Intrinsic::dx_resource_load_level,
631 Args: {Handle, Coords, MipLevel, Offsets});
632
633 Type *ScalarType = ContainedType->getScalarType();
634 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
635 Value *Offset =
636 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
637 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
638 if (!ConstantOffset || !ConstantOffset->isZero())
639 V = Builder.CreateExtractElement(Vec: V, Idx: Offset);
640
641 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a
642 // shufflevector), then make sure we're maintaining the resulting type.
643 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType()))
644 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(Val: V->getType()))
645 V = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: V,
646 Idx: Builder.getInt32(C: 0));
647
648 LI->replaceAllUsesWith(V);
649}
650
651static Value *emitRawLoad(IRBuilder<> &Builder, Type *Ty, Value *Buffer,
652 Value *Index, Value *Offset,
653 dxil::ResourceTypeInfo &RTI) {
654 // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
655 // entirely into the index.
656 if (!RTI.isStruct()) {
657 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
658 if (!ConstantOffset || !ConstantOffset->isZero())
659 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
660 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
661 }
662
663 // The load intrinsic includes the bit for CheckAccessFullyMapped, so we need
664 // to add that to the return type.
665 Type *TypeWithCheck = StructType::get(elt1: Ty, elts: Builder.getInt1Ty());
666 Value *V = Builder.CreateIntrinsic(RetTy: TypeWithCheck,
667 ID: Intrinsic::dx_resource_load_rawbuffer,
668 Args: {Buffer, Index, Offset});
669 return Builder.CreateExtractValue(Agg: V, Idxs: {0});
670}
671
672static void createRawLoads(IntrinsicInst *II, LoadInst *LI,
673 dxil::ResourceTypeInfo &RTI) {
674 const DataLayout &DL = LI->getDataLayout();
675 IRBuilder<> Builder(LI);
676
677 Value *Index = II->getOperand(i_nocapture: 1);
678 // The offset for the rawbuffer load and store ops is always in bytes.
679 uint64_t AccessSize = 1;
680 Value *Offset =
681 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
682
683 // TODO: We could make this handle aggregates by walking the structure and
684 // handling each field individually, but we don't ever generate code that
685 // would hit that so it seems superfluous.
686 assert(!LI->getType()->isAggregateType() &&
687 "Resource load should be scalar or vector type");
688
689 Value *V;
690 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType())) {
691 // Split into loads of at most 4 elements.
692 Type *EltTy = VT->getElementType();
693 Value *Stride = ConstantInt::get(Ty: Builder.getInt32Ty(),
694 V: 4 * (DL.getTypeSizeInBits(Ty: EltTy) / 8));
695
696 SmallVector<Value *> Parts;
697 for (unsigned int I = 0, N = VT->getNumElements(); I < N; I += 4) {
698 Type *Ty = FixedVectorType::get(ElementType: EltTy, NumElts: N - I < 4 ? N - I : 4);
699 if (I > 0)
700 Offset = Builder.CreateAdd(LHS: Offset, RHS: Stride);
701 Parts.push_back(
702 Elt: emitRawLoad(Builder, Ty, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, RTI));
703 }
704
705 V = Parts.size() > 1 ? concatenateVectors(Builder, Vecs: Parts) : Parts[0];
706 } else
707 V = emitRawLoad(Builder, Ty: LI->getType(), Buffer: II->getOperand(i_nocapture: 0), Index, Offset,
708 RTI);
709
710 LI->replaceAllUsesWith(V);
711}
712
713namespace {
714/// Helper for building a `load.cbufferrow` intrinsic given a simple type.
715struct CBufferRowIntrin {
716 Intrinsic::ID IID;
717 Type *RetTy;
718 unsigned int EltSize;
719 unsigned int NumElts;
720
721 CBufferRowIntrin(const DataLayout &DL, Type *Ty) {
722 assert(Ty == Ty->getScalarType() && "Expected scalar type");
723
724 switch (DL.getTypeSizeInBits(Ty)) {
725 case 16:
726 IID = Intrinsic::dx_resource_load_cbufferrow_8;
727 RetTy = StructType::get(elt1: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty);
728 EltSize = 2;
729 NumElts = 8;
730 break;
731 case 32:
732 IID = Intrinsic::dx_resource_load_cbufferrow_4;
733 RetTy = StructType::get(elt1: Ty, elts: Ty, elts: Ty, elts: Ty);
734 EltSize = 4;
735 NumElts = 4;
736 break;
737 case 64:
738 IID = Intrinsic::dx_resource_load_cbufferrow_2;
739 RetTy = StructType::get(elt1: Ty, elts: Ty);
740 EltSize = 8;
741 NumElts = 2;
742 break;
743 default:
744 llvm_unreachable("Only 16, 32, and 64 bit types supported");
745 }
746 }
747};
748} // namespace
749
750static void createCBufferLoad(IntrinsicInst *II, LoadInst *LI,
751 dxil::ResourceTypeInfo &RTI) {
752 const DataLayout &DL = LI->getDataLayout();
753
754 Type *Ty = LI->getType();
755 assert(!isa<StructType>(Ty) && "Structs not handled yet");
756 CBufferRowIntrin Intrin(DL, Ty->getScalarType());
757
758 StringRef Name = LI->getName();
759 Value *Handle = II->getOperand(i_nocapture: 0);
760
761 IRBuilder<> Builder(LI);
762
763 ConstantInt *GlobalOffset =
764 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer
765 ? ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)
766 : dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1));
767 assert(GlobalOffset && "CBuffer getpointer index must be constant");
768
769 uint64_t GlobalOffsetVal = GlobalOffset->getZExtValue();
770 Value *CurrentRow = ConstantInt::get(
771 Ty: Builder.getInt32Ty(), V: GlobalOffsetVal / hlsl::CBufferRowSizeInBytes);
772 unsigned int CurrentIndex =
773 (GlobalOffsetVal % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
774
775 // Every object in a cbuffer either fits in a row or is aligned to a row. This
776 // means that only the very last pointer access can point into a row.
777 auto *LastGEP = dyn_cast<GEPOperator>(Val: LI->getPointerOperand());
778 if (!LastGEP) {
779 // If we don't have a GEP at all we're just accessing the resource through
780 // the result of getpointer directly.
781 assert(LI->getPointerOperand() == II &&
782 "Unexpected indirect access to resource without GEP");
783 } else {
784 Value *GEPOffset = traverseGEPOffsets(
785 DL, Builder, Ptr: LastGEP->getPointerOperand(), AccessSize: hlsl::CBufferRowSizeInBytes);
786 CurrentRow = Builder.CreateAdd(LHS: GEPOffset, RHS: CurrentRow);
787
788 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: LastGEP->getType()), 0);
789 if (LastGEP->accumulateConstantOffset(DL, Offset&: ConstantOffset)) {
790 APInt Remainder(DL.getIndexTypeSizeInBits(Ty: LastGEP->getType()),
791 hlsl::CBufferRowSizeInBytes);
792 APInt::udivrem(LHS: ConstantOffset, RHS: Remainder, Quotient&: ConstantOffset, Remainder);
793 CurrentRow = Builder.CreateAdd(
794 LHS: CurrentRow, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: ConstantOffset));
795 CurrentIndex += Remainder.udiv(RHS: Intrin.EltSize).getZExtValue();
796 } else {
797 assert(LastGEP->getNumIndices() == 1 &&
798 "Last GEP of cbuffer access is not array or struct access");
799 // We assume a non-constant access will be row-aligned. This is safe
800 // because arrays and structs are always row aligned, and accesses to
801 // vector elements will show up as a load of the vector followed by an
802 // extractelement.
803 CurrentRow = cast<ConstantInt>(Val: CurrentRow)->isZero()
804 ? *LastGEP->idx_begin()
805 : Builder.CreateAdd(LHS: CurrentRow, RHS: *LastGEP->idx_begin());
806 CurrentIndex = 0;
807 }
808 }
809
810 auto *CBufLoad = Builder.CreateIntrinsic(
811 RetTy: Intrin.RetTy, ID: Intrin.IID, Args: {Handle, CurrentRow}, FMFSource: nullptr, Name: Name + ".load");
812 auto *Elt =
813 Builder.CreateExtractValue(Agg: CBufLoad, Idxs: {CurrentIndex++}, Name: Name + ".extract");
814
815 // At this point we've loaded the first scalar of our result, but our original
816 // type may have been a vector.
817 unsigned int Remaining =
818 ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
819 if (Remaining == 0) {
820 // We only have a single element, so we're done.
821 Value *Result = Elt;
822
823 // However, if we loaded a <1 x T>, then we need to adjust the type.
824 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
825 assert(VT->getNumElements() == 1 && "Can't have multiple elements here");
826 Result = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: Result,
827 Idx: Builder.getInt32(C: 0), Name);
828 }
829 LI->replaceAllUsesWith(V: Result);
830 return;
831 }
832
833 // Walk each element and extract it, wrapping to new rows as needed.
834 SmallVector<Value *> Extracts{Elt};
835 while (Remaining--) {
836 CurrentIndex %= Intrin.NumElts;
837
838 if (CurrentIndex == 0) {
839 CurrentRow = Builder.CreateAdd(LHS: CurrentRow,
840 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
841 CBufLoad = Builder.CreateIntrinsic(RetTy: Intrin.RetTy, ID: Intrin.IID,
842 Args: {Handle, CurrentRow}, FMFSource: nullptr,
843 Name: Name + ".load");
844 }
845
846 Extracts.push_back(Elt: Builder.CreateExtractValue(Agg: CBufLoad, Idxs: {CurrentIndex++},
847 Name: Name + ".extract"));
848 }
849
850 // Finally, we build up the original loaded value.
851 Value *Result = PoisonValue::get(T: Ty);
852 for (int I = 0, E = Extracts.size(); I < E; ++I)
853 Result = Builder.CreateInsertElement(
854 Vec: Result, NewElt: Extracts[I], Idx: Builder.getInt32(C: I), Name: Name + formatv(Fmt: ".upto{}", Vals&: I));
855 LI->replaceAllUsesWith(V: Result);
856}
857
858static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI,
859 dxil::ResourceTypeInfo &RTI) {
860 switch (RTI.getResourceKind()) {
861 case dxil::ResourceKind::TypedBuffer:
862 return createTypedBufferLoad(II, LI, RTI);
863 case dxil::ResourceKind::RawBuffer:
864 case dxil::ResourceKind::StructuredBuffer:
865 return createRawLoads(II, LI, RTI);
866 case dxil::ResourceKind::CBuffer:
867 return createCBufferLoad(II, LI, RTI);
868 case dxil::ResourceKind::Texture1D:
869 case dxil::ResourceKind::Texture2D:
870 case dxil::ResourceKind::Texture2DMS:
871 case dxil::ResourceKind::Texture3D:
872 case dxil::ResourceKind::TextureCube:
873 case dxil::ResourceKind::Texture1DArray:
874 case dxil::ResourceKind::Texture2DArray:
875 case dxil::ResourceKind::Texture2DMSArray:
876 case dxil::ResourceKind::TextureCubeArray:
877 return createTextureLoad(II, LI, RTI);
878 case dxil::ResourceKind::FeedbackTexture2D:
879 case dxil::ResourceKind::FeedbackTexture2DArray:
880 case dxil::ResourceKind::TBuffer:
881 reportFatalUsageError(reason: "Load not yet implemented for resource type");
882 return;
883 case dxil::ResourceKind::Sampler:
884 case dxil::ResourceKind::RTAccelerationStructure:
885 case dxil::ResourceKind::Invalid:
886 case dxil::ResourceKind::NumEntries:
887 llvm_unreachable("Invalid resource kind for load");
888 }
889 llvm_unreachable("Unhandled case in switch");
890}
891
892static Instruction *getHandleOperand(Instruction *AI) {
893 if (auto *LI = dyn_cast<LoadInst>(Val: AI))
894 return dyn_cast<Instruction>(Val: LI->getPointerOperand());
895 if (auto *SI = dyn_cast<StoreInst>(Val: AI))
896 return dyn_cast<Instruction>(Val: SI->getPointerOperand());
897 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: AI))
898 return dyn_cast<Instruction>(Val: RMWI->getPointerOperand());
899 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: AI))
900 return dyn_cast<Instruction>(Val: CXI->getPointerOperand());
901 if (auto *II = dyn_cast<IntrinsicInst>(Val: AI))
902 if (II->getIntrinsicID() == Intrinsic::dx_resource_updatecounter)
903 return dyn_cast<Instruction>(Val: II->getArgOperand(i: 0));
904
905 return nullptr;
906}
907
908static const std::array<Intrinsic::ID, 2> HandleIntrins = {
909 Intrinsic::dx_resource_handlefrombinding,
910 Intrinsic::dx_resource_handlefromimplicitbinding,
911};
912
913static SmallVector<IntrinsicInst *> collectUsedHandles(Value *Ptr) {
914 SmallVector<Value *> Worklist = {Ptr};
915 SmallVector<IntrinsicInst *> Handles;
916 SmallPtrSet<Value *, 4> VisitedPhis;
917
918 while (!Worklist.empty()) {
919 Value *X = Worklist.pop_back_val();
920
921 if (!X->getType()->isPointerTy() && !X->getType()->isTargetExtTy())
922 return {}; // Early exit on store/load into non-resource
923
924 if (auto *Phi = dyn_cast<PHINode>(Val: X)) {
925 if (VisitedPhis.contains(Ptr: X))
926 continue;
927 for (Use &V : Phi->incoming_values())
928 Worklist.push_back(Elt: V.get());
929 VisitedPhis.insert(Ptr: Phi);
930 } else if (auto *Select = dyn_cast<SelectInst>(Val: X))
931 for (Value *V : {Select->getTrueValue(), Select->getFalseValue()})
932 Worklist.push_back(Elt: V);
933 else if (auto *II = dyn_cast<IntrinsicInst>(Val: X)) {
934 Intrinsic::ID IID = II->getIntrinsicID();
935
936 if (IID == Intrinsic::dx_resource_getpointer)
937 Worklist.push_back(Elt: II->getArgOperand(/*Handle=*/i: 0));
938
939 if (llvm::is_contained(Range: HandleIntrins, Element: IID))
940 Handles.push_back(Elt: II);
941 }
942 }
943
944 return Handles;
945}
946
947static hlsl::Binding getHandleIntrinsicBinding(IntrinsicInst *Handle,
948 DXILResourceTypeMap &DRTM) {
949 assert(llvm::is_contained(HandleIntrins, Handle->getIntrinsicID()) &&
950 "Only expects a Handle as determined from collectUsedHandles.");
951
952 auto *HandleTy = cast<TargetExtType>(Val: Handle->getType());
953 dxil::ResourceClass Class = DRTM[HandleTy].getResourceClass();
954 uint32_t Space = cast<ConstantInt>(Val: Handle->getArgOperand(i: 0))->getZExtValue();
955 uint32_t LowerBound =
956 cast<ConstantInt>(Val: Handle->getArgOperand(i: 1))->getZExtValue();
957 uint32_t Size = cast<ConstantInt>(Val: Handle->getArgOperand(i: 2))->getZExtValue();
958 uint32_t UpperBound = Size == UINT32_MAX ? UINT32_MAX : LowerBound + Size - 1;
959
960 return hlsl::Binding(Class, Space, LowerBound, UpperBound, nullptr);
961}
962
963namespace {
964/// Helper for propagating the current handle and ptr indices.
965struct AccessIndices {
966 Value *GetPtrIdx;
967 Value *HandleIdx;
968
969 bool hasGetPtrIdx() { return GetPtrIdx != nullptr; }
970 bool hasHandleIdx() { return HandleIdx != nullptr; }
971};
972} // namespace
973
974// getAccessIndices traverses up the control flow that a ptr came from and
975// propagates back the indicies used to access the resource (AccessIndices):
976//
977// - GetPtrIdx is the index of dx.resource.getpointer
978// - HandleIdx is the index of dx.resource.handlefrom.*
979static AccessIndices
980getAccessIndices(Instruction *I, SmallSetVector<Instruction *, 16> &DeadInsts,
981 SmallDenseMap<PHINode *, PHINode *> &VisitedPhis) {
982 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
983 if (llvm::is_contained(Range: HandleIntrins, Element: II->getIntrinsicID())) {
984 DeadInsts.insert(X: II);
985 return {.GetPtrIdx: nullptr, .HandleIdx: II->getArgOperand(/*Index=*/i: 3)};
986 }
987
988 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer) {
989 auto *V = dyn_cast<Instruction>(Val: II->getArgOperand(/*Handle=*/i: 0));
990 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
991 assert(!AccessIdx.hasGetPtrIdx() &&
992 "Encountered multiple dx.resource.getpointers in ptr chain?");
993 AccessIdx.GetPtrIdx = II->getArgOperand(i: 1);
994
995 DeadInsts.insert(X: II);
996 return AccessIdx;
997 }
998 }
999
1000 if (auto *Phi = dyn_cast<PHINode>(Val: I)) {
1001 // If we're already building indices for this phi, return a ref to the phi
1002 if (auto It = VisitedPhis.find(Val: Phi); It != VisitedPhis.end())
1003 return {.GetPtrIdx: nullptr, .HandleIdx: It->second};
1004
1005 unsigned NumEdges = Phi->getNumIncomingValues();
1006 assert(NumEdges != 0 && "Malformed Phi Node");
1007
1008 IRBuilder<> Builder(Phi);
1009 std::unique_ptr<PHINode> GetPtrPhi(
1010 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
1011 std::unique_ptr<PHINode> HandlePhi(
1012 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
1013
1014 // Register a ref to this phi for a recursive phi. This is safe to add to
1015 // the map even if we end up deleting newly created phi below since we can't
1016 // possibly have a constant value if we recursed.
1017 if (Phi->getType()->isTargetExtTy())
1018 VisitedPhis[Phi] = HandlePhi.get();
1019
1020 for (unsigned Idx = 0; Idx < NumEdges; Idx++) {
1021 auto *BB = Phi->getIncomingBlock(i: Idx);
1022 auto *V = dyn_cast<Instruction>(Val: Phi->getIncomingValue(i: Idx));
1023 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
1024 if (AccessIdx.hasGetPtrIdx())
1025 GetPtrPhi->addIncoming(V: AccessIdx.GetPtrIdx, BB);
1026 HandlePhi->addIncoming(V: AccessIdx.HandleIdx, BB);
1027 }
1028
1029 Value *GetPtrIdx;
1030 if (GetPtrPhi->getNumIncomingValues() == 0)
1031 GetPtrIdx = nullptr;
1032 else if (Value *ConstantGetPtr = GetPtrPhi->hasConstantValue())
1033 GetPtrIdx = ConstantGetPtr;
1034 else {
1035 GetPtrIdx = GetPtrPhi.release();
1036 Builder.Insert(V: GetPtrIdx);
1037 }
1038
1039 Value *HandleIdx;
1040 if (Value *ConstantHandle = HandlePhi->hasConstantValue())
1041 HandleIdx = ConstantHandle;
1042 else {
1043 HandleIdx = HandlePhi.release();
1044 Builder.Insert(V: HandleIdx);
1045 }
1046
1047 DeadInsts.insert(X: Phi);
1048 return {.GetPtrIdx: GetPtrIdx, .HandleIdx: HandleIdx};
1049 }
1050
1051 if (auto *Select = dyn_cast<SelectInst>(Val: I)) {
1052 auto *TrueV = dyn_cast<Instruction>(Val: Select->getTrueValue());
1053 auto TrueAccessIdx = getAccessIndices(I: TrueV, DeadInsts, VisitedPhis);
1054
1055 auto *FalseV = dyn_cast<Instruction>(Val: Select->getFalseValue());
1056 auto FalseAccessIdx = getAccessIndices(I: FalseV, DeadInsts, VisitedPhis);
1057
1058 IRBuilder<> Builder(Select);
1059 Value *GetPtrSelect = nullptr;
1060
1061 if (TrueAccessIdx.hasGetPtrIdx() && FalseAccessIdx.hasGetPtrIdx())
1062 GetPtrSelect =
1063 Builder.CreateSelect(C: Select->getCondition(), True: TrueAccessIdx.GetPtrIdx,
1064 False: FalseAccessIdx.GetPtrIdx);
1065
1066 auto *HandleSelect =
1067 Builder.CreateSelect(C: Select->getCondition(), True: TrueAccessIdx.HandleIdx,
1068 False: FalseAccessIdx.HandleIdx);
1069 DeadInsts.insert(X: Select);
1070 return {.GetPtrIdx: GetPtrSelect, .HandleIdx: HandleSelect};
1071 }
1072
1073 llvm_unreachable("collectUsedHandles should assure this does not occur");
1074}
1075
1076static void
1077replaceHandleWithIndices(Instruction *Ptr, IntrinsicInst *OldHandle,
1078 SmallSetVector<Instruction *, 16> &DeadInsts,
1079 SmallDenseMap<PHINode *, PHINode *> &VisitedPhis) {
1080 auto AccessIdx = getAccessIndices(I: Ptr, DeadInsts, VisitedPhis);
1081 assert(AccessIdx.hasHandleIdx() &&
1082 "Couldn't retrieve handle index. This is guaranteed by "
1083 "getAccessIndices");
1084
1085 IRBuilder<> Builder(Ptr);
1086 if (isa<PHINode>(Val: Ptr))
1087 Builder.SetInsertPoint(Ptr->getParent()->getFirstNonPHIIt());
1088 IntrinsicInst *Handle = cast<IntrinsicInst>(Val: OldHandle->clone());
1089 Handle->setArgOperand(/*Index=*/i: 3, v: AccessIdx.HandleIdx);
1090 Builder.Insert(I: Handle);
1091
1092 if (Ptr->getType()->isPointerTy()) {
1093 assert(AccessIdx.hasGetPtrIdx() &&
1094 "Couldn't retrieve getpointer index. This is guaranteed by "
1095 "getAccessIndices");
1096 auto *GetPtr = Builder.CreateIntrinsic(RetTy: Ptr->getType(),
1097 ID: Intrinsic::dx_resource_getpointer,
1098 Args: {Handle, AccessIdx.GetPtrIdx});
1099 Ptr->replaceAllUsesWith(V: GetPtr);
1100 } else {
1101 assert(Ptr->getType()->isTargetExtTy() && !AccessIdx.hasGetPtrIdx() &&
1102 "Unexpected resource access operand type");
1103 Ptr->replaceAllUsesWith(V: Handle);
1104 }
1105
1106 DeadInsts.insert(X: Ptr);
1107}
1108
1109// Try to legalize dx.resource.handlefrom.*.binding and dx.resource.getpointer
1110// calls with their respective index values and propagate the index values to
1111// be used at resource access.
1112//
1113// If it can't be transformed to be legal then:
1114//
1115// Reports an error if a resource access is not guaranteed into a unique global
1116// resource.
1117//
1118// Returns true if any changes are made.
1119static bool legalizeResourceHandles(Function &F, DXILResourceTypeMap &DRTM) {
1120 SmallSetVector<Instruction *, 16> DeadInsts;
1121 SmallDenseMap<PHINode *, PHINode *> VisitedPhis;
1122
1123 for (BasicBlock &BB : make_early_inc_range(Range&: F)) {
1124 for (Instruction &I : BB) {
1125 if (auto *HandleOp = getHandleOperand(AI: &I)) {
1126 SmallVector<IntrinsicInst *> Handles = collectUsedHandles(Ptr: HandleOp);
1127 unsigned NumHandles = Handles.size();
1128 if (NumHandles <= 1)
1129 continue; // Legal, no-replacement required
1130
1131 bool SameGlobalBinding = true;
1132 hlsl::Binding B = getHandleIntrinsicBinding(Handle: Handles[0], DRTM);
1133 for (unsigned Idx = 1; Idx < NumHandles; Idx++)
1134 SameGlobalBinding &=
1135 (B == getHandleIntrinsicBinding(Handle: Handles[Idx], DRTM));
1136
1137 if (!SameGlobalBinding)
1138 diagnoseNonUniqueResourceAccess(I: &I, Handles);
1139
1140 replaceHandleWithIndices(Ptr: HandleOp, OldHandle: Handles[0], DeadInsts, VisitedPhis);
1141 }
1142 }
1143 }
1144
1145 bool MadeChanges = false;
1146
1147 // Set up the phis to track if they are erased below
1148 SmallVector<WeakTrackingVH> ResourcePhis;
1149 for (const auto &HandleToIndex : VisitedPhis)
1150 ResourcePhis.push_back(Elt: HandleToIndex.first);
1151
1152 for (auto *I : llvm::reverse(C&: DeadInsts))
1153 if (I->hasNUses(N: 0)) { // Handle can still be used outside of replaced path
1154 I->eraseFromParent();
1155 MadeChanges = true;
1156 }
1157
1158 // Any remaining phi nodes are now looped with another phi node and have no
1159 // other uses
1160 for (WeakTrackingVH &VH : ResourcePhis)
1161 if (VH) // True if not removed above or already in this loop
1162 MadeChanges |= RecursivelyDeleteDeadPHINode(PN: cast<PHINode>(Val&: VH));
1163
1164 return MadeChanges;
1165}
1166
1167static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
1168 SmallVector<User *> Worklist;
1169 for (User *U : II->users())
1170 Worklist.push_back(Elt: U);
1171
1172 SmallVector<Instruction *> DeadInsts;
1173 while (!Worklist.empty()) {
1174 User *U = Worklist.back();
1175 Worklist.pop_back();
1176
1177 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: U)) {
1178 for (User *U : GEP->users())
1179 Worklist.push_back(Elt: U);
1180 DeadInsts.push_back(Elt: GEP);
1181
1182 } else if (auto *SI = dyn_cast<StoreInst>(Val: U)) {
1183 assert(SI->getValueOperand() != II && "Pointer escaped!");
1184 createStoreIntrinsic(II, SI, RTI);
1185 DeadInsts.push_back(Elt: SI);
1186
1187 } else if (auto *LI = dyn_cast<LoadInst>(Val: U)) {
1188 createLoadIntrinsic(II, LI, RTI);
1189 DeadInsts.push_back(Elt: LI);
1190 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: U)) {
1191 createAtomicBinOpIntrinsic(II, AI, RTI);
1192 DeadInsts.push_back(Elt: AI);
1193 } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: U)) {
1194 createAtomicCompareExchangeIntrinsic(II, AI: CXI, RTI);
1195 DeadInsts.push_back(Elt: CXI);
1196 } else
1197 llvm_unreachable("Unhandled instruction - pointer escaped?");
1198 }
1199
1200 // Traverse the now-dead instructions in RPO and remove them.
1201 for (Instruction *Dead : llvm::reverse(C&: DeadInsts))
1202 Dead->eraseFromParent();
1203 II->eraseFromParent();
1204}
1205
1206static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) {
1207 SmallVector<std::pair<IntrinsicInst *, dxil::ResourceTypeInfo>> Resources;
1208 for (BasicBlock &BB : make_early_inc_range(Range&: F))
1209 for (Instruction &I : BB)
1210 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1211 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer ||
1212 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer) {
1213 auto *HandleTy = cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType());
1214 assert(
1215 (DRTM[HandleTy].isCBuffer() ||
1216 II->getIntrinsicID() != Intrinsic::dx_resource_getbasepointer) &&
1217 "dx_resource_getbasepointer should only be used by cbuffers");
1218 Resources.emplace_back(Args&: II, Args&: DRTM[HandleTy]);
1219 }
1220
1221 for (auto &[II, RI] : Resources)
1222 replaceAccess(II, RTI&: RI);
1223
1224 return !Resources.empty();
1225}
1226
1227PreservedAnalyses DXILResourceAccess::run(Function &F,
1228 FunctionAnalysisManager &FAM) {
1229 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1230 DXILResourceTypeMap *DRTM =
1231 MAMProxy.getCachedResult<DXILResourceTypeAnalysis>(IR&: *F.getParent());
1232 assert(DRTM && "DXILResourceTypeAnalysis must be available");
1233
1234 bool MadeHandleChanges = legalizeResourceHandles(F, DRTM&: *DRTM);
1235 bool MadeResourceChanges = transformResourcePointers(F, DRTM&: *DRTM);
1236 if (!(MadeHandleChanges || MadeResourceChanges))
1237 return PreservedAnalyses::all();
1238
1239 PreservedAnalyses PA;
1240 PA.preserve<DXILResourceTypeAnalysis>();
1241 PA.preserve<DominatorTreeAnalysis>();
1242 return PA;
1243}
1244
1245namespace {
1246class DXILResourceAccessLegacy : public FunctionPass {
1247public:
1248 bool runOnFunction(Function &F) override {
1249 DXILResourceTypeMap &DRTM =
1250 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
1251 bool MadeHandleChanges = legalizeResourceHandles(F, DRTM);
1252 bool MadeResourceChanges = transformResourcePointers(F, DRTM);
1253 return MadeHandleChanges || MadeResourceChanges;
1254 }
1255 StringRef getPassName() const override { return "DXIL Resource Access"; }
1256 DXILResourceAccessLegacy() : FunctionPass(ID) {}
1257
1258 static char ID; // Pass identification.
1259 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
1260 AU.addRequired<DXILResourceTypeWrapperPass>();
1261 AU.addPreserved<DominatorTreeWrapperPass>();
1262 }
1263};
1264char DXILResourceAccessLegacy::ID = 0;
1265} // end anonymous namespace
1266
1267INITIALIZE_PASS_BEGIN(DXILResourceAccessLegacy, DEBUG_TYPE,
1268 "DXIL Resource Access", false, false)
1269INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
1270INITIALIZE_PASS_END(DXILResourceAccessLegacy, DEBUG_TYPE,
1271 "DXIL Resource Access", false, false)
1272
1273FunctionPass *llvm::createDXILResourceAccessLegacyPass() {
1274 return new DXILResourceAccessLegacy();
1275}
1276