1//===- DXILResourceAccess.cpp - Resource access via load/store ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DXILResourceAccess.h"
10#include "DirectX.h"
11#include "llvm/ADT/DenseMap.h"
12#include "llvm/ADT/MapVector.h"
13#include "llvm/ADT/SetVector.h"
14#include "llvm/Analysis/DXILResource.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/Analysis/VectorUtils.h"
17#include "llvm/Frontend/HLSL/HLSLResource.h"
18#include "llvm/IR/BasicBlock.h"
19#include "llvm/IR/Dominators.h"
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/Instruction.h"
22#include "llvm/IR/Instructions.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsDirectX.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/IR/ProfDataUtils.h"
28#include "llvm/IR/User.h"
29#include "llvm/IR/ValueHandle.h"
30#include "llvm/InitializePasses.h"
31#include "llvm/Support/DXILABI.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/FormatVariadic.h"
34#include "llvm/Support/KnownBits.h"
35#include "llvm/Transforms/Utils/BasicBlockUtils.h"
36#include "llvm/Transforms/Utils/Local.h"
37#include "llvm/Transforms/Utils/ValueMapper.h"
38#include <optional>
39
40#define DEBUG_TYPE "dxil-resource-access"
41
42using namespace llvm;
43
44[[noreturn]] static void
45diagnoseNonUniqueResourceAccess(Instruction *I,
46 ArrayRef<IntrinsicInst *> Handles) {
47 LLVMContext &Context = I->getContext();
48 std::string InstStr;
49 raw_string_ostream InstOS(InstStr);
50 I->print(O&: InstOS);
51 Context.diagnose(
52 DI: DiagnosticInfoGeneric("At resource access:" + Twine(InstStr), DS_Note));
53
54 for (auto *Handle : Handles) {
55 std::string HandleStr;
56 raw_string_ostream HandleOS(HandleStr);
57 Handle->print(O&: HandleOS);
58 Context.diagnose(DI: DiagnosticInfoGeneric(
59 "Uses resource handle:" + Twine(HandleStr), DS_Note));
60 }
61 report_fatal_error(
62 reason: "Resource access is not guaranteed to map to a unique global resource",
63 /*gen_crash_diag=*/false);
64}
65
66static Value *traverseGEPOffsets(const DataLayout &DL, IRBuilder<> &Builder,
67 Value *Ptr, uint64_t AccessSize) {
68 Value *Offset = nullptr;
69
70 while (Ptr) {
71 if ([[maybe_unused]] auto *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
72 assert((II->getIntrinsicID() == Intrinsic::dx_resource_getpointer ||
73 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer) &&
74 "Resource access through unexpected intrinsic");
75 return Offset ? Offset : ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
76 }
77
78 auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
79 assert(GEP && "Resource access through unexpected instruction");
80
81 unsigned NumIndices = GEP->getNumIndices();
82 uint64_t IndexScale = DL.getTypeAllocSize(Ty: GEP->getSourceElementType());
83 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: GEP->getType()), 0);
84 Value *GEPOffset;
85 if (GEP->accumulateConstantOffset(DL, Offset&: ConstantOffset)) {
86 // We have a constant offset (in bytes).
87 GEPOffset =
88 ConstantInt::get(Ty: DL.getIndexType(PtrTy: GEP->getType()), V: ConstantOffset);
89 IndexScale = 1;
90 } else if (NumIndices == 1) {
91 // If we have a single index we're indexing into a top level array. This
92 // generally only happens with cbuffers.
93 GEPOffset = *GEP->idx_begin();
94 } else if (NumIndices == 2) {
95 // If we have two indices, this should be an access through a pointer.
96 auto *IndexIt = GEP->idx_begin();
97 assert(cast<ConstantInt>(IndexIt)->getZExtValue() == 0 &&
98 "GEP is not indexing through pointer");
99 GEPOffset = *(++IndexIt);
100 } else
101 llvm_unreachable("Unhandled GEP structure for resource access");
102
103 uint64_t ElemSize = AccessSize;
104 if (!(IndexScale % ElemSize)) {
105 // If our scale is an exact multiple of the access size, adjust the
106 // scaling to avoid an unnecessary division.
107 IndexScale /= ElemSize;
108 ElemSize = 1;
109 }
110 if (IndexScale != 1)
111 GEPOffset = Builder.CreateMul(
112 LHS: GEPOffset, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: IndexScale));
113 if (ElemSize != 1)
114 GEPOffset = Builder.CreateUDiv(
115 LHS: GEPOffset, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: ElemSize));
116
117 Offset = Offset ? Builder.CreateAdd(LHS: Offset, RHS: GEPOffset) : GEPOffset;
118 Ptr = GEP->getPointerOperand();
119 }
120
121 llvm_unreachable("GEP of null pointer?");
122}
123
124static void createTypedBufferStore(IntrinsicInst *II, StoreInst *SI,
125 dxil::ResourceTypeInfo &RTI) {
126 const DataLayout &DL = SI->getDataLayout();
127 IRBuilder<> Builder(SI);
128 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
129 Type *ScalarType = ContainedType->getScalarType();
130 Type *LoadType = StructType::get(elt1: ContainedType, elts: Builder.getInt1Ty());
131
132 Value *V = SI->getValueOperand();
133 if (V->getType() == ContainedType) {
134 // V is already the right type.
135 assert(SI->getPointerOperand() == II &&
136 "Store of whole element has mismatched address to store to");
137 } else if (V->getType() == ScalarType) {
138 // We're storing a scalar, so we need to load the current value and only
139 // replace the relevant part.
140 auto *Load = Builder.CreateIntrinsic(
141 RetTy: LoadType, ID: Intrinsic::dx_resource_load_typedbuffer,
142 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1)});
143 auto *Struct = Builder.CreateExtractValue(Agg: Load, Idxs: {0});
144
145 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
146 Value *Offset =
147 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
148 V = Builder.CreateInsertElement(Vec: Struct, NewElt: V, Idx: Offset);
149 } else {
150 llvm_unreachable("Store to typed resource has invalid type");
151 }
152
153 auto *Inst = Builder.CreateIntrinsic(
154 RetTy: Builder.getVoidTy(), ID: Intrinsic::dx_resource_store_typedbuffer,
155 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1), V});
156 SI->replaceAllUsesWith(V: Inst);
157}
158
159/// Build a zero-initialized offset operand matching the shape of the given
160/// coordinate operand. Accesses through `operator[]` never have offsets.
161static Value *getNullOffsetsFor(IRBuilder<> &Builder, Value *Coords) {
162 Type *CoordTy = Coords->getType();
163 Type *OffsetTy;
164 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: CoordTy))
165 OffsetTy =
166 FixedVectorType::get(ElementType: Builder.getInt32Ty(), NumElts: VecTy->getNumElements());
167 else
168 OffsetTy = Builder.getInt32Ty();
169 return Constant::getNullValue(Ty: OffsetTy);
170}
171
172static void createTextureStore(IntrinsicInst *II, StoreInst *SI,
173 dxil::ResourceTypeInfo &RTI) {
174 const DataLayout &DL = SI->getDataLayout();
175 IRBuilder<> Builder(SI);
176 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
177 Type *ScalarType = ContainedType->getScalarType();
178
179 Value *Handle = II->getOperand(i_nocapture: 0);
180 Value *Coords = II->getOperand(i_nocapture: 1);
181
182 Value *V = SI->getValueOperand();
183 if (V->getType() == ContainedType) {
184 // V is already the right type.
185 assert(SI->getPointerOperand() == II &&
186 "Store of whole element has mismatched address to store to");
187 } else if (V->getType() == ScalarType) {
188 // We're storing a scalar, so we need to load the current value and only
189 // replace the relevant part. For operator[] the mip level and the offsets
190 // are always zero; DXILOpLowering drops the mip level for UAVs.
191 Value *MipLevel = Builder.getInt32(C: 0);
192 Value *Offsets = getNullOffsetsFor(Builder, Coords);
193 auto *Load = Builder.CreateIntrinsic(RetTy: ContainedType,
194 ID: Intrinsic::dx_resource_load_level,
195 Args: {Handle, Coords, MipLevel, Offsets});
196
197 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
198 Value *Offset =
199 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
200 V = Builder.CreateInsertElement(Vec: Load, NewElt: V, Idx: Offset);
201 } else {
202 llvm_unreachable("Store to texture resource has invalid type");
203 }
204
205 auto *Inst = Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(),
206 ID: Intrinsic::dx_resource_store_texture,
207 Args: {Handle, Coords, V});
208 SI->replaceAllUsesWith(V: Inst);
209}
210
211static void emitRawStore(IRBuilder<> &Builder, Value *Buffer, Value *Index,
212 Value *Offset, Value *V, dxil::ResourceTypeInfo &RTI) {
213 // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
214 // entirely into the index.
215 if (!RTI.isStruct()) {
216 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
217 if (!ConstantOffset || !ConstantOffset->isZero())
218 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
219 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
220 }
221
222 Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(),
223 ID: Intrinsic::dx_resource_store_rawbuffer,
224 Args: {Buffer, Index, Offset, V});
225}
226
227static void createRawStores(IntrinsicInst *II, StoreInst *SI,
228 dxil::ResourceTypeInfo &RTI) {
229 const DataLayout &DL = SI->getDataLayout();
230 IRBuilder<> Builder(SI);
231
232 Value *V = SI->getValueOperand();
233 assert(!V->getType()->isAggregateType() &&
234 "Resource store should be scalar or vector type");
235
236 Value *Index = II->getOperand(i_nocapture: 1);
237 // The offset for the rawbuffer load and store ops is always in bytes.
238 uint64_t AccessSize = 1;
239 Value *Offset =
240 traverseGEPOffsets(DL, Builder, Ptr: SI->getPointerOperand(), AccessSize);
241
242 auto *VT = dyn_cast<FixedVectorType>(Val: V->getType());
243 if (VT && VT->getNumElements() > 4) {
244 // Split into stores of at most 4 elements.
245 Type *EltTy = VT->getElementType();
246 Value *Stride = ConstantInt::get(Ty: Builder.getInt32Ty(),
247 V: 4 * (DL.getTypeSizeInBits(Ty: EltTy) / 8));
248
249 SmallVector<int, 4> Indices;
250 for (unsigned int I = 0, N = VT->getNumElements(); I < N; I += 4) {
251 if (I > 0)
252 Offset = Builder.CreateAdd(LHS: Offset, RHS: Stride);
253
254 for (unsigned int J = I, E = std::min(a: N, b: J + 4); J < E; ++J)
255 Indices.push_back(Elt: J);
256 Value *Part = Builder.CreateShuffleVector(V, Mask: Indices);
257 emitRawStore(Builder, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, V: Part, RTI);
258
259 Indices.clear();
260 }
261 } else
262 emitRawStore(Builder, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, V, RTI);
263}
264
265static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI,
266 dxil::ResourceTypeInfo &RTI) {
267 switch (RTI.getResourceKind()) {
268 case dxil::ResourceKind::TypedBuffer:
269 return createTypedBufferStore(II, SI, RTI);
270 case dxil::ResourceKind::RawBuffer:
271 case dxil::ResourceKind::StructuredBuffer:
272 return createRawStores(II, SI, RTI);
273 case dxil::ResourceKind::Texture1D:
274 case dxil::ResourceKind::Texture2D:
275 case dxil::ResourceKind::Texture3D:
276 case dxil::ResourceKind::Texture1DArray:
277 case dxil::ResourceKind::Texture2DArray:
278 return createTextureStore(II, SI, RTI);
279 case dxil::ResourceKind::Texture2DMS:
280 case dxil::ResourceKind::Texture2DMSArray:
281 case dxil::ResourceKind::TextureCube:
282 case dxil::ResourceKind::TextureCubeArray:
283 case dxil::ResourceKind::FeedbackTexture2D:
284 case dxil::ResourceKind::FeedbackTexture2DArray:
285 reportFatalUsageError(
286 reason: "DXIL Store not implemented for this texture resource kind");
287 return;
288 case dxil::ResourceKind::CBuffer:
289 case dxil::ResourceKind::Sampler:
290 case dxil::ResourceKind::TBuffer:
291 case dxil::ResourceKind::RTAccelerationStructure:
292 case dxil::ResourceKind::Invalid:
293 case dxil::ResourceKind::NumEntries:
294 llvm_unreachable("Invalid resource kind for store");
295 }
296 llvm_unreachable("Unhandled case in switch");
297}
298
299static std::optional<dxil::AtomicBinOpCode>
300getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) {
301 switch (BinOp) {
302 case AtomicRMWInst::Add:
303 return dxil::AtomicBinOpCode::Add;
304 case AtomicRMWInst::And:
305 return dxil::AtomicBinOpCode::And;
306 case AtomicRMWInst::Or:
307 return dxil::AtomicBinOpCode::Or;
308 case AtomicRMWInst::Xor:
309 return dxil::AtomicBinOpCode::Xor;
310 case AtomicRMWInst::Min:
311 return dxil::AtomicBinOpCode::IMin;
312 case AtomicRMWInst::Max:
313 return dxil::AtomicBinOpCode::IMax;
314 case AtomicRMWInst::UMin:
315 return dxil::AtomicBinOpCode::UMin;
316 case AtomicRMWInst::UMax:
317 return dxil::AtomicBinOpCode::UMax;
318 case AtomicRMWInst::Xchg:
319 return dxil::AtomicBinOpCode::Exchange;
320 case AtomicRMWInst::Sub:
321 case AtomicRMWInst::Nand:
322 case AtomicRMWInst::FAdd:
323 case AtomicRMWInst::FSub:
324 case AtomicRMWInst::FMax:
325 case AtomicRMWInst::FMin:
326 case AtomicRMWInst::FMaximum:
327 case AtomicRMWInst::FMinimum:
328 case AtomicRMWInst::FMaximumNum:
329 case AtomicRMWInst::FMinimumNum:
330 case AtomicRMWInst::UIncWrap:
331 case AtomicRMWInst::UDecWrap:
332 case AtomicRMWInst::USubCond:
333 case AtomicRMWInst::USubSat:
334 case AtomicRMWInst::BAD_BINOP:
335 return std::nullopt;
336 }
337 llvm_unreachable("Unhandled atomicrmw operation");
338}
339
340// Compute the (coord0, coord1) pair for a buffer resource atomic operation.
341// Non-struct buffers (RawBuffer or TypedBuffer) fold the byte offset into the
342// index and leave coord1 poison. Only StructuredBuffer atomics use both a
343// struct index and a byte offset.
344static std::pair<Value *, Value *>
345getAtomicResourceCoords(IntrinsicInst *II, Value *PointerOperand,
346 dxil::ResourceTypeInfo &RTI, IRBuilder<> &Builder,
347 const DataLayout &DL) {
348 Value *Index = II->getOperand(i_nocapture: 1);
349
350 // The offset for the rawbuffer load/store/atomic ops is always in bytes.
351 uint64_t AccessSize = 1;
352 Value *Offset = traverseGEPOffsets(DL, Builder, Ptr: PointerOperand, AccessSize);
353
354 if (!RTI.isStruct()) {
355 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
356 if (!ConstantOffset || !ConstantOffset->isZero())
357 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
358 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
359 }
360
361 return {Index, Offset};
362}
363
364// The coordinates of a texture access are a scalar or a vector with one element
365// per texture dimension, including the array slice if there is one. These map
366// directly onto the coordinate operands of the atomic ops.
367static SmallVector<Value *, 3> getTextureAtomicCoords(IntrinsicInst *II,
368 IRBuilder<> &Builder) {
369 Value *Coords = II->getOperand(i_nocapture: 1);
370 SmallVector<Value *, 3> CoordArgs;
371 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Coords->getType())) {
372 assert(VecTy->getNumElements() <= 3 && "Too many texture coordinates");
373 for (unsigned I = 0, E = VecTy->getNumElements(); I != E; ++I)
374 CoordArgs.push_back(Elt: Builder.CreateExtractElement(Vec: Coords, Idx: I));
375 } else {
376 CoordArgs.push_back(Elt: Coords);
377 }
378 return CoordArgs;
379}
380
381static void emitAtomicBinOp(IRBuilder<> &Builder, AtomicRMWInst *AI,
382 Value *Handle, ArrayRef<Value *> Coords) {
383 assert(!Coords.empty() && Coords.size() <= 3 &&
384 "Atomic operations take between one and three coordinates");
385
386 std::optional<dxil::AtomicBinOpCode> BinOpCode =
387 getAtomicBinOpCode(BinOp: AI->getOperation());
388 if (!BinOpCode) {
389 reportFatalUsageError(reason: "DXIL resource atomicrmw operation not implemented");
390 return;
391 }
392
393 // DXIL has no floating-point atomic op. A float exchange only moves the bit
394 // pattern, so cast the value to an integer of the same width, exchange, and
395 // cast the result back. This matches what DXC emits.
396 Value *Val = AI->getValOperand();
397 Type *ValTy = Val->getType();
398 Type *OpTy = ValTy;
399 if (ValTy->isFloatingPointTy()) {
400 OpTy = Builder.getIntNTy(N: ValTy->getPrimitiveSizeInBits());
401 Val = Builder.CreateBitCast(V: Val, DestTy: OpTy);
402 }
403
404 SmallVector<Value *, 6> Args{
405 Handle, Builder.getInt32(C: static_cast<uint32_t>(*BinOpCode))};
406 append_range(C&: Args, R&: Coords);
407 Args.append(NumInputs: 3 - Coords.size(), Elt: PoisonValue::get(T: Builder.getInt32Ty()));
408 Args.push_back(Elt: Val);
409
410 // Emit the target-independent intrinsic; DXILOpLowering lowers it to the
411 // DXIL `AtomicBinOp` op and handles the target-ext-typed handle cast via
412 // its `createTmpHandleCast` bookkeeping.
413 Value *Result =
414 Builder.CreateIntrinsic(RetTy: OpTy, ID: Intrinsic::dx_resource_atomic_binop, Args);
415
416 if (OpTy != ValTy)
417 Result = Builder.CreateBitCast(V: Result, DestTy: ValTy);
418
419 AI->replaceAllUsesWith(V: Result);
420}
421
422static void createBufferAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
423 dxil::ResourceTypeInfo &RTI) {
424 const DataLayout &DL = AI->getDataLayout();
425 IRBuilder<> Builder(AI);
426 auto [Index, Offset] =
427 getAtomicResourceCoords(II, PointerOperand: AI->getPointerOperand(), RTI, Builder, DL);
428
429 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: {Index, Offset});
430}
431
432static void createTextureAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI,
433 dxil::ResourceTypeInfo &RTI) {
434 // A texture atomic operates on a whole texel, so a multi-component texel has
435 // no single addressable component. A scalar float texel is allowed, because
436 // emitAtomicBinOp exchanges its bit pattern as an integer.
437 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
438 if (!ContainedType->isIntegerTy() && !ContainedType->isFloatingPointTy()) {
439 reportFatalUsageError(reason: "DXIL atomicrmw requires a texture resource with a "
440 "scalar element type");
441 return;
442 }
443
444 IRBuilder<> Builder(AI);
445
446 emitAtomicBinOp(Builder, AI, Handle: II->getOperand(i_nocapture: 0),
447 Coords: getTextureAtomicCoords(II, Builder));
448}
449
450static void emitAtomicCompareExchange(IRBuilder<> &Builder,
451 AtomicCmpXchgInst *AI, Value *Handle,
452 ArrayRef<Value *> Coords) {
453 assert(!Coords.empty() && Coords.size() <= 3 &&
454 "Atomic operations take between one and three coordinates");
455
456 Value *Compare = AI->getCompareOperand();
457 Value *NewValue = AI->getNewValOperand();
458
459 SmallVector<Value *, 6> Args{Handle};
460 append_range(C&: Args, R&: Coords);
461 Args.append(NumInputs: 3 - Coords.size(), Elt: PoisonValue::get(T: Builder.getInt32Ty()));
462 Args.push_back(Elt: Compare);
463 Args.push_back(Elt: NewValue);
464
465 Value *Original = Builder.CreateIntrinsic(
466 RetTy: NewValue->getType(), ID: Intrinsic::dx_resource_atomic_compare_exchange,
467 Args);
468
469 // `cmpxchg` yields a { original, success } pair, but the DXIL op returns
470 // only the original value. DXIL has no way to express the success flag, and
471 // no HLSL builtin reads it, so replace the users of the pair directly
472 // instead of building it again. No pass after this one removes dead code.
473 SmallVector<ExtractValueInst *> Extracts;
474 for (User *U : AI->users()) {
475 auto *EV = dyn_cast<ExtractValueInst>(Val: U);
476 if (!EV || EV->getIndices()[0] != 0)
477 reportFatalUsageError(reason: "DXIL cmpxchg provides only the original value");
478 Extracts.push_back(Elt: EV);
479 }
480
481 for (ExtractValueInst *EV : Extracts) {
482 EV->replaceAllUsesWith(V: Original);
483 EV->eraseFromParent();
484 }
485}
486
487static void createBufferAtomicCompareExchange(IntrinsicInst *II,
488 AtomicCmpXchgInst *AI,
489 dxil::ResourceTypeInfo &RTI) {
490 const DataLayout &DL = AI->getDataLayout();
491 IRBuilder<> Builder(AI);
492 auto [Index, Offset] =
493 getAtomicResourceCoords(II, PointerOperand: AI->getPointerOperand(), RTI, Builder, DL);
494
495 emitAtomicCompareExchange(Builder, AI, Handle: II->getOperand(i_nocapture: 0), Coords: {Index, Offset});
496}
497
498// `cmpxchg` operands are always integers, so unlike atomicrmw there is no
499// float element type to convert here. The element type must be scalar.
500static void createTextureAtomicCompareExchange(IntrinsicInst *II,
501 AtomicCmpXchgInst *AI,
502 dxil::ResourceTypeInfo &RTI) {
503 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
504 if (!ContainedType->isIntegerTy() && !ContainedType->isFloatingPointTy()) {
505 reportFatalUsageError(reason: "DXIL cmpxchg requires a texture resource with a "
506 "scalar element type");
507 return;
508 }
509
510 IRBuilder<> Builder(AI);
511
512 emitAtomicCompareExchange(Builder, AI, Handle: II->getOperand(i_nocapture: 0),
513 Coords: getTextureAtomicCoords(II, Builder));
514}
515
516static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI,
517 dxil::ResourceTypeInfo &RTI) {
518 switch (RTI.getResourceKind()) {
519 case dxil::ResourceKind::TypedBuffer:
520 case dxil::ResourceKind::RawBuffer:
521 case dxil::ResourceKind::StructuredBuffer:
522 return createBufferAtomicBinOp(II, AI, RTI);
523 case dxil::ResourceKind::Texture1D:
524 case dxil::ResourceKind::Texture2D:
525 case dxil::ResourceKind::Texture3D:
526 case dxil::ResourceKind::Texture1DArray:
527 case dxil::ResourceKind::Texture2DArray:
528 return createTextureAtomicBinOp(II, AI, RTI);
529 case dxil::ResourceKind::Texture2DMS:
530 case dxil::ResourceKind::Texture2DMSArray:
531 case dxil::ResourceKind::TextureCube:
532 case dxil::ResourceKind::TextureCubeArray:
533 case dxil::ResourceKind::FeedbackTexture2D:
534 case dxil::ResourceKind::FeedbackTexture2DArray:
535 reportFatalUsageError(
536 reason: "DXIL atomicrmw not implemented for this texture resource kind");
537 return;
538 case dxil::ResourceKind::CBuffer:
539 case dxil::ResourceKind::Sampler:
540 case dxil::ResourceKind::TBuffer:
541 reportFatalUsageError(
542 reason: "DXIL atomicrmw not implemented for this resource type");
543 return;
544 case dxil::ResourceKind::RTAccelerationStructure:
545 case dxil::ResourceKind::Invalid:
546 case dxil::ResourceKind::NumEntries:
547 llvm_unreachable("Invalid resource kind for atomicrmw");
548 }
549 llvm_unreachable("Unhandled case in switch");
550}
551
552static void createAtomicCompareExchangeIntrinsic(IntrinsicInst *II,
553 AtomicCmpXchgInst *AI,
554 dxil::ResourceTypeInfo &RTI) {
555 switch (RTI.getResourceKind()) {
556 case dxil::ResourceKind::TypedBuffer:
557 case dxil::ResourceKind::RawBuffer:
558 case dxil::ResourceKind::StructuredBuffer:
559 return createBufferAtomicCompareExchange(II, AI, RTI);
560 case dxil::ResourceKind::Texture1D:
561 case dxil::ResourceKind::Texture2D:
562 case dxil::ResourceKind::Texture3D:
563 case dxil::ResourceKind::Texture1DArray:
564 case dxil::ResourceKind::Texture2DArray:
565 return createTextureAtomicCompareExchange(II, AI, RTI);
566 case dxil::ResourceKind::Texture2DMS:
567 case dxil::ResourceKind::Texture2DMSArray:
568 case dxil::ResourceKind::TextureCube:
569 case dxil::ResourceKind::TextureCubeArray:
570 case dxil::ResourceKind::FeedbackTexture2D:
571 case dxil::ResourceKind::FeedbackTexture2DArray:
572 reportFatalUsageError(
573 reason: "DXIL cmpxchg not implemented for this texture resource kind");
574 return;
575 case dxil::ResourceKind::CBuffer:
576 case dxil::ResourceKind::Sampler:
577 case dxil::ResourceKind::TBuffer:
578 reportFatalUsageError(
579 reason: "DXIL cmpxchg not implemented for this resource type");
580 return;
581 case dxil::ResourceKind::RTAccelerationStructure:
582 case dxil::ResourceKind::Invalid:
583 case dxil::ResourceKind::NumEntries:
584 llvm_unreachable("Invalid resource kind for cmpxchg");
585 }
586 llvm_unreachable("Unhandled case in switch");
587}
588
589static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI,
590 dxil::ResourceTypeInfo &RTI) {
591 const DataLayout &DL = LI->getDataLayout();
592 IRBuilder<> Builder(LI);
593 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
594 Type *LoadType = StructType::get(elt1: ContainedType, elts: Builder.getInt1Ty());
595
596 Value *V =
597 Builder.CreateIntrinsic(RetTy: LoadType, ID: Intrinsic::dx_resource_load_typedbuffer,
598 Args: {II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1)});
599 V = Builder.CreateExtractValue(Agg: V, Idxs: {0});
600
601 Type *ScalarType = ContainedType->getScalarType();
602 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
603 Value *Offset =
604 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
605 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
606 if (!ConstantOffset || !ConstantOffset->isZero())
607 V = Builder.CreateExtractElement(Vec: V, Idx: Offset);
608
609 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a
610 // shufflevector), then make sure we're maintaining the resulting type.
611 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType()))
612 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(Val: V->getType()))
613 V = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: V,
614 Idx: Builder.getInt32(C: 0));
615
616 LI->replaceAllUsesWith(V);
617}
618
619static void createTextureLoad(IntrinsicInst *II, LoadInst *LI,
620 dxil::ResourceTypeInfo &RTI) {
621 const DataLayout &DL = LI->getDataLayout();
622 IRBuilder<> Builder(LI);
623 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(i: 0);
624
625 Value *Handle = II->getOperand(i_nocapture: 0);
626 Value *Coords = II->getOperand(i_nocapture: 1);
627
628 // For operator[], mip level is 0.
629 Value *MipLevel = Builder.getInt32(C: 0);
630
631 // For operator[], offsets are zero.
632 Value *Offsets = getNullOffsetsFor(Builder, Coords);
633
634 Value *V =
635 Builder.CreateIntrinsic(RetTy: ContainedType, ID: Intrinsic::dx_resource_load_level,
636 Args: {Handle, Coords, MipLevel, Offsets});
637
638 Type *ScalarType = ContainedType->getScalarType();
639 uint64_t AccessSize = DL.getTypeSizeInBits(Ty: ScalarType) / 8;
640 Value *Offset =
641 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
642 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
643 if (!ConstantOffset || !ConstantOffset->isZero())
644 V = Builder.CreateExtractElement(Vec: V, Idx: Offset);
645
646 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a
647 // shufflevector), then make sure we're maintaining the resulting type.
648 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType()))
649 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(Val: V->getType()))
650 V = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: V,
651 Idx: Builder.getInt32(C: 0));
652
653 LI->replaceAllUsesWith(V);
654}
655
656static Value *emitRawLoad(IRBuilder<> &Builder, Type *Ty, Value *Buffer,
657 Value *Index, Value *Offset,
658 dxil::ResourceTypeInfo &RTI) {
659 // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access
660 // entirely into the index.
661 if (!RTI.isStruct()) {
662 auto *ConstantOffset = dyn_cast<ConstantInt>(Val: Offset);
663 if (!ConstantOffset || !ConstantOffset->isZero())
664 Index = Builder.CreateAdd(LHS: Index, RHS: Offset);
665 Offset = llvm::PoisonValue::get(T: Builder.getInt32Ty());
666 }
667
668 // The load intrinsic includes the bit for CheckAccessFullyMapped, so we need
669 // to add that to the return type.
670 Type *TypeWithCheck = StructType::get(elt1: Ty, elts: Builder.getInt1Ty());
671 Value *V = Builder.CreateIntrinsic(RetTy: TypeWithCheck,
672 ID: Intrinsic::dx_resource_load_rawbuffer,
673 Args: {Buffer, Index, Offset});
674 return Builder.CreateExtractValue(Agg: V, Idxs: {0});
675}
676
677static void createRawLoads(IntrinsicInst *II, LoadInst *LI,
678 dxil::ResourceTypeInfo &RTI) {
679 const DataLayout &DL = LI->getDataLayout();
680 IRBuilder<> Builder(LI);
681
682 Value *Index = II->getOperand(i_nocapture: 1);
683 // The offset for the rawbuffer load and store ops is always in bytes.
684 uint64_t AccessSize = 1;
685 Value *Offset =
686 traverseGEPOffsets(DL, Builder, Ptr: LI->getPointerOperand(), AccessSize);
687
688 // TODO: We could make this handle aggregates by walking the structure and
689 // handling each field individually, but we don't ever generate code that
690 // would hit that so it seems superfluous.
691 assert(!LI->getType()->isAggregateType() &&
692 "Resource load should be scalar or vector type");
693
694 Value *V;
695 if (auto *VT = dyn_cast<FixedVectorType>(Val: LI->getType())) {
696 // Split into loads of at most 4 elements.
697 Type *EltTy = VT->getElementType();
698 Value *Stride = ConstantInt::get(Ty: Builder.getInt32Ty(),
699 V: 4 * (DL.getTypeSizeInBits(Ty: EltTy) / 8));
700
701 SmallVector<Value *> Parts;
702 for (unsigned int I = 0, N = VT->getNumElements(); I < N; I += 4) {
703 Type *Ty = FixedVectorType::get(ElementType: EltTy, NumElts: N - I < 4 ? N - I : 4);
704 if (I > 0)
705 Offset = Builder.CreateAdd(LHS: Offset, RHS: Stride);
706 Parts.push_back(
707 Elt: emitRawLoad(Builder, Ty, Buffer: II->getOperand(i_nocapture: 0), Index, Offset, RTI));
708 }
709
710 V = Parts.size() > 1 ? concatenateVectors(Builder, Vecs: Parts) : Parts[0];
711 } else
712 V = emitRawLoad(Builder, Ty: LI->getType(), Buffer: II->getOperand(i_nocapture: 0), Index, Offset,
713 RTI);
714
715 LI->replaceAllUsesWith(V);
716}
717
718namespace {
719/// Helper for building a `load.cbufferrow` intrinsic given a simple type.
720struct CBufferRowIntrin {
721 Intrinsic::ID IID;
722 Type *RetTy;
723 unsigned int EltSize;
724 unsigned int NumElts;
725
726 CBufferRowIntrin(const DataLayout &DL, Type *Ty) {
727 assert(Ty == Ty->getScalarType() && "Expected scalar type");
728
729 switch (DL.getTypeSizeInBits(Ty)) {
730 case 16:
731 IID = Intrinsic::dx_resource_load_cbufferrow_8;
732 RetTy = StructType::get(elt1: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty, elts: Ty);
733 EltSize = 2;
734 NumElts = 8;
735 break;
736 case 32:
737 IID = Intrinsic::dx_resource_load_cbufferrow_4;
738 RetTy = StructType::get(elt1: Ty, elts: Ty, elts: Ty, elts: Ty);
739 EltSize = 4;
740 NumElts = 4;
741 break;
742 case 64:
743 IID = Intrinsic::dx_resource_load_cbufferrow_2;
744 RetTy = StructType::get(elt1: Ty, elts: Ty);
745 EltSize = 8;
746 NumElts = 2;
747 break;
748 default:
749 llvm_unreachable("Only 16, 32, and 64 bit types supported");
750 }
751 }
752};
753} // namespace
754
755static Value *loadCBufferRowIntoWindow(IRBuilder<> &Builder,
756 const CBufferRowIntrin &Intrin,
757 Value *Handle, Value *FirstRow,
758 Value *Window, unsigned RowNumber,
759 StringRef Name) {
760 Value *RowIndex =
761 RowNumber == 0 ? FirstRow
762 : Builder.CreateAdd(LHS: FirstRow, RHS: Builder.getInt32(C: RowNumber));
763 Value *RowLoad = Builder.CreateIntrinsic(
764 RetTy: Intrin.RetTy, ID: Intrin.IID, Args: {Handle, RowIndex}, FMFSource: nullptr, Name: Name + ".load");
765 for (unsigned J = 0; J != Intrin.NumElts; ++J) {
766 Value *Element = Builder.CreateExtractValue(Agg: RowLoad, Idxs: {J});
767 Window = Builder.CreateInsertElement(
768 Vec: Window, NewElt: Element, Idx: Builder.getInt32(C: RowNumber * Intrin.NumElts + J));
769 }
770 return Window;
771}
772
773static Value *createDynamicCBufferLoad(IntrinsicInst *II, LoadInst *LI,
774 const CBufferRowIntrin &Intrin,
775 Value *BaseRow, Value *ByteOffset,
776 const KnownBits &WithinRow) {
777 IRBuilder<> Builder(LI);
778 Type *Ty = LI->getType();
779 auto *VT = dyn_cast<FixedVectorType>(Val: Ty);
780 unsigned NumElements = VT ? VT->getNumElements() : 1;
781 StringRef Name = LI->getName();
782 Value *Handle = II->getOperand(i_nocapture: 0);
783 Value *RowSize = Builder.getInt32(C: hlsl::CBufferRowSizeInBytes);
784 Value *FirstRow =
785 Builder.CreateAdd(LHS: BaseRow, RHS: Builder.CreateUDiv(LHS: ByteOffset, RHS: RowSize));
786 Value *Component = Builder.CreateUDiv(LHS: Builder.CreateURem(LHS: ByteOffset, RHS: RowSize),
787 RHS: Builder.getInt32(C: Intrin.EltSize));
788 unsigned MinComponent =
789 WithinRow.getMinValue().getZExtValue() / Intrin.EltSize;
790 unsigned MaxComponent =
791 WithinRow.getMaxValue().getZExtValue() / Intrin.EltSize;
792 unsigned MinRows = 1 + (MinComponent + NumElements - 1) / Intrin.NumElts;
793 unsigned MaxRows = 1 + (MaxComponent + NumElements - 1) / Intrin.NumElts;
794 auto *WindowTy =
795 FixedVectorType::get(ElementType: Ty->getScalarType(), NumElts: MaxRows * Intrin.NumElts);
796 Value *Window = PoisonValue::get(T: WindowTy);
797 for (unsigned I = 0; I != MinRows; ++I)
798 Window = loadCBufferRowIntoWindow(Builder, Intrin, Handle, FirstRow, Window,
799 RowNumber: I, Name);
800
801 // The unknown starting component can require at most one extra row.
802 // Load it only when needed, rather than accessing beyond the load.
803 if (MaxRows != MinRows) {
804 BasicBlock *Head = LI->getParent();
805 Value *LastComponent =
806 Builder.CreateAdd(LHS: Component, RHS: Builder.getInt32(C: NumElements - 1));
807 Value *CrossesRow = Builder.CreateICmpUGE(
808 LHS: LastComponent, RHS: Builder.getInt32(C: MinRows * Intrin.NumElts));
809 Instruction *ThenTerm = SplitBlockAndInsertIfThen(Cond: CrossesRow, SplitBefore: LI, Unreachable: false);
810 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *Head->getTerminator(),
811 DEBUG_TYPE);
812 Builder.SetInsertPoint(ThenTerm);
813 Value *NextWindow = loadCBufferRowIntoWindow(
814 Builder, Intrin, Handle, FirstRow, Window, RowNumber: MinRows, Name);
815 Builder.SetInsertPoint(LI);
816 auto *Rows = Builder.CreatePHI(Ty: WindowTy, NumReservedValues: 2, Name: Name + ".rows");
817 Rows->addIncoming(V: Window, BB: Head);
818 Rows->addIncoming(V: NextWindow, BB: ThenTerm->getParent());
819 Window = Rows;
820 }
821
822 Value *Result = PoisonValue::get(T: Ty);
823 for (unsigned I = 0; I != NumElements; ++I) {
824 Value *Index =
825 I == 0 ? Component : Builder.CreateAdd(LHS: Component, RHS: Builder.getInt32(C: I));
826 Value *Element =
827 Builder.CreateExtractElement(Vec: Window, Idx: Index, Name: Name + ".extract");
828 Result =
829 VT ? Builder.CreateInsertElement(Vec: Result, NewElt: Element, Idx: Builder.getInt32(C: I))
830 : Element;
831 }
832 return Result;
833}
834
835static KnownBits computeCBufferOffsetKnownBits(Value *Offset,
836 const DataLayout &DL,
837 const Instruction *CxtI) {
838 KnownBits Known = computeKnownBits(V: Offset, DL, AC: nullptr, CtxI: CxtI);
839 auto *Phi = dyn_cast<PHINode>(Val: Offset);
840 if (!Phi || Phi->getNumIncomingValues() == 0)
841 return Known;
842
843 // PHI queries limit recursion into each input, which can hide row alignment.
844 KnownBits IncomingKnown(Known.getBitWidth());
845 IncomingKnown.setAllConflict();
846 for (unsigned I = 0; I != Phi->getNumIncomingValues(); ++I)
847 IncomingKnown = IncomingKnown.intersectWith(
848 RHS: computeKnownBits(V: Phi->getIncomingValue(i: I), DL, AC: nullptr,
849 CtxI: Phi->getIncomingBlock(i: I)->getTerminator()));
850 return Known.unionWith(RHS: IncomingKnown);
851}
852
853static void createCBufferLoad(IntrinsicInst *II, LoadInst *LI,
854 dxil::ResourceTypeInfo &RTI) {
855 const DataLayout &DL = LI->getDataLayout();
856
857 Type *Ty = LI->getType();
858 assert(!isa<StructType>(Ty) && "Structs not handled yet");
859 CBufferRowIntrin Intrin(DL, Ty->getScalarType());
860
861 StringRef Name = LI->getName();
862 Value *Handle = II->getOperand(i_nocapture: 0);
863
864 IRBuilder<> Builder(LI);
865
866 ConstantInt *GlobalOffset =
867 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer
868 ? ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)
869 : dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1));
870 assert(GlobalOffset && "CBuffer getpointer index must be constant");
871
872 uint64_t GlobalOffsetVal = GlobalOffset->getZExtValue();
873 Value *CurrentRow = ConstantInt::get(
874 Ty: Builder.getInt32Ty(), V: GlobalOffsetVal / hlsl::CBufferRowSizeInBytes);
875 unsigned int CurrentIndex =
876 (GlobalOffsetVal % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
877
878 // Every object in a cbuffer either fits in a row or is aligned to a row. This
879 // means that only the very last pointer access can point into a row.
880 auto *LastGEP = dyn_cast<GEPOperator>(Val: LI->getPointerOperand());
881 if (!LastGEP) {
882 // If we don't have a GEP at all we're just accessing the resource through
883 // the result of getpointer directly.
884 assert(LI->getPointerOperand() == II &&
885 "Unexpected indirect access to resource without GEP");
886 } else {
887 Value *GEPOffset = traverseGEPOffsets(
888 DL, Builder, Ptr: LastGEP->getPointerOperand(), AccessSize: hlsl::CBufferRowSizeInBytes);
889 CurrentRow = Builder.CreateAdd(LHS: GEPOffset, RHS: CurrentRow);
890
891 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: LastGEP->getType()), 0);
892 if (LastGEP->accumulateConstantOffset(DL, Offset&: ConstantOffset)) {
893 APInt Remainder(DL.getIndexTypeSizeInBits(Ty: LastGEP->getType()),
894 hlsl::CBufferRowSizeInBytes);
895 APInt::udivrem(LHS: ConstantOffset, RHS: Remainder, Quotient&: ConstantOffset, Remainder);
896 CurrentRow = Builder.CreateAdd(
897 LHS: CurrentRow, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: ConstantOffset));
898 CurrentIndex += Remainder.udiv(RHS: Intrin.EltSize).getZExtValue();
899 } else {
900 assert(LastGEP->getNumIndices() == 1 &&
901 "Last GEP of cbuffer access is not array or struct access");
902 // Aggregate strides are row-aligned, but merged byte offsets can select
903 // different components within a row.
904 Value *RowIndex = Builder.CreateSExtOrTrunc(V: *LastGEP->idx_begin(),
905 DestTy: Builder.getInt32Ty());
906 uint64_t IndexScale =
907 DL.getTypeAllocSize(Ty: LastGEP->getSourceElementType());
908 if (IndexScale < hlsl::CBufferRowSizeInBytes) {
909 assert(hlsl::CBufferRowSizeInBytes % IndexScale == 0 &&
910 "CBuffer GEP index is not row aligned");
911 Value *ByteOffset =
912 IndexScale == 1
913 ? RowIndex
914 : Builder.CreateMul(LHS: RowIndex, RHS: Builder.getInt32(C: IndexScale));
915 uint64_t BaseWithinRow = GlobalOffsetVal % hlsl::CBufferRowSizeInBytes;
916 if (BaseWithinRow != 0)
917 ByteOffset =
918 Builder.CreateAdd(LHS: ByteOffset, RHS: Builder.getInt32(C: BaseWithinRow));
919 Value *RowSize = Builder.getInt32(C: hlsl::CBufferRowSizeInBytes);
920 KnownBits WithinRow =
921 computeCBufferOffsetKnownBits(Offset: ByteOffset, DL, CxtI: LI) &
922 KnownBits::makeConstant(C: APInt(32, hlsl::CBufferRowSizeInBytes - 1));
923 if (!WithinRow.isConstant()) {
924 LI->replaceAllUsesWith(V: createDynamicCBufferLoad(
925 II, LI, Intrin, BaseRow: CurrentRow, ByteOffset, WithinRow));
926 return;
927 }
928 RowIndex = Builder.CreateUDiv(LHS: ByteOffset, RHS: RowSize, Name: "",
929 isExact: WithinRow.getConstant().isZero());
930 CurrentIndex = WithinRow.getConstant().getZExtValue() / Intrin.EltSize;
931 } else if (IndexScale > hlsl::CBufferRowSizeInBytes) {
932 assert(IndexScale % hlsl::CBufferRowSizeInBytes == 0 &&
933 "CBuffer GEP index is not row aligned");
934 RowIndex = Builder.CreateMul(
935 LHS: RowIndex,
936 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(),
937 V: IndexScale / hlsl::CBufferRowSizeInBytes));
938 }
939 if (auto *BaseRow = dyn_cast<ConstantInt>(Val: CurrentRow);
940 BaseRow && BaseRow->isZero())
941 CurrentRow = RowIndex;
942 else
943 CurrentRow = Builder.CreateAdd(LHS: CurrentRow, RHS: RowIndex);
944 }
945 }
946
947 auto *CBufLoad = Builder.CreateIntrinsic(
948 RetTy: Intrin.RetTy, ID: Intrin.IID, Args: {Handle, CurrentRow}, FMFSource: nullptr, Name: Name + ".load");
949 auto *Elt =
950 Builder.CreateExtractValue(Agg: CBufLoad, Idxs: {CurrentIndex++}, Name: Name + ".extract");
951
952 // At this point we've loaded the first scalar of our result, but our original
953 // type may have been a vector.
954 unsigned int Remaining =
955 ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
956 if (Remaining == 0) {
957 // We only have a single element, so we're done.
958 Value *Result = Elt;
959
960 // However, if we loaded a <1 x T>, then we need to adjust the type.
961 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
962 assert(VT->getNumElements() == 1 && "Can't have multiple elements here");
963 Result = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VT), NewElt: Result,
964 Idx: Builder.getInt32(C: 0), Name);
965 }
966 LI->replaceAllUsesWith(V: Result);
967 return;
968 }
969
970 // Walk each element and extract it, wrapping to new rows as needed.
971 SmallVector<Value *> Extracts{Elt};
972 while (Remaining--) {
973 CurrentIndex %= Intrin.NumElts;
974
975 if (CurrentIndex == 0) {
976 CurrentRow = Builder.CreateAdd(LHS: CurrentRow,
977 RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
978 CBufLoad = Builder.CreateIntrinsic(RetTy: Intrin.RetTy, ID: Intrin.IID,
979 Args: {Handle, CurrentRow}, FMFSource: nullptr,
980 Name: Name + ".load");
981 }
982
983 Extracts.push_back(Elt: Builder.CreateExtractValue(Agg: CBufLoad, Idxs: {CurrentIndex++},
984 Name: Name + ".extract"));
985 }
986
987 // Finally, we build up the original loaded value.
988 Value *Result = PoisonValue::get(T: Ty);
989 for (int I = 0, E = Extracts.size(); I < E; ++I)
990 Result = Builder.CreateInsertElement(
991 Vec: Result, NewElt: Extracts[I], Idx: Builder.getInt32(C: I), Name: Name + formatv(Fmt: ".upto{}", Vals&: I));
992 LI->replaceAllUsesWith(V: Result);
993}
994
995static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI,
996 dxil::ResourceTypeInfo &RTI) {
997 switch (RTI.getResourceKind()) {
998 case dxil::ResourceKind::TypedBuffer:
999 return createTypedBufferLoad(II, LI, RTI);
1000 case dxil::ResourceKind::RawBuffer:
1001 case dxil::ResourceKind::StructuredBuffer:
1002 return createRawLoads(II, LI, RTI);
1003 case dxil::ResourceKind::CBuffer:
1004 return createCBufferLoad(II, LI, RTI);
1005 case dxil::ResourceKind::Texture1D:
1006 case dxil::ResourceKind::Texture2D:
1007 case dxil::ResourceKind::Texture2DMS:
1008 case dxil::ResourceKind::Texture3D:
1009 case dxil::ResourceKind::TextureCube:
1010 case dxil::ResourceKind::Texture1DArray:
1011 case dxil::ResourceKind::Texture2DArray:
1012 case dxil::ResourceKind::Texture2DMSArray:
1013 case dxil::ResourceKind::TextureCubeArray:
1014 return createTextureLoad(II, LI, RTI);
1015 case dxil::ResourceKind::FeedbackTexture2D:
1016 case dxil::ResourceKind::FeedbackTexture2DArray:
1017 case dxil::ResourceKind::TBuffer:
1018 reportFatalUsageError(reason: "Load not yet implemented for resource type");
1019 return;
1020 case dxil::ResourceKind::Sampler:
1021 case dxil::ResourceKind::RTAccelerationStructure:
1022 case dxil::ResourceKind::Invalid:
1023 case dxil::ResourceKind::NumEntries:
1024 llvm_unreachable("Invalid resource kind for load");
1025 }
1026 llvm_unreachable("Unhandled case in switch");
1027}
1028
1029static Instruction *getHandleOperand(Instruction *AI) {
1030 if (auto *LI = dyn_cast<LoadInst>(Val: AI))
1031 return dyn_cast<Instruction>(Val: LI->getPointerOperand());
1032 if (auto *SI = dyn_cast<StoreInst>(Val: AI))
1033 return dyn_cast<Instruction>(Val: SI->getPointerOperand());
1034 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: AI))
1035 return dyn_cast<Instruction>(Val: RMWI->getPointerOperand());
1036 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: AI))
1037 return dyn_cast<Instruction>(Val: CXI->getPointerOperand());
1038 if (auto *II = dyn_cast<IntrinsicInst>(Val: AI))
1039 if (II->getIntrinsicID() == Intrinsic::dx_resource_updatecounter)
1040 return dyn_cast<Instruction>(Val: II->getArgOperand(i: 0));
1041
1042 return nullptr;
1043}
1044
1045static Instruction *getHandleRoot(Instruction *I) {
1046 while (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1047 auto *PointerI = dyn_cast<Instruction>(Val: GEP->getPointerOperand());
1048 if (!PointerI)
1049 break;
1050 I = PointerI;
1051 }
1052 return I;
1053}
1054
1055static const std::array<Intrinsic::ID, 2> HandleIntrins = {
1056 Intrinsic::dx_resource_handlefrombinding,
1057 Intrinsic::dx_resource_handlefromimplicitbinding,
1058};
1059
1060static SmallVector<IntrinsicInst *> collectUsedHandles(Value *Ptr) {
1061 SmallVector<Value *> Worklist = {Ptr};
1062 SmallVector<IntrinsicInst *> Handles;
1063 SmallPtrSet<Value *, 4> VisitedPhis;
1064
1065 while (!Worklist.empty()) {
1066 Value *X = Worklist.pop_back_val();
1067
1068 if (!X->getType()->isPointerTy() && !X->getType()->isTargetExtTy())
1069 return {}; // Early exit on store/load into non-resource
1070
1071 if (auto *Phi = dyn_cast<PHINode>(Val: X)) {
1072 if (VisitedPhis.contains(Ptr: X))
1073 continue;
1074 for (Use &V : Phi->incoming_values())
1075 Worklist.push_back(Elt: V.get());
1076 VisitedPhis.insert(Ptr: Phi);
1077 } else if (auto *Select = dyn_cast<SelectInst>(Val: X)) {
1078 Worklist.push_back(Elt: Select->getTrueValue());
1079 Worklist.push_back(Elt: Select->getFalseValue());
1080 } else if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: X)) {
1081 Worklist.push_back(Elt: GEP->getPointerOperand());
1082 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: X)) {
1083 Intrinsic::ID IID = II->getIntrinsicID();
1084
1085 if (IID == Intrinsic::dx_resource_getpointer ||
1086 IID == Intrinsic::dx_resource_getbasepointer)
1087 Worklist.push_back(Elt: II->getArgOperand(/*Handle=*/i: 0));
1088
1089 if (llvm::is_contained(Range: HandleIntrins, Element: IID))
1090 Handles.push_back(Elt: II);
1091 }
1092 }
1093
1094 return Handles;
1095}
1096
1097static hlsl::Binding getHandleIntrinsicBinding(IntrinsicInst *Handle,
1098 DXILResourceTypeMap &DRTM) {
1099 assert(llvm::is_contained(HandleIntrins, Handle->getIntrinsicID()) &&
1100 "Only expects a Handle as determined from collectUsedHandles.");
1101
1102 auto *HandleTy = cast<TargetExtType>(Val: Handle->getType());
1103 dxil::ResourceClass Class = DRTM[HandleTy].getResourceClass();
1104 uint32_t Space = cast<ConstantInt>(Val: Handle->getArgOperand(i: 0))->getZExtValue();
1105 uint32_t LowerBound =
1106 cast<ConstantInt>(Val: Handle->getArgOperand(i: 1))->getZExtValue();
1107 uint32_t Size = cast<ConstantInt>(Val: Handle->getArgOperand(i: 2))->getZExtValue();
1108 uint32_t UpperBound = Size == UINT32_MAX ? UINT32_MAX : LowerBound + Size - 1;
1109
1110 return hlsl::Binding(Class, Space, LowerBound, UpperBound, nullptr);
1111}
1112
1113namespace {
1114/// Helper for propagating the current handle and ptr indices.
1115struct AccessIndices {
1116 Value *GetPtrIdx;
1117 Value *HandleIdx;
1118 Value *OffsetIdx;
1119
1120 bool hasGetPtrIdx() { return GetPtrIdx != nullptr; }
1121 bool hasHandleIdx() { return HandleIdx != nullptr; }
1122 bool hasOffsetIdx() { return OffsetIdx != nullptr; }
1123};
1124
1125/// AccessIndices cached per phi. The values may be placeholder phis that are
1126/// later folded away (via replaceAllUsesWith) and freed, so track them with
1127/// value handles rather than raw pointers.
1128struct TrackedAccessIndices {
1129 WeakTrackingVH GetPtrIdx;
1130 WeakTrackingVH HandleIdx;
1131 WeakTrackingVH OffsetIdx;
1132
1133 TrackedAccessIndices() = default;
1134 TrackedAccessIndices(const AccessIndices &AI)
1135 : GetPtrIdx(AI.GetPtrIdx), HandleIdx(AI.HandleIdx),
1136 OffsetIdx(AI.OffsetIdx) {}
1137 operator AccessIndices() const { return {.GetPtrIdx: GetPtrIdx, .HandleIdx: HandleIdx, .OffsetIdx: OffsetIdx}; }
1138};
1139
1140using PhiAccessIndicesMap = SmallDenseMap<PHINode *, TrackedAccessIndices>;
1141} // namespace
1142
1143// Compute the total byte offset described by a GEP as a single i32 value,
1144// handling both constant and variable indices.
1145static Value *accumulateGEPOffset(GetElementPtrInst *GEP,
1146 IRBuilder<> &Builder) {
1147 const DataLayout &DL = GEP->getDataLayout();
1148 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1149 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1150 APInt ConstantOffset(BitWidth, 0);
1151 bool Success =
1152 GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset);
1153 assert(Success && "Unhandled GEP structure for resource access");
1154 (void)Success;
1155
1156 Type *I32 = Builder.getInt32Ty();
1157 Value *Offset = ConstantInt::get(Ty: I32, V: ConstantOffset.getSExtValue());
1158 for (auto &[V, Scale] : VariableOffsets) {
1159 Value *Index = Builder.CreateSExtOrTrunc(V, DestTy: I32);
1160 if (!Scale.isOne())
1161 Index =
1162 Builder.CreateMul(LHS: Index, RHS: ConstantInt::get(Ty: I32, V: Scale.getSExtValue()));
1163 Offset = Builder.CreateAdd(LHS: Offset, RHS: Index);
1164 }
1165 return Offset;
1166}
1167
1168// getAccessIndices traverses up the control flow that a ptr came from and
1169// propagates back the indicies used to access the resource (AccessIndices):
1170//
1171// - GetPtrIdx is the index of dx.resource.getpointer
1172// - HandleIdx is the index of dx.resource.handlefrom.*
1173// - OffsetIdx is the accumulated byte offset of any GEPs in the ptr chain
1174static AccessIndices
1175getAccessIndices(Instruction *I, SmallSetVector<Instruction *, 16> &DeadInsts,
1176 PhiAccessIndicesMap &VisitedPhis) {
1177 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
1178 if (llvm::is_contained(Range: HandleIntrins, Element: II->getIntrinsicID())) {
1179 DeadInsts.insert(X: II);
1180 return {.GetPtrIdx: nullptr, .HandleIdx: II->getArgOperand(/*Index=*/i: 3), .OffsetIdx: nullptr};
1181 }
1182
1183 Intrinsic::ID IID = II->getIntrinsicID();
1184 if (IID == Intrinsic::dx_resource_getpointer ||
1185 IID == Intrinsic::dx_resource_getbasepointer) {
1186 auto *V = dyn_cast<Instruction>(Val: II->getArgOperand(/*Handle=*/i: 0));
1187 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
1188 assert(!AccessIdx.hasGetPtrIdx() &&
1189 "Encountered multiple dx.resource.getpointers in ptr chain?");
1190 IRBuilder<> Builder(II);
1191 AccessIdx.GetPtrIdx = ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
1192 if (IID == Intrinsic::dx_resource_getpointer)
1193 AccessIdx.GetPtrIdx = II->getArgOperand(i: 1);
1194
1195 DeadInsts.insert(X: II);
1196 return AccessIdx;
1197 }
1198 }
1199
1200 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1201 auto *V = dyn_cast<Instruction>(Val: GEP->getPointerOperand());
1202 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
1203
1204 IRBuilder<> Builder(GEP);
1205 Value *GEPOffset = accumulateGEPOffset(GEP, Builder);
1206 AccessIdx.OffsetIdx =
1207 AccessIdx.hasOffsetIdx()
1208 ? Builder.CreateAdd(LHS: AccessIdx.OffsetIdx, RHS: GEPOffset)
1209 : GEPOffset;
1210
1211 DeadInsts.insert(X: GEP);
1212 return AccessIdx;
1213 }
1214
1215 if (auto *Phi = dyn_cast<PHINode>(Val: I)) {
1216 // If we're already building indices for this phi, return a ref to the phi
1217 if (auto It = VisitedPhis.find(Val: Phi); It != VisitedPhis.end())
1218 return It->second;
1219
1220 unsigned NumEdges = Phi->getNumIncomingValues();
1221 assert(NumEdges != 0 && "Malformed Phi Node");
1222
1223 IRBuilder<> Builder(Phi);
1224 std::unique_ptr<PHINode> GetPtrPhi(
1225 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
1226 std::unique_ptr<PHINode> HandlePhi(
1227 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
1228 std::unique_ptr<PHINode> OffsetPhi(
1229 PHINode::Create(Ty: Builder.getInt32Ty(), NumReservedValues: NumEdges));
1230 bool HasOffset = false;
1231
1232 // Register a ref to this phi for a recursive phi. If a placeholder phi
1233 // below folds to a single value, replaceAllUsesWith updates any cached
1234 // entries that captured it.
1235 if (Phi->getType()->isTargetExtTy())
1236 VisitedPhis[Phi] = AccessIndices{/*GetPtrIdx=*/nullptr, .HandleIdx: HandlePhi.get(),
1237 /*OffsetIdx=*/nullptr};
1238 else if (Phi->getType()->isPointerTy())
1239 VisitedPhis[Phi] =
1240 AccessIndices{.GetPtrIdx: GetPtrPhi.get(), .HandleIdx: HandlePhi.get(), .OffsetIdx: OffsetPhi.get()};
1241
1242 for (unsigned Idx = 0; Idx < NumEdges; Idx++) {
1243 auto *BB = Phi->getIncomingBlock(i: Idx);
1244 auto *V = dyn_cast<Instruction>(Val: Phi->getIncomingValue(i: Idx));
1245 auto AccessIdx = getAccessIndices(I: V, DeadInsts, VisitedPhis);
1246 if (AccessIdx.hasGetPtrIdx())
1247 GetPtrPhi->addIncoming(V: AccessIdx.GetPtrIdx, BB);
1248 HandlePhi->addIncoming(V: AccessIdx.HandleIdx, BB);
1249 if (AccessIdx.hasOffsetIdx()) {
1250 OffsetPhi->addIncoming(V: AccessIdx.OffsetIdx, BB);
1251 HasOffset = true;
1252 } else {
1253 OffsetPhi->addIncoming(V: ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0), BB);
1254 }
1255 }
1256
1257 Value *GetPtrIdx;
1258 if (GetPtrPhi->getNumIncomingValues() == 0) {
1259 GetPtrIdx = nullptr;
1260 } else if (Value *ConstantGetPtr = GetPtrPhi->hasConstantValue()) {
1261 GetPtrIdx = ConstantGetPtr;
1262 GetPtrPhi->replaceAllUsesWith(V: GetPtrIdx);
1263 } else {
1264 GetPtrIdx = GetPtrPhi.release();
1265 Builder.Insert(V: GetPtrIdx);
1266 }
1267
1268 Value *HandleIdx;
1269 if (Value *ConstantHandle = HandlePhi->hasConstantValue()) {
1270 HandleIdx = ConstantHandle;
1271 HandlePhi->replaceAllUsesWith(V: HandleIdx);
1272 } else {
1273 HandleIdx = HandlePhi.release();
1274 Builder.Insert(V: HandleIdx);
1275 }
1276
1277 Value *OffsetIdx = nullptr;
1278 if (HasOffset) {
1279 if (Value *ConstantIdx = OffsetPhi->hasConstantValue()) {
1280 OffsetIdx = ConstantIdx;
1281 OffsetPhi->replaceAllUsesWith(V: OffsetIdx);
1282 } else {
1283 OffsetIdx = OffsetPhi.release();
1284 Builder.Insert(V: OffsetIdx);
1285 }
1286 }
1287
1288 // Replace the placeholder entry with the final indices.
1289 AccessIndices Result = {.GetPtrIdx: GetPtrIdx, .HandleIdx: HandleIdx, .OffsetIdx: OffsetIdx};
1290 VisitedPhis[Phi] = Result;
1291 DeadInsts.insert(X: Phi);
1292 return Result;
1293 }
1294
1295 if (auto *Select = dyn_cast<SelectInst>(Val: I)) {
1296 auto *TrueV = dyn_cast<Instruction>(Val: Select->getTrueValue());
1297 auto TrueAccessIdx = getAccessIndices(I: TrueV, DeadInsts, VisitedPhis);
1298
1299 auto *FalseV = dyn_cast<Instruction>(Val: Select->getFalseValue());
1300 auto FalseAccessIdx = getAccessIndices(I: FalseV, DeadInsts, VisitedPhis);
1301
1302 IRBuilder<> Builder(Select);
1303 Value *GetPtrSelect = nullptr;
1304 Value *OffsetSelect = nullptr;
1305
1306 if (TrueAccessIdx.hasGetPtrIdx() && FalseAccessIdx.hasGetPtrIdx())
1307 GetPtrSelect =
1308 Builder.CreateSelect(C: Select->getCondition(), True: TrueAccessIdx.GetPtrIdx,
1309 False: FalseAccessIdx.GetPtrIdx, Name: "", MDFrom: Select);
1310
1311 auto *HandleSelect =
1312 Builder.CreateSelect(C: Select->getCondition(), True: TrueAccessIdx.HandleIdx,
1313 False: FalseAccessIdx.HandleIdx, Name: "", MDFrom: Select);
1314
1315 if (TrueAccessIdx.hasOffsetIdx() || FalseAccessIdx.hasOffsetIdx()) {
1316 Value *Zero = ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0);
1317 Value *TrueOffset =
1318 TrueAccessIdx.hasOffsetIdx() ? TrueAccessIdx.OffsetIdx : Zero;
1319 Value *FalseOffset =
1320 FalseAccessIdx.hasOffsetIdx() ? FalseAccessIdx.OffsetIdx : Zero;
1321 OffsetSelect = Builder.CreateSelect(C: Select->getCondition(), True: TrueOffset,
1322 False: FalseOffset, Name: "", MDFrom: Select);
1323 }
1324
1325 DeadInsts.insert(X: Select);
1326 return {.GetPtrIdx: GetPtrSelect, .HandleIdx: HandleSelect, .OffsetIdx: OffsetSelect};
1327 }
1328
1329 llvm_unreachable("collectUsedHandles should assure this does not occur");
1330}
1331
1332static void
1333replaceHandleWithIndices(Instruction *Ptr, IntrinsicInst *OldHandle,
1334 SmallSetVector<Instruction *, 16> &DeadInsts,
1335 PhiAccessIndicesMap &VisitedPhis) {
1336 auto AccessIdx = getAccessIndices(I: Ptr, DeadInsts, VisitedPhis);
1337 assert(AccessIdx.hasHandleIdx() &&
1338 "Couldn't retrieve handle index. This is guaranteed by "
1339 "getAccessIndices");
1340
1341 IRBuilder<> Builder(Ptr);
1342 if (isa<PHINode>(Val: Ptr))
1343 Builder.SetInsertPoint(Ptr->getParent()->getFirstNonPHIIt());
1344 IntrinsicInst *Handle = cast<IntrinsicInst>(Val: OldHandle->clone());
1345 Handle->setArgOperand(/*Index=*/i: 3, v: AccessIdx.HandleIdx);
1346 Builder.Insert(I: Handle);
1347
1348 if (Ptr->getType()->isPointerTy()) {
1349 assert(AccessIdx.hasGetPtrIdx() &&
1350 "Couldn't retrieve getpointer index. This is guaranteed by "
1351 "getAccessIndices");
1352 auto *GetPtr = Builder.CreateIntrinsic(RetTy: Ptr->getType(),
1353 ID: Intrinsic::dx_resource_getpointer,
1354 Args: {Handle, AccessIdx.GetPtrIdx});
1355
1356 // Reapply any byte offset that came from GEPs in the original ptr chain so
1357 // that later access lowering still sees it.
1358 Value *Result = GetPtr;
1359 if (AccessIdx.hasOffsetIdx())
1360 Result = Builder.CreatePtrAdd(Ptr: GetPtr, Offset: AccessIdx.OffsetIdx);
1361 Ptr->replaceAllUsesWith(V: Result);
1362 } else {
1363 assert(Ptr->getType()->isTargetExtTy() && !AccessIdx.hasGetPtrIdx() &&
1364 "Unexpected resource access operand type");
1365 Ptr->replaceAllUsesWith(V: Handle);
1366 }
1367
1368 DeadInsts.insert(X: Ptr);
1369}
1370
1371// Try to legalize dx.resource.handlefrom.*.binding and dx.resource.getpointer
1372// calls with their respective index values and propagate the index values to
1373// be used at resource access.
1374//
1375// If it can't be transformed to be legal then:
1376//
1377// Reports an error if a resource access is not guaranteed into a unique global
1378// resource.
1379//
1380// Returns true if any changes are made.
1381static bool legalizeResourceHandles(Function &F, DXILResourceTypeMap &DRTM) {
1382 SmallSetVector<Instruction *, 16> DeadInsts;
1383 PhiAccessIndicesMap VisitedPhis;
1384
1385 for (BasicBlock &BB : make_early_inc_range(Range&: F)) {
1386 for (Instruction &I : BB) {
1387 if (auto *HandleOp = getHandleOperand(AI: &I)) {
1388 SmallVector<IntrinsicInst *> Handles = collectUsedHandles(Ptr: HandleOp);
1389 unsigned NumHandles = Handles.size();
1390 if (NumHandles <= 1)
1391 continue; // Legal, no-replacement required
1392
1393 bool SameGlobalBinding = true;
1394 hlsl::Binding B = getHandleIntrinsicBinding(Handle: Handles[0], DRTM);
1395 for (unsigned Idx = 1; Idx < NumHandles; Idx++)
1396 SameGlobalBinding &=
1397 (B == getHandleIntrinsicBinding(Handle: Handles[Idx], DRTM));
1398
1399 if (!SameGlobalBinding)
1400 diagnoseNonUniqueResourceAccess(I: &I, Handles);
1401
1402 replaceHandleWithIndices(Ptr: getHandleRoot(I: HandleOp), OldHandle: Handles[0], DeadInsts,
1403 VisitedPhis);
1404 }
1405 }
1406 }
1407
1408 bool MadeChanges = false;
1409
1410 // Set up the phis to track if they are erased below
1411 SmallVector<WeakTrackingVH> ResourcePhis;
1412 for (const auto &HandleToIndex : VisitedPhis)
1413 ResourcePhis.push_back(Elt: HandleToIndex.first);
1414
1415 for (auto *I : llvm::reverse(C&: DeadInsts))
1416 if (I->hasNUses(N: 0)) { // Handle can still be used outside of replaced path
1417 I->eraseFromParent();
1418 MadeChanges = true;
1419 }
1420
1421 // Any remaining phi nodes are now looped with another phi node and have no
1422 // other uses
1423 for (WeakTrackingVH &VH : ResourcePhis)
1424 if (VH) // True if not removed above or already in this loop
1425 MadeChanges |= RecursivelyDeleteDeadPHINode(PN: cast<PHINode>(Val&: VH));
1426
1427 return MadeChanges;
1428}
1429
1430static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
1431 SmallVector<User *> Worklist;
1432 for (User *U : II->users())
1433 Worklist.push_back(Elt: U);
1434
1435 SmallVector<Instruction *> DeadInsts;
1436 while (!Worklist.empty()) {
1437 User *U = Worklist.back();
1438 Worklist.pop_back();
1439
1440 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: U)) {
1441 for (User *U : GEP->users())
1442 Worklist.push_back(Elt: U);
1443 DeadInsts.push_back(Elt: GEP);
1444
1445 } else if (auto *SI = dyn_cast<StoreInst>(Val: U)) {
1446 assert(SI->getValueOperand() != II && "Pointer escaped!");
1447 createStoreIntrinsic(II, SI, RTI);
1448 DeadInsts.push_back(Elt: SI);
1449
1450 } else if (auto *LI = dyn_cast<LoadInst>(Val: U)) {
1451 createLoadIntrinsic(II, LI, RTI);
1452 DeadInsts.push_back(Elt: LI);
1453 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: U)) {
1454 createAtomicBinOpIntrinsic(II, AI, RTI);
1455 DeadInsts.push_back(Elt: AI);
1456 } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: U)) {
1457 createAtomicCompareExchangeIntrinsic(II, AI: CXI, RTI);
1458 DeadInsts.push_back(Elt: CXI);
1459 } else
1460 llvm_unreachable("Unhandled instruction - pointer escaped?");
1461 }
1462
1463 // Traverse the now-dead instructions in RPO and remove them.
1464 for (Instruction *Dead : llvm::reverse(C&: DeadInsts))
1465 Dead->eraseFromParent();
1466 II->eraseFromParent();
1467}
1468
1469static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) {
1470 SmallVector<std::pair<IntrinsicInst *, dxil::ResourceTypeInfo>> Resources;
1471 for (BasicBlock &BB : make_early_inc_range(Range&: F))
1472 for (Instruction &I : BB)
1473 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1474 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer ||
1475 II->getIntrinsicID() == Intrinsic::dx_resource_getbasepointer) {
1476 auto *HandleTy = cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType());
1477 assert(
1478 (DRTM[HandleTy].isCBuffer() ||
1479 II->getIntrinsicID() != Intrinsic::dx_resource_getbasepointer) &&
1480 "dx_resource_getbasepointer should only be used by cbuffers");
1481 Resources.emplace_back(Args&: II, Args&: DRTM[HandleTy]);
1482 }
1483
1484 for (auto &[II, RI] : Resources)
1485 replaceAccess(II, RTI&: RI);
1486
1487 return !Resources.empty();
1488}
1489
1490PreservedAnalyses DXILResourceAccess::run(Function &F,
1491 FunctionAnalysisManager &FAM) {
1492 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1493 DXILResourceTypeMap *DRTM =
1494 MAMProxy.getCachedResult<DXILResourceTypeAnalysis>(IR&: *F.getParent());
1495 assert(DRTM && "DXILResourceTypeAnalysis must be available");
1496
1497 bool MadeHandleChanges = legalizeResourceHandles(F, DRTM&: *DRTM);
1498 bool MadeResourceChanges = transformResourcePointers(F, DRTM&: *DRTM);
1499 if (!(MadeHandleChanges || MadeResourceChanges))
1500 return PreservedAnalyses::all();
1501
1502 PreservedAnalyses PA;
1503 PA.preserve<DXILResourceTypeAnalysis>();
1504 return PA;
1505}
1506
1507namespace {
1508class DXILResourceAccessLegacy : public FunctionPass {
1509public:
1510 bool runOnFunction(Function &F) override {
1511 DXILResourceTypeMap &DRTM =
1512 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
1513 bool MadeHandleChanges = legalizeResourceHandles(F, DRTM);
1514 bool MadeResourceChanges = transformResourcePointers(F, DRTM);
1515 return MadeHandleChanges || MadeResourceChanges;
1516 }
1517 StringRef getPassName() const override { return "DXIL Resource Access"; }
1518 DXILResourceAccessLegacy() : FunctionPass(ID) {}
1519
1520 static char ID; // Pass identification.
1521 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
1522 AU.addRequired<DXILResourceTypeWrapperPass>();
1523 }
1524};
1525char DXILResourceAccessLegacy::ID = 0;
1526} // end anonymous namespace
1527
1528INITIALIZE_PASS_BEGIN(DXILResourceAccessLegacy, DEBUG_TYPE,
1529 "DXIL Resource Access", false, false)
1530INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
1531INITIALIZE_PASS_END(DXILResourceAccessLegacy, DEBUG_TYPE,
1532 "DXIL Resource Access", false, false)
1533
1534FunctionPass *llvm::createDXILResourceAccessLegacyPass() {
1535 return new DXILResourceAccessLegacy();
1536}
1537