1//===- DXILOpLowering.cpp - Lowering to DXIL operations -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DXILConstants.h"
10#include "DXILOpBuilder.h"
11#include "DXILRootSignature.h"
12#include "DXILShaderFlags.h"
13#include "DXILSignatureAnalysis.h"
14#include "DirectX.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/Analysis/DXILMetadataAnalysis.h"
17#include "llvm/Analysis/DXILResource.h"
18#include "llvm/CodeGen/Passes.h"
19#include "llvm/IR/Constant.h"
20#include "llvm/IR/DiagnosticInfo.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsDirectX.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PassManager.h"
28#include "llvm/IR/Use.h"
29#include "llvm/IR/ValueHandle.h"
30#include "llvm/InitializePasses.h"
31#include "llvm/Pass.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/FormatVariadic.h"
34
35#define DEBUG_TYPE "dxil-op-lower"
36
37using namespace llvm;
38using namespace llvm::dxil;
39
40/// Write mask covering all four components of a UAV element. Typed UAV stores
41/// (textures and typed buffers) must always use this mask - the DXIL validator
42/// rejects anything narrower. Only raw and / structured buffer stores may use a
43/// partial mask.
44static constexpr uint8_t TypedUAVStoreWriteMask = 0xF;
45
46namespace {
47class OpLowerer {
48 Module &M;
49 DXILOpBuilder OpBuilder;
50 DXILResourceMap &DRM;
51 DXILResourceTypeMap &DRTM;
52 const ModuleMetadataInfo &MMDI;
53 SmallVector<CallInst *> CleanupCasts;
54 Function *CleanupNURI = nullptr;
55
56public:
57 OpLowerer(Module &M, DXILResourceMap &DRM, DXILResourceTypeMap &DRTM,
58 const ModuleMetadataInfo &MMDI)
59 : M(M), OpBuilder(M), DRM(DRM), DRTM(DRTM), MMDI(MMDI) {}
60
61 /// Replace every call to \c F using \c ReplaceCall, and then erase \c F. If
62 /// there is an error replacing a call, we emit a diagnostic and return true.
63 [[nodiscard]] bool
64 replaceFunction(Function &F,
65 llvm::function_ref<Error(CallInst *CI)> ReplaceCall) {
66 for (User *U : make_early_inc_range(Range: F.users())) {
67 CallInst *CI = dyn_cast<CallInst>(Val: U);
68 if (!CI)
69 continue;
70
71 if (Error E = ReplaceCall(CI)) {
72 std::string Message(toString(E: std::move(E)));
73 M.getContext().diagnose(DI: DiagnosticInfoUnsupported(
74 *CI->getFunction(), Message, CI->getDebugLoc()));
75
76 return true;
77 }
78 }
79 if (F.user_empty())
80 F.eraseFromParent();
81 return false;
82 }
83
84 struct IntrinArgSelect {
85 enum class Type {
86#define DXIL_OP_INTRINSIC_ARG_SELECT_TYPE(name) name,
87#include "DXILOperation.inc"
88 };
89 Type Type;
90 int Value;
91 };
92
93 /// Replaces uses of a struct with uses of an equivalent named struct.
94 ///
95 /// DXIL operations that return structs give them well known names, so we need
96 /// to update uses when we switch from an LLVM intrinsic to an op.
97 Error replaceNamedStructUses(CallInst *Intrin, CallInst *DXILOp) {
98 auto *IntrinTy = cast<StructType>(Val: Intrin->getType());
99 auto *DXILOpTy = cast<StructType>(Val: DXILOp->getType());
100 if (!IntrinTy->isLayoutIdentical(Other: DXILOpTy))
101 return make_error<StringError>(
102 Args: "Type mismatch between intrinsic and DXIL op",
103 Args: inconvertibleErrorCode());
104
105 for (Use &U : make_early_inc_range(Range: Intrin->uses()))
106 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: U.getUser()))
107 EVI->setOperand(i_nocapture: 0, Val_nocapture: DXILOp);
108 else if (auto *IVI = dyn_cast<InsertValueInst>(Val: U.getUser()))
109 IVI->setOperand(i_nocapture: 0, Val_nocapture: DXILOp);
110 else
111 return make_error<StringError>(Args: "DXIL ops that return structs may only "
112 "be used by insert- and extractvalue",
113 Args: inconvertibleErrorCode());
114 return Error::success();
115 }
116
117 bool isFast(FastMathFlags Flags) {
118 // HLSL Fast Math doesn't enable AllowContract flag; This can be
119 // removed when we enable it in the future.
120 return Flags.allowReassoc() && Flags.noNaNs() && Flags.noInfs() &&
121 Flags.noSignedZeros() && Flags.allowReciprocal() &&
122 Flags.approxFunc();
123 }
124
125 void setDxPrecise(CallInst *CI) {
126 const StringRef Key = "dx.precise";
127 Module *M = CI->getModule();
128
129 LLVMContext &Ctx = M->getContext();
130 MDNode *One =
131 llvm::MDNode::get(Context&: Ctx, MDs: ConstantAsMetadata::get(C: ConstantInt::get(
132 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1)));
133
134 CI->setMetadata(Kind: Key, Node: One);
135 }
136
137 [[nodiscard]] bool
138 replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp,
139 ArrayRef<IntrinArgSelect> ArgSelects) {
140 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
141 OpBuilder.getIRB().SetInsertPoint(CI);
142 SmallVector<Value *> Args;
143 if (ArgSelects.size()) {
144 for (const IntrinArgSelect &A : ArgSelects) {
145 switch (A.Type) {
146 case IntrinArgSelect::Type::Index:
147 Args.push_back(Elt: CI->getArgOperand(i: A.Value));
148 break;
149 case IntrinArgSelect::Type::I8:
150 Args.push_back(Elt: OpBuilder.getIRB().getInt8(C: (uint8_t)A.Value));
151 break;
152 case IntrinArgSelect::Type::I32:
153 Args.push_back(Elt: OpBuilder.getIRB().getInt32(C: A.Value));
154 break;
155 }
156 }
157 } else {
158 Args.append(in_start: CI->arg_begin(), in_end: CI->arg_end());
159 }
160
161 Expected<CallInst *> OpCall =
162 OpBuilder.tryCreateOp(Op: DXILOp, Args, Name: CI->getName(), RetTy: F.getReturnType());
163 if (Error E = OpCall.takeError())
164 return E;
165
166 if (isa<FPMathOperator>(Val: CI) &&
167 !isFast(Flags: cast<FPMathOperator>(Val: CI)->getFastMathFlags()))
168 setDxPrecise(*OpCall);
169
170 if (isa<StructType>(Val: CI->getType())) {
171 if (Error E = replaceNamedStructUses(Intrin: CI, DXILOp: *OpCall))
172 return E;
173 } else
174 CI->replaceAllUsesWith(V: *OpCall);
175
176 CI->eraseFromParent();
177 return Error::success();
178 });
179 }
180
181 /// Create a cast between a `target("dx")` type and `dx.types.Handle`, which
182 /// is intended to be removed by the end of lowering. This is used to allow
183 /// lowering of ops which need to change their return or argument types in a
184 /// piecemeal way - we can add the casts in to avoid updating all of the uses
185 /// or defs, and by the end all of the casts will be redundant.
186 Value *createTmpHandleCast(Value *V, Type *Ty) {
187 CallInst *Cast = OpBuilder.getIRB().CreateIntrinsicWithoutFolding(
188 ID: Intrinsic::dx_resource_casthandle, OverloadTypes: {Ty, V->getType()}, Args: {V});
189 CleanupCasts.push_back(Elt: Cast);
190 return Cast;
191 }
192
193 void cleanupHandleCasts() {
194 SmallVector<CallInst *> ToRemove;
195 SmallVector<Function *> CastFns;
196
197 for (CallInst *Cast : CleanupCasts) {
198 // These casts were only put in to ease the move from `target("dx")` types
199 // to `dx.types.Handle in a piecemeal way. At this point, all of the
200 // non-cast uses should now be `dx.types.Handle`, and remaining casts
201 // should all form pairs to and from the now unused `target("dx")` type.
202 CastFns.push_back(Elt: Cast->getCalledFunction());
203
204 // If the cast is not to `dx.types.Handle`, it should be the first part of
205 // the pair. Keep track so we can remove it once it has no more uses.
206 if (Cast->getType() != OpBuilder.getHandleType()) {
207 ToRemove.push_back(Elt: Cast);
208 continue;
209 }
210 // Otherwise, we're the second handle in a pair. Forward the arguments and
211 // remove the (second) cast.
212 CallInst *Def = cast<CallInst>(Val: Cast->getOperand(i_nocapture: 0));
213 assert(Def->getIntrinsicID() == Intrinsic::dx_resource_casthandle &&
214 "Unbalanced pair of temporary handle casts");
215 Cast->replaceAllUsesWith(V: Def->getOperand(i_nocapture: 0));
216 Cast->eraseFromParent();
217 }
218 for (CallInst *Cast : ToRemove) {
219 assert(Cast->user_empty() && "Temporary handle cast still has users");
220 Cast->eraseFromParent();
221 }
222
223 // Deduplicate the cast functions so that we only erase each one once.
224 llvm::sort(C&: CastFns);
225 CastFns.erase(CS: llvm::unique(R&: CastFns), CE: CastFns.end());
226 for (Function *F : CastFns)
227 F->eraseFromParent();
228
229 CleanupCasts.clear();
230 }
231
232 void cleanupNonUniformResourceIndexCalls() {
233 // Replace all NonUniformResourceIndex calls with their argument.
234 if (!CleanupNURI)
235 return;
236 for (User *U : make_early_inc_range(Range: CleanupNURI->users())) {
237 CallInst *CI = dyn_cast<CallInst>(Val: U);
238 if (!CI)
239 continue;
240 CI->replaceAllUsesWith(V: CI->getArgOperand(i: 0));
241 CI->eraseFromParent();
242 }
243 CleanupNURI->eraseFromParent();
244 CleanupNURI = nullptr;
245 }
246
247 // Remove the resource global associated with the handleFromBinding call
248 // instruction and their uses as they aren't needed anymore.
249 // TODO: We should verify that all the globals get removed.
250 // It's expected we'll need a custom pass in the future that will eliminate
251 // the need for this here.
252 void removeResourceGlobals(CallInst *CI) {
253 for (User *User : make_early_inc_range(Range: CI->users())) {
254 if (StoreInst *Store = dyn_cast<StoreInst>(Val: User)) {
255 Value *V = Store->getOperand(i_nocapture: 1);
256 Store->eraseFromParent();
257 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: V))
258 if (GV->use_empty()) {
259 GV->removeDeadConstantUsers();
260 GV->eraseFromParent();
261 }
262 }
263 }
264 }
265
266 void replaceHandleFromBindingCall(CallInst *CI, Value *Replacement) {
267 assert(CI->getCalledFunction()->getIntrinsicID() ==
268 Intrinsic::dx_resource_handlefrombinding);
269
270 removeResourceGlobals(CI);
271
272 auto *NameGlobal = dyn_cast<llvm::GlobalVariable>(Val: CI->getArgOperand(i: 4));
273
274 CI->replaceAllUsesWith(V: Replacement);
275 CI->eraseFromParent();
276
277 if (NameGlobal && NameGlobal->use_empty())
278 NameGlobal->eraseFromParent();
279 }
280
281 bool hasNonUniformIndex(Value *IndexOp) {
282 if (isa<llvm::Constant>(Val: IndexOp))
283 return false;
284
285 SmallVector<Value *, 16> Worklist;
286 SmallPtrSet<Value *, 16> Visited;
287 Worklist.push_back(Elt: IndexOp);
288
289 while (!Worklist.empty()) {
290 Value *V = Worklist.pop_back_val();
291
292 if (isa<llvm::Constant>(Val: V))
293 continue;
294
295 if (!Visited.insert(Ptr: V).second)
296 continue;
297
298 if (auto *CI = dyn_cast<CallInst>(Val: V))
299 if (CI->getIntrinsicID() == Intrinsic::dx_resource_nonuniformindex)
300 return true;
301
302 // If it's a PHI node, check ALL incoming values —
303 // taint from ANY predecessor counts
304 if (auto *Phi = dyn_cast<PHINode>(Val: V)) {
305 for (Value *Incoming : Phi->incoming_values())
306 Worklist.push_back(Elt: Incoming);
307 continue;
308 }
309
310 if (auto *Inst = dyn_cast<Instruction>(Val: V))
311 if (Inst->getNumOperands() > 0 && !Inst->isTerminator())
312 for (Value *Op : Inst->operands())
313 Worklist.push_back(Elt: Op);
314 }
315 return false;
316 }
317
318 Error validateRawBufferElementIndex(Value *Resource, Value *ElementIndex) {
319 bool IsStructured =
320 cast<RawBufferExtType>(Val: Resource->getType())->isStructured();
321 bool IsPoison = isa<PoisonValue>(Val: ElementIndex);
322
323 if (IsStructured && IsPoison)
324 return make_error<StringError>(
325 Args: "Element index of structured buffer may not be poison",
326 Args: inconvertibleErrorCode());
327
328 if (!IsStructured && !IsPoison)
329 return make_error<StringError>(
330 Args: "Element index of raw buffer must be poison",
331 Args: inconvertibleErrorCode());
332
333 return Error::success();
334 }
335
336 [[nodiscard]] bool lowerToCreateHandle(Function &F) {
337 IRBuilder<> &IRB = OpBuilder.getIRB();
338 Type *Int8Ty = IRB.getInt8Ty();
339 Type *Int32Ty = IRB.getInt32Ty();
340 Type *Int1Ty = IRB.getInt1Ty();
341
342 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
343 IRB.SetInsertPoint(CI);
344
345 auto *It = DRM.find(Key: CI);
346 assert(It != DRM.end() && "Resource not in map?");
347 dxil::ResourceInfo &RI = *It;
348
349 const auto &Binding = RI.getBinding();
350 dxil::ResourceClass RC = DRTM[RI.getHandleTy()].getResourceClass();
351
352 Value *IndexOp = CI->getArgOperand(i: 3);
353 if (Binding.LowerBound != 0)
354 IndexOp = IRB.CreateAdd(LHS: IndexOp,
355 RHS: ConstantInt::get(Ty: Int32Ty, V: Binding.LowerBound));
356
357 bool HasNonUniformIndex =
358 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);
359 std::array<Value *, 4> Args{
360 ConstantInt::get(Ty: Int8Ty, V: llvm::to_underlying(E: RC)),
361 ConstantInt::get(Ty: Int32Ty, V: Binding.BindingID), IndexOp,
362 ConstantInt::get(Ty: Int1Ty, V: HasNonUniformIndex)};
363 Expected<CallInst *> OpCall =
364 OpBuilder.tryCreateOp(Op: OpCode::CreateHandle, Args, Name: CI->getName());
365 if (Error E = OpCall.takeError())
366 return E;
367
368 Value *Cast = createTmpHandleCast(V: *OpCall, Ty: CI->getType());
369 replaceHandleFromBindingCall(CI, Replacement: Cast);
370 return Error::success();
371 });
372 }
373
374 [[nodiscard]] bool lowerToBindAndAnnotateHandle(Function &F) {
375 IRBuilder<> &IRB = OpBuilder.getIRB();
376 Type *Int32Ty = IRB.getInt32Ty();
377 Type *Int1Ty = IRB.getInt1Ty();
378
379 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
380 IRB.SetInsertPoint(CI);
381
382 auto *It = DRM.find(Key: CI);
383 assert(It != DRM.end() && "Resource not in map?");
384 dxil::ResourceInfo &RI = *It;
385
386 const auto &Binding = RI.getBinding();
387 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];
388 dxil::ResourceClass RC = RTI.getResourceClass();
389
390 Value *IndexOp = CI->getArgOperand(i: 3);
391 if (Binding.LowerBound != 0)
392 IndexOp = IRB.CreateAdd(LHS: IndexOp,
393 RHS: ConstantInt::get(Ty: Int32Ty, V: Binding.LowerBound));
394
395 std::pair<uint32_t, uint32_t> Props =
396 RI.getAnnotateProps(M&: *F.getParent(), RTI);
397
398 // For `CreateHandleFromBinding` we need the upper bound rather than the
399 // size, so we need to be careful about the difference for "unbounded".
400 uint32_t UpperBound = Binding.Size == 0
401 ? std::numeric_limits<uint32_t>::max()
402 : Binding.LowerBound + Binding.Size - 1;
403 Constant *ResBind = OpBuilder.getResBind(LowerBound: Binding.LowerBound, UpperBound,
404 SpaceID: Binding.Space, RC);
405 bool NonUniformIndex =
406 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);
407 Constant *NonUniformOp = ConstantInt::get(Ty: Int1Ty, V: NonUniformIndex);
408 std::array<Value *, 3> BindArgs{ResBind, IndexOp, NonUniformOp};
409 Expected<CallInst *> OpBind = OpBuilder.tryCreateOp(
410 Op: OpCode::CreateHandleFromBinding, Args: BindArgs, Name: CI->getName());
411 if (Error E = OpBind.takeError())
412 return E;
413
414 std::array<Value *, 2> AnnotateArgs{
415 *OpBind, OpBuilder.getResProps(Word0: Props.first, Word1: Props.second)};
416 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(
417 Op: OpCode::AnnotateHandle, Args: AnnotateArgs,
418 Name: CI->hasName() ? CI->getName() + "_annot" : Twine());
419 if (Error E = OpAnnotate.takeError())
420 return E;
421
422 Value *Cast = createTmpHandleCast(V: *OpAnnotate, Ty: CI->getType());
423 replaceHandleFromBindingCall(CI, Replacement: Cast);
424 return Error::success();
425 });
426 }
427
428 /// Lower `dx.resource.handlefrombinding` intrinsics depending on the shader
429 /// model and taking into account binding information from
430 /// DXILResourceAnalysis.
431 bool lowerHandleFromBinding(Function &F) {
432 if (MMDI.DXILVersion < VersionTuple(1, 6))
433 return lowerToCreateHandle(F);
434 return lowerToBindAndAnnotateHandle(F);
435 }
436
437 bool lowerHandleFromHeap(Function &F) {
438 IRBuilder<> &IRB = OpBuilder.getIRB();
439
440 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
441 IRB.SetInsertPoint(CI);
442
443 auto *It = DRM.find(Key: CI);
444 assert(It != DRM.end() && "Resource not in map?");
445 dxil::ResourceInfo &RI = *It;
446 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];
447
448 Value *IndexOp = CI->getArgOperand(i: 0);
449 Value *IsSamplerHeap =
450 ConstantInt::getBool(Context&: IRB.getContext(), V: RTI.isSampler());
451
452 std::pair<uint32_t, uint32_t> Props =
453 RI.getAnnotateProps(M&: *F.getParent(), RTI);
454
455 bool NonUniformIndex = hasNonUniformIndex(IndexOp);
456 Value *NonUniformOp =
457 ConstantInt::getBool(Context&: IRB.getContext(), V: NonUniformIndex);
458
459 std::array<Value *, 3> Args{IndexOp, IsSamplerHeap, NonUniformOp};
460 Expected<CallInst *> OpCreateHandle = OpBuilder.tryCreateOp(
461 Op: OpCode::CreateHandleFromHeap, Args, Name: CI->getName());
462 if (Error E = OpCreateHandle.takeError())
463 return E;
464
465 std::array<Value *, 2> AnnotateArgs{
466 *OpCreateHandle, OpBuilder.getResProps(Word0: Props.first, Word1: Props.second)};
467 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(
468 Op: OpCode::AnnotateHandle, Args: AnnotateArgs,
469 Name: CI->hasName() ? CI->getName() + "_annot" : Twine());
470 if (Error E = OpAnnotate.takeError())
471 return E;
472
473 Value *Cast = createTmpHandleCast(V: *OpAnnotate, Ty: CI->getType());
474 CI->replaceAllUsesWith(V: Cast);
475 CI->eraseFromParent();
476 return Error::success();
477 });
478 }
479
480 /// Replace uses of \c Intrin with the values in the `dx.ResRet` of \c Op.
481 /// Since we expect to be post-scalarization, make an effort to avoid vectors.
482 Error replaceResRetUses(CallInst *Intrin, CallInst *Op, bool HasCheckBit) {
483 IRBuilder<> &IRB = OpBuilder.getIRB();
484
485 Instruction *OldResult = Intrin;
486 Type *OldTy = Intrin->getType();
487
488 if (HasCheckBit) {
489 auto *ST = cast<StructType>(Val: OldTy);
490
491 Value *CheckOp = nullptr;
492 Type *Int32Ty = IRB.getInt32Ty();
493 for (Use &U : make_early_inc_range(Range: OldResult->uses())) {
494 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: U.getUser())) {
495 ArrayRef<unsigned> Indices = EVI->getIndices();
496 assert(Indices.size() == 1);
497 // We're only interested in uses of the check bit for now.
498 if (Indices[0] != 1)
499 continue;
500 if (!CheckOp) {
501 Value *NewEVI = IRB.CreateExtractValue(Agg: Op, Idxs: 4);
502 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
503 Op: OpCode::CheckAccessFullyMapped, Args: {NewEVI},
504 Name: OldResult->hasName() ? OldResult->getName() + "_check"
505 : Twine(),
506 RetTy: Int32Ty);
507 if (Error E = OpCall.takeError())
508 return E;
509 CheckOp = *OpCall;
510 }
511 EVI->replaceAllUsesWith(V: CheckOp);
512 EVI->eraseFromParent();
513 }
514 }
515
516 if (OldResult->use_empty()) {
517 // Only the check bit was used, so we're done here.
518 OldResult->eraseFromParent();
519 return Error::success();
520 }
521
522 assert(OldResult->hasOneUse() &&
523 isa<ExtractValueInst>(*OldResult->user_begin()) &&
524 "Expected only use to be extract of first element");
525 OldResult = cast<Instruction>(Val: *OldResult->user_begin());
526 OldTy = ST->getElementType(N: 0);
527 }
528
529 // For scalars, we just extract the first element.
530 if (!isa<FixedVectorType>(Val: OldTy)) {
531 Value *EVI = IRB.CreateExtractValue(Agg: Op, Idxs: 0);
532 OldResult->replaceAllUsesWith(V: EVI);
533 OldResult->eraseFromParent();
534 if (OldResult != Intrin) {
535 assert(Intrin->use_empty() && "Intrinsic still has uses?");
536 Intrin->eraseFromParent();
537 }
538 return Error::success();
539 }
540
541 std::array<Value *, 4> Extracts = {};
542 SmallVector<ExtractElementInst *> DynamicAccesses;
543
544 // The users of the operation should all be scalarized, so we attempt to
545 // replace the extractelements with extractvalues directly.
546 for (Use &U : make_early_inc_range(Range: OldResult->uses())) {
547 if (auto *EEI = dyn_cast<ExtractElementInst>(Val: U.getUser())) {
548 if (auto *IndexOp = dyn_cast<ConstantInt>(Val: EEI->getIndexOperand())) {
549 size_t IndexVal = IndexOp->getZExtValue();
550 assert(IndexVal < 4 && "Index into buffer load out of range");
551 if (!Extracts[IndexVal])
552 Extracts[IndexVal] = IRB.CreateExtractValue(Agg: Op, Idxs: IndexVal);
553 EEI->replaceAllUsesWith(V: Extracts[IndexVal]);
554 EEI->eraseFromParent();
555 } else {
556 DynamicAccesses.push_back(Elt: EEI);
557 }
558 }
559 }
560
561 const auto *VecTy = cast<FixedVectorType>(Val: OldTy);
562 const unsigned N = VecTy->getNumElements();
563
564 // If there's a dynamic access we need to round trip through stack memory so
565 // that we don't leave vectors around.
566 if (!DynamicAccesses.empty()) {
567 Type *Int32Ty = IRB.getInt32Ty();
568 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
569
570 Type *ElTy = VecTy->getElementType();
571 Type *ArrayTy = ArrayType::get(ElementType: ElTy, NumElements: N);
572 Value *Alloca = IRB.CreateAlloca(Ty: ArrayTy);
573
574 for (int I = 0, E = N; I != E; ++I) {
575 if (!Extracts[I])
576 Extracts[I] = IRB.CreateExtractValue(Agg: Op, Idxs: I);
577 Value *GEP = GetElementPtrInst::CreateInBounds(
578 PointeeType: ArrayTy, Ptr: Alloca, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: I)}, NameStr: "",
579 InsertBefore: IRB.GetInsertPoint());
580 IRB.CreateStore(Val: Extracts[I], Ptr: GEP);
581 }
582
583 for (ExtractElementInst *EEI : DynamicAccesses) {
584 Value *GEP = GetElementPtrInst::CreateInBounds(
585 PointeeType: ArrayTy, Ptr: Alloca, IdxList: {Zero, EEI->getIndexOperand()}, NameStr: "",
586 InsertBefore: IRB.GetInsertPoint());
587 Value *Load = IRB.CreateLoad(Ty: ElTy, Ptr: GEP);
588 EEI->replaceAllUsesWith(V: Load);
589 EEI->eraseFromParent();
590 }
591 }
592
593 // If we still have uses, then we're not fully scalarized and need to
594 // recreate the vector. This should only happen for things like exported
595 // functions from libraries.
596 if (!OldResult->use_empty()) {
597 for (int I = 0, E = N; I != E; ++I)
598 if (!Extracts[I])
599 Extracts[I] = IRB.CreateExtractValue(Agg: Op, Idxs: I);
600
601 Value *Vec = PoisonValue::get(T: OldTy);
602 for (int I = 0, E = N; I != E; ++I)
603 Vec = IRB.CreateInsertElement(Vec, NewElt: Extracts[I], Idx: I);
604 OldResult->replaceAllUsesWith(V: Vec);
605 }
606
607 OldResult->eraseFromParent();
608 if (OldResult != Intrin) {
609 assert(Intrin->use_empty() && "Intrinsic still has uses?");
610 Intrin->eraseFromParent();
611 }
612
613 return Error::success();
614 }
615
616 [[nodiscard]] bool lowerTypedBufferLoad(Function &F, bool HasCheckBit) {
617 IRBuilder<> &IRB = OpBuilder.getIRB();
618 Type *Int32Ty = IRB.getInt32Ty();
619
620 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
621 IRB.SetInsertPoint(CI);
622
623 Value *Handle =
624 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
625 Value *Index0 = CI->getArgOperand(i: 1);
626 Value *Index1 = UndefValue::get(T: Int32Ty);
627
628 Type *OldTy = CI->getType();
629 if (HasCheckBit)
630 OldTy = cast<StructType>(Val: OldTy)->getElementType(N: 0);
631 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
632
633 std::array<Value *, 3> Args{Handle, Index0, Index1};
634 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
635 Op: OpCode::BufferLoad, Args, Name: CI->getName(), RetTy: NewRetTy);
636 if (Error E = OpCall.takeError())
637 return E;
638 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, HasCheckBit))
639 return E;
640
641 return Error::success();
642 });
643 }
644
645 /// Recover the scalar components of `Vec` from the `insertelement` chain that
646 /// built it. Since we run after the scalarizer, such a chain is usually just
647 /// a temporary gathered to pass the vector to a call. Components the chain
648 /// leaves unset are taken from its base when that base is a constant, which
649 /// is where constant folding puts leading literals.
650 static void collectInsertedElements(Value *Vec,
651 MutableArrayRef<Value *> Elements) {
652 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
653 assert(NumElts <= Elements.size() && "Not enough room for the components");
654
655 SmallVector<InsertElementInst *, 4> Chain;
656 Value *Base = Vec;
657 while (auto *IEI = dyn_cast<InsertElementInst>(Val: Base)) {
658 if (!isa<ConstantInt>(Val: IEI->getOperand(i_nocapture: 2)))
659 break; // This break should never happen below SM6.9.
660 Chain.push_back(Elt: IEI);
661 Base = IEI->getOperand(i_nocapture: 0);
662 }
663
664 if (auto *C = dyn_cast<Constant>(Val: Base))
665 for (unsigned I = 0; I < NumElts; ++I)
666 Elements[I] = C->getAggregateElement(Elt: I);
667
668 // Replay element insertion from the innermost first, so that a repeated
669 // index ends up holding the live value.
670 while (!Chain.empty()) {
671 InsertElementInst *IEI = Chain.pop_back_val();
672 uint64_t IndexVal = cast<ConstantInt>(Val: IEI->getOperand(i_nocapture: 2))->getZExtValue();
673 if (IndexVal < NumElts)
674 Elements[IndexVal] = IEI->getOperand(i_nocapture: 1);
675 }
676 }
677
678 // Copies `Src` into `Args` starting at `ArgIdx`. If `Src` is a vector, its
679 // elements are placed in consecutive slots; otherwise `Src` is stored
680 // directly. At most `MaxElements` elements are expected.
681 static void extractElementsIntoArgs(IRBuilder<> &IRB,
682 MutableArrayRef<Value *> Args,
683 unsigned ArgIdx, Value *Src,
684 unsigned MaxElements) {
685 auto *VecTy = dyn_cast<FixedVectorType>(Val: Src->getType());
686 if (!VecTy) {
687 Args[ArgIdx] = Src;
688 return;
689 }
690
691 unsigned Count = VecTy->getNumElements();
692 assert(Count <= MaxElements && "Too many elements for the arg list");
693
694 SmallVector<Value *, 4> Elements(Count, nullptr);
695 collectInsertedElements(Vec: Src, Elements);
696
697 for (unsigned I = 0; I < Count; ++I)
698 Args[ArgIdx + I] = Elements[I]
699 ? Elements[I]
700 : IRB.CreateExtractElement(
701 Vec: Src, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: I));
702 }
703
704 /// Copy offsets into the argument list at the given index, unless
705 /// the offsets are known to be zero (i.e., a null constant).
706 static void extractNonZeroOffsets(IRBuilder<> &IRB,
707 MutableArrayRef<Value *> Args,
708 unsigned ArgIdx, Value *Offsets,
709 unsigned MaxElements) {
710 auto *COff = dyn_cast<Constant>(Val: Offsets);
711 bool OffsetsAreZero = COff && COff->isNullValue();
712 if (!OffsetsAreZero)
713 extractElementsIntoArgs(IRB, Args, ArgIdx, Src: Offsets, MaxElements);
714 }
715
716 [[nodiscard]] bool lowerTextureLoad(Function &F) {
717 IRBuilder<> &IRB = OpBuilder.getIRB();
718 Type *Int32Ty = IRB.getInt32Ty();
719
720 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
721 IRB.SetInsertPoint(CI);
722
723 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
724 Value *Handle =
725 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
726 Value *Coords = CI->getArgOperand(i: 1);
727 Value *MipLevel = CI->getArgOperand(i: 2);
728 Value *Offsets = CI->getArgOperand(i: 3);
729
730 // A UAV descriptor binds a single mip slice, so there is no mip to select
731 // in the case of a UAV. Multisampled UAVs are the exception: the slot
732 // carries a sample index and stays live.
733 auto *HandleTy = cast<TargetExtType>(Val: CI->getArgOperand(i: 0)->getType());
734 dxil::ResourceTypeInfo &RTI = DRTM[HandleTy];
735 dxil::ResourceKind Kind = RTI.getResourceKind();
736 if (RTI.isUAV() && Kind != dxil::ResourceKind::Texture2DMS &&
737 Kind != dxil::ResourceKind::Texture2DMSArray)
738 MipLevel = UndefValue::get(T: Int32Ty);
739
740 Type *OldTy = CI->getType();
741 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
742
743 Value *Undef = UndefValue::get(T: Int32Ty);
744 std::array<Value *, 8> Args{Handle, MipLevel, Undef, Undef,
745 Undef, Undef, Undef, Undef};
746
747 // Copy coordinates and offsets into Args.
748 extractElementsIntoArgs(IRB, Args, ArgIdx: 2, Src: Coords, MaxElements: 3);
749 extractNonZeroOffsets(IRB, Args, ArgIdx: 5, Offsets, MaxElements: 3);
750
751 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
752 Op: OpCode::TextureLoad, Args, Name: CI->getName(), RetTy: NewRetTy);
753 if (Error E = OpCall.takeError())
754 return E;
755 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/false))
756 return E;
757
758 eraseDeadInsertElementChains(Vectors: VectorArgs);
759
760 return Error::success();
761 });
762 }
763
764 /// Common helper for lowering sample operations (SampleBias, SampleGrad,
765 /// etc.) that share the same pattern: extract handle/sampler, unpack
766 /// coordinates and offsets, build the DXIL arg list, and replace uses.
767 [[nodiscard]] bool lowerSampleOp(
768 Function &F, OpCode Op, unsigned CoordsIdx, unsigned OffsetsIdx,
769 llvm::function_ref<void(IRBuilder<> &, CallInst *,
770 SmallVectorImpl<Value *> &)> EmitExtraArgs) {
771 IRBuilder<> &IRB = OpBuilder.getIRB();
772 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
773 IRB.SetInsertPoint(CI);
774
775 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
776 Value *Handle =
777 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
778 Value *Sampler =
779 createTmpHandleCast(V: CI->getArgOperand(i: 1), Ty: OpBuilder.getHandleType());
780 Value *Coords = CI->getArgOperand(i: CoordsIdx);
781 Value *Offsets = CI->getArgOperand(i: OffsetsIdx);
782
783 Type *OldTy = CI->getType();
784 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
785
786 Value *UndefF = UndefValue::get(T: IRB.getFloatTy());
787 Value *UndefI = UndefValue::get(T: IRB.getInt32Ty());
788 // Common prefix: Handle, Sampler, Coord0..3, Offset0..2
789 SmallVector<Value *, 17> Args{Handle, Sampler, UndefF, UndefF, UndefF,
790 UndefF, UndefI, UndefI, UndefI};
791
792 // Copy coordinates and offsets into Args.
793 extractElementsIntoArgs(IRB, Args, ArgIdx: 2, Src: Coords, MaxElements: 4);
794 extractNonZeroOffsets(IRB, Args, ArgIdx: 6, Offsets, MaxElements: 3);
795
796 // Emit op-specific trailing arguments (e.g. Bias+Clamp, DDX+DDY+Clamp).
797 EmitExtraArgs(IRB, CI, Args);
798
799 Expected<CallInst *> OpCall =
800 OpBuilder.tryCreateOp(Op, Args, Name: CI->getName(), RetTy: NewRetTy);
801 if (Error E = OpCall.takeError())
802 return E;
803 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/false))
804 return E;
805
806 eraseDeadInsertElementChains(Vectors: VectorArgs);
807
808 return Error::success();
809 });
810 }
811
812 [[nodiscard]] bool lowerSample(Function &F, bool HasClamp) {
813 return lowerSampleOp(F, Op: OpCode::Sample, /*CoordsIdx=*/2, /*OffsetsIdx=*/3,
814 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
815 SmallVectorImpl<Value *> &Args) {
816 // Clamp
817 Args.push_back(
818 Elt: HasClamp ? CI->getArgOperand(i: 4)
819 : UndefValue::get(T: IRB.getFloatTy()));
820 });
821 }
822
823 [[nodiscard]] bool lowerSampleBias(Function &F, bool HasClamp) {
824 return lowerSampleOp(
825 F, Op: OpCode::SampleBias, /*CoordsIdx=*/2, /*OffsetsIdx=*/4,
826 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
827 SmallVectorImpl<Value *> &Args) {
828 // Bias is operand 3.
829 Args.push_back(Elt: CI->getArgOperand(i: 3));
830 // Clamp
831 Args.push_back(Elt: HasClamp ? CI->getArgOperand(i: 5)
832 : UndefValue::get(T: IRB.getFloatTy()));
833 });
834 }
835
836 [[nodiscard]] bool lowerSampleLevel(Function &F) {
837 return lowerSampleOp(
838 F, Op: OpCode::SampleLevel, /*CoordsIdx=*/2, /*OffsetsIdx=*/4,
839 EmitExtraArgs: [](IRBuilder<> &, CallInst *CI, SmallVectorImpl<Value *> &Args) {
840 // LOD is operand 3.
841 Args.push_back(Elt: CI->getArgOperand(i: 3));
842 });
843 }
844
845 [[nodiscard]] bool lowerSampleGrad(Function &F, bool HasClamp) {
846 return lowerSampleOp(
847 F, Op: OpCode::SampleGrad, /*CoordsIdx=*/2, /*OffsetsIdx=*/5,
848 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
849 SmallVectorImpl<Value *> &Args) {
850 Value *DDX = CI->getArgOperand(i: 3);
851 Value *DDY = CI->getArgOperand(i: 4);
852 Value *UndefF = UndefValue::get(T: IRB.getFloatTy());
853 // DDX0..2
854 size_t DDXStart = Args.size();
855 Args.append(NumInputs: 3, Elt: UndefF);
856 extractElementsIntoArgs(IRB, Args, ArgIdx: DDXStart, Src: DDX, MaxElements: 3);
857 // DDY0..2
858 size_t DDYStart = Args.size();
859 Args.append(NumInputs: 3, Elt: UndefF);
860 extractElementsIntoArgs(IRB, Args, ArgIdx: DDYStart, Src: DDY, MaxElements: 3);
861 // Clamp
862 Args.push_back(Elt: HasClamp ? CI->getArgOperand(i: 6) : UndefF);
863 });
864 }
865
866 [[nodiscard]] bool lowerRawBufferLoad(Function &F) {
867 const DataLayout &DL = F.getDataLayout();
868 IRBuilder<> &IRB = OpBuilder.getIRB();
869 Type *Int8Ty = IRB.getInt8Ty();
870 Type *Int32Ty = IRB.getInt32Ty();
871
872 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
873 IRB.SetInsertPoint(CI);
874
875 Type *OldTy = cast<StructType>(Val: CI->getType())->getElementType(N: 0);
876 Type *ScalarTy = OldTy->getScalarType();
877 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: ScalarTy);
878
879 Value *Handle =
880 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
881 Value *Index0 = CI->getArgOperand(i: 1);
882 Value *Index1 = CI->getArgOperand(i: 2);
883 uint64_t NumElements =
884 DL.getTypeSizeInBits(Ty: OldTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
885 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: ~(~0U << NumElements));
886 Value *Align =
887 ConstantInt::get(Ty: Int32Ty, V: DL.getPrefTypeAlign(Ty: ScalarTy).value());
888
889 if (Error E = validateRawBufferElementIndex(Resource: CI->getOperand(i_nocapture: 0), ElementIndex: Index1))
890 return E;
891 if (isa<PoisonValue>(Val: Index1))
892 Index1 = UndefValue::get(T: Index1->getType());
893
894 Expected<CallInst *> OpCall =
895 MMDI.DXILVersion >= VersionTuple(1, 2)
896 ? OpBuilder.tryCreateOp(Op: OpCode::RawBufferLoad,
897 Args: {Handle, Index0, Index1, Mask, Align},
898 Name: CI->getName(), RetTy: NewRetTy)
899 : OpBuilder.tryCreateOp(Op: OpCode::BufferLoad,
900 Args: {Handle, Index0, Index1}, Name: CI->getName(),
901 RetTy: NewRetTy);
902 if (Error E = OpCall.takeError())
903 return E;
904 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/true))
905 return E;
906
907 return Error::success();
908 });
909 }
910
911 [[nodiscard]] bool lowerCBufferLoad(Function &F) {
912 IRBuilder<> &IRB = OpBuilder.getIRB();
913
914 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
915 IRB.SetInsertPoint(CI);
916
917 Type *OldTy = cast<StructType>(Val: CI->getType())->getElementType(N: 0);
918 Type *ScalarTy = OldTy->getScalarType();
919 Type *NewRetTy = OpBuilder.getCBufRetType(ElementTy: ScalarTy);
920
921 Value *Handle =
922 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
923 Value *Index = CI->getArgOperand(i: 1);
924
925 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
926 Op: OpCode::CBufferLoadLegacy, Args: {Handle, Index}, Name: CI->getName(), RetTy: NewRetTy);
927 if (Error E = OpCall.takeError())
928 return E;
929 if (Error E = replaceNamedStructUses(Intrin: CI, DXILOp: *OpCall))
930 return E;
931
932 CI->eraseFromParent();
933 return Error::success();
934 });
935 }
936
937 [[nodiscard]] bool lowerUpdateCounter(Function &F) {
938 IRBuilder<> &IRB = OpBuilder.getIRB();
939 Type *Int32Ty = IRB.getInt32Ty();
940
941 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
942 IRB.SetInsertPoint(CI);
943 Value *Handle =
944 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
945 Value *Op1 = CI->getArgOperand(i: 1);
946
947 std::array<Value *, 2> Args{Handle, Op1};
948
949 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
950 Op: OpCode::UpdateCounter, Args, Name: CI->getName(), RetTy: Int32Ty);
951
952 if (Error E = OpCall.takeError())
953 return E;
954
955 CI->replaceAllUsesWith(V: *OpCall);
956 CI->eraseFromParent();
957 return Error::success();
958 });
959 }
960
961 [[nodiscard]] bool lowerBarrierByMemoryHandle(Function &F) {
962 IRBuilder<> &IRB = OpBuilder.getIRB();
963
964 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
965 IRB.SetInsertPoint(CI);
966 Value *Handle =
967 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
968 Value *SemanticFlags = CI->getArgOperand(i: 1);
969
970 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
971 Op: OpCode::BarrierByMemoryHandle, Args: {Handle, SemanticFlags});
972 if (Error E = OpCall.takeError())
973 return E;
974
975 CI->eraseFromParent();
976 return Error::success();
977 });
978 }
979
980 [[nodiscard]] bool lowerGetDimensionsX(Function &F) {
981 IRBuilder<> &IRB = OpBuilder.getIRB();
982 Type *Int32Ty = IRB.getInt32Ty();
983
984 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
985 IRB.SetInsertPoint(CI);
986 Value *Handle =
987 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
988 Value *Undef = UndefValue::get(T: Int32Ty);
989
990 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
991 Op: OpCode::GetDimensions, Args: {Handle, Undef}, Name: CI->getName(), RetTy: Int32Ty);
992 if (Error E = OpCall.takeError())
993 return E;
994 Value *Dim = IRB.CreateExtractValue(Agg: *OpCall, Idxs: 0);
995
996 CI->replaceAllUsesWith(V: Dim);
997 CI->eraseFromParent();
998 return Error::success();
999 });
1000 }
1001
1002 [[nodiscard]] bool lowerGetPointer(Function &F) {
1003 // These should have already been handled in DXILResourceAccess, so we can
1004 // just clean up the dead prototype.
1005 assert(F.user_empty() && "getpointer operations should have been removed");
1006 F.eraseFromParent();
1007 return false;
1008 }
1009
1010 /// Splits the value operand of a resource store into its (at most four)
1011 /// scalar components. Slots beyond the length of `Data` are filled with
1012 /// `undef` when `FillWithUndef` is set (raw and structured buffers), or with
1013 /// the first component otherwise (typed UAVs, which must write all four
1014 /// components - repeating the first one matches DXC).
1015 static std::array<Value *, 4> splitStoreData(IRBuilder<> &IRB, Value *Data,
1016 uint64_t NumElements,
1017 bool FillWithUndef) {
1018 std::array<Value *, 4> DataElements{nullptr, nullptr, nullptr, nullptr};
1019 extractElementsIntoArgs(IRB, Args: DataElements, ArgIdx: 0, Src: Data, MaxElements: 4);
1020
1021 // For any elements beyond the length of the vector, we should fill it up
1022 // with undef - however, for typed UAVs we repeat the first element to
1023 // match DXC.
1024 for (uint64_t I = NumElements, E = 4; I < E; ++I)
1025 if (DataElements[I] == nullptr)
1026 DataElements[I] =
1027 FillWithUndef ? UndefValue::get(T: Data->getType()->getScalarType())
1028 : DataElements[0];
1029
1030 return DataElements;
1031 }
1032
1033 /// Erase the chain of `insertelement`s that only existed to build up a vector
1034 /// operand of an intrinsic we've just replaced.
1035 static void eraseDeadInsertElementChain(Value *Data) {
1036 auto *IEI = dyn_cast<InsertElementInst>(Val: Data);
1037 while (IEI && IEI->use_empty()) {
1038 InsertElementInst *Tmp = IEI;
1039 IEI = dyn_cast<InsertElementInst>(Val: IEI->getOperand(i_nocapture: 0));
1040 Tmp->eraseFromParent();
1041 }
1042 }
1043
1044 [[nodiscard]] bool lowerBufferStore(Function &F, bool IsRaw) {
1045 const DataLayout &DL = F.getDataLayout();
1046 IRBuilder<> &IRB = OpBuilder.getIRB();
1047 Type *Int8Ty = IRB.getInt8Ty();
1048 Type *Int32Ty = IRB.getInt32Ty();
1049
1050 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1051 IRB.SetInsertPoint(CI);
1052
1053 Value *Handle =
1054 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1055 Value *Index0 = CI->getArgOperand(i: 1);
1056 Value *Index1 = IsRaw ? CI->getArgOperand(i: 2) : UndefValue::get(T: Int32Ty);
1057
1058 if (IsRaw) {
1059 if (Error E = validateRawBufferElementIndex(Resource: CI->getOperand(i_nocapture: 0), ElementIndex: Index1))
1060 return E;
1061 if (isa<PoisonValue>(Val: Index1))
1062 Index1 = UndefValue::get(T: Index1->getType());
1063 }
1064
1065 Value *Data = CI->getArgOperand(i: IsRaw ? 3 : 2);
1066 Type *DataTy = Data->getType();
1067 Type *ScalarTy = DataTy->getScalarType();
1068
1069 uint64_t NumElements =
1070 DL.getTypeSizeInBits(Ty: DataTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
1071 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: IsRaw ? ~(~0U << NumElements)
1072 : TypedUAVStoreWriteMask);
1073
1074 // TODO: check that we only have vector or scalar...
1075 if (NumElements > 4)
1076 return make_error<StringError>(
1077 Args: "Buffer store data must have at most 4 elements",
1078 Args: inconvertibleErrorCode());
1079
1080 std::array<Value *, 4> DataElements =
1081 splitStoreData(IRB, Data, NumElements, /*FillWithUndef=*/IsRaw);
1082
1083 dxil::OpCode Op = OpCode::BufferStore;
1084 SmallVector<Value *, 9> Args{
1085 Handle, Index0, Index1, DataElements[0],
1086 DataElements[1], DataElements[2], DataElements[3], Mask};
1087 if (IsRaw && MMDI.DXILVersion >= VersionTuple(1, 2)) {
1088 Op = OpCode::RawBufferStore;
1089 // RawBufferStore requires the alignment
1090 Args.push_back(
1091 Elt: ConstantInt::get(Ty: Int32Ty, V: DL.getPrefTypeAlign(Ty: ScalarTy).value()));
1092 }
1093 Expected<CallInst *> OpCall =
1094 OpBuilder.tryCreateOp(Op, Args, Name: CI->getName());
1095 if (Error E = OpCall.takeError())
1096 return E;
1097
1098 CI->eraseFromParent();
1099 eraseDeadInsertElementChain(Data);
1100
1101 return Error::success();
1102 });
1103 }
1104
1105 /// Snapshot the vector-typed arguments of `CI` so their `insertelement`
1106 /// chains can be cleaned up once `CI` has been replaced. The handles are weak
1107 /// because two arguments can share an `insertelement` chain.
1108 static SmallVector<WeakTrackingVH, 4> collectVectorArgs(CallInst *CI) {
1109 SmallVector<WeakTrackingVH, 4> Vectors;
1110 for (Value *Arg : CI->args())
1111 if (isa<FixedVectorType>(Val: Arg->getType()))
1112 Vectors.emplace_back(Args&: Arg);
1113 return Vectors;
1114 }
1115
1116 static void eraseDeadInsertElementChains(ArrayRef<WeakTrackingVH> Vectors) {
1117 for (const WeakTrackingVH &VH : Vectors)
1118 if (Value *V = VH)
1119 eraseDeadInsertElementChain(Data: V);
1120 }
1121
1122 [[nodiscard]] bool lowerTextureStore(Function &F) {
1123 const DataLayout &DL = F.getDataLayout();
1124 IRBuilder<> &IRB = OpBuilder.getIRB();
1125 Type *Int8Ty = IRB.getInt8Ty();
1126 Type *Int32Ty = IRB.getInt32Ty();
1127
1128 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1129 IRB.SetInsertPoint(CI);
1130
1131 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
1132 Value *Handle =
1133 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1134 Value *Coords = CI->getArgOperand(i: 1);
1135 Value *Data = CI->getArgOperand(i: 2);
1136
1137 Type *DataTy = Data->getType();
1138 Type *ScalarTy = DataTy->getScalarType();
1139 uint64_t NumElements =
1140 DL.getTypeSizeInBits(Ty: DataTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
1141 if (NumElements > 4)
1142 return make_error<StringError>(
1143 Args: "Texture store data must have at most 4 elements",
1144 Args: inconvertibleErrorCode());
1145
1146 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: TypedUAVStoreWriteMask);
1147 std::array<Value *, 4> DataElements =
1148 splitStoreData(IRB, Data, NumElements, /*FillWithUndef=*/false);
1149
1150 Value *Undef = UndefValue::get(T: Int32Ty);
1151 std::array<Value *, 9> Args{
1152 Handle, Undef, Undef,
1153 Undef, DataElements[0], DataElements[1],
1154 DataElements[2], DataElements[3], Mask};
1155
1156 // Copy the coordinates into Args.
1157 extractElementsIntoArgs(IRB, Args, ArgIdx: 1, Src: Coords, MaxElements: 3);
1158
1159 Expected<CallInst *> OpCall =
1160 OpBuilder.tryCreateOp(Op: OpCode::TextureStore, Args, Name: CI->getName());
1161 if (Error E = OpCall.takeError())
1162 return E;
1163
1164 CI->eraseFromParent();
1165 eraseDeadInsertElementChains(Vectors: VectorArgs);
1166
1167 return Error::success();
1168 });
1169 }
1170
1171 [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) {
1172 IRBuilder<> &IRB = OpBuilder.getIRB();
1173
1174 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1175 IRB.SetInsertPoint(CI);
1176
1177 // Cast the target-extension typed handle to `%dx.types.Handle`, tracked
1178 // via CleanupCasts so the pair is reconciled by `cleanupHandleCasts`.
1179 Value *Handle =
1180 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1181 Value *BinOp = CI->getArgOperand(i: 1);
1182 Value *Coord0 = CI->getArgOperand(i: 2);
1183 Value *Coord1 = CI->getArgOperand(i: 3);
1184 Value *Coord2 = CI->getArgOperand(i: 4);
1185 Value *NewValue = CI->getArgOperand(i: 5);
1186
1187 std::array<Value *, 6> Args{Handle, BinOp, Coord0,
1188 Coord1, Coord2, NewValue};
1189 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
1190 Op: dxil::OpCode::AtomicBinOp, Args, Name: CI->getName(), RetTy: CI->getType());
1191 if (Error E = OpCall.takeError()) {
1192 // Preserve the DXIL op error text but attach it as a
1193 // DiagnosticInfoUnsupported so we don't crash with a dangling call.
1194 std::string Message(toString(E: std::move(E)));
1195 CI->getContext().diagnose(DI: DiagnosticInfoUnsupported(
1196 *CI->getFunction(), Message, CI->getDebugLoc()));
1197 CI->replaceAllUsesWith(V: PoisonValue::get(T: CI->getType()));
1198 CI->eraseFromParent();
1199 return Error::success();
1200 }
1201
1202 CI->replaceAllUsesWith(V: *OpCall);
1203 CI->eraseFromParent();
1204 return Error::success();
1205 });
1206 }
1207
1208 [[nodiscard]] bool lowerResourceAtomicCompareExchange(Function &F) {
1209 IRBuilder<> &IRB = OpBuilder.getIRB();
1210
1211 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1212 IRB.SetInsertPoint(CI);
1213
1214 // Cast the target-extension typed handle to `%dx.types.Handle`, tracked
1215 // via CleanupCasts so the pair is reconciled by `cleanupHandleCasts`.
1216 Value *Handle =
1217 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1218 Value *Coord0 = CI->getArgOperand(i: 1);
1219 Value *Coord1 = CI->getArgOperand(i: 2);
1220 Value *Coord2 = CI->getArgOperand(i: 3);
1221 Value *CompareValue = CI->getArgOperand(i: 4);
1222 Value *NewValue = CI->getArgOperand(i: 5);
1223
1224 std::array<Value *, 6> Args{Handle, Coord0, Coord1,
1225 Coord2, CompareValue, NewValue};
1226 Expected<CallInst *> OpCall =
1227 OpBuilder.tryCreateOp(Op: dxil::OpCode::AtomicCompareExchange, Args,
1228 Name: CI->getName(), RetTy: CI->getType());
1229 if (Error E = OpCall.takeError()) {
1230 // Preserve the DXIL op error text but attach it as a
1231 // DiagnosticInfoUnsupported so we don't crash with a dangling call.
1232 std::string Message(toString(E: std::move(E)));
1233 CI->getContext().diagnose(DI: DiagnosticInfoUnsupported(
1234 *CI->getFunction(), Message, CI->getDebugLoc()));
1235 CI->replaceAllUsesWith(V: PoisonValue::get(T: CI->getType()));
1236 CI->eraseFromParent();
1237 return Error::success();
1238 }
1239
1240 CI->replaceAllUsesWith(V: *OpCall);
1241 CI->eraseFromParent();
1242 return Error::success();
1243 });
1244 }
1245
1246 [[nodiscard]] bool lowerCtpopToCountBits(Function &F) {
1247 IRBuilder<> &IRB = OpBuilder.getIRB();
1248 Type *Int32Ty = IRB.getInt32Ty();
1249
1250 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1251 IRB.SetInsertPoint(CI);
1252 SmallVector<Value *> Args;
1253 Args.append(in_start: CI->arg_begin(), in_end: CI->arg_end());
1254
1255 Type *RetTy = Int32Ty;
1256 Type *FRT = F.getReturnType();
1257 if (const auto *VT = dyn_cast<VectorType>(Val: FRT))
1258 RetTy = VectorType::get(ElementType: RetTy, Other: VT);
1259
1260 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
1261 Op: dxil::OpCode::CountBits, Args, Name: CI->getName(), RetTy);
1262 if (Error E = OpCall.takeError())
1263 return E;
1264
1265 // If the result type is 32 bits we can do a direct replacement.
1266 if (FRT->isIntOrIntVectorTy(BitWidth: 32)) {
1267 CI->replaceAllUsesWith(V: *OpCall);
1268 CI->eraseFromParent();
1269 return Error::success();
1270 }
1271
1272 unsigned CastOp;
1273 unsigned CastOp2;
1274 if (FRT->isIntOrIntVectorTy(BitWidth: 16)) {
1275 CastOp = Instruction::ZExt;
1276 CastOp2 = Instruction::SExt;
1277 } else { // must be 64 bits
1278 assert(FRT->isIntOrIntVectorTy(64) &&
1279 "Currently only lowering 16, 32, or 64 bit ctpop to CountBits \
1280 is supported.");
1281 CastOp = Instruction::Trunc;
1282 CastOp2 = Instruction::Trunc;
1283 }
1284
1285 // It is correct to replace the ctpop with the dxil op and
1286 // remove all casts to i32
1287 bool NeedsCast = false;
1288 for (User *User : make_early_inc_range(Range: CI->users())) {
1289 Instruction *I = dyn_cast<Instruction>(Val: User);
1290 if (I && (I->getOpcode() == CastOp || I->getOpcode() == CastOp2) &&
1291 I->getType() == RetTy) {
1292 I->replaceAllUsesWith(V: *OpCall);
1293 I->eraseFromParent();
1294 } else
1295 NeedsCast = true;
1296 }
1297
1298 // It is correct to replace a ctpop with the dxil op and
1299 // a cast from i32 to the return type of the ctpop
1300 // the cast is emitted here if there is a non-cast to i32
1301 // instr which uses the ctpop
1302 if (NeedsCast) {
1303 Value *Cast =
1304 IRB.CreateZExtOrTrunc(V: *OpCall, DestTy: F.getReturnType(), Name: "ctpop.cast");
1305 CI->replaceAllUsesWith(V: Cast);
1306 }
1307
1308 CI->eraseFromParent();
1309 return Error::success();
1310 });
1311 }
1312
1313 [[nodiscard]] bool lowerLifetimeIntrinsic(Function &F) {
1314 IRBuilder<> &IRB = OpBuilder.getIRB();
1315 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1316 IRB.SetInsertPoint(CI);
1317 Value *Ptr = CI->getArgOperand(i: 0);
1318 assert(Ptr->getType()->isPointerTy() &&
1319 "Expected operand of lifetime intrinsic to be a pointer");
1320
1321 auto ZeroOrUndef = [&](Type *Ty) {
1322 return MMDI.ValidatorVersion < VersionTuple(1, 6)
1323 ? Constant::getNullValue(Ty)
1324 : UndefValue::get(T: Ty);
1325 };
1326
1327 Value *Val = nullptr;
1328 if (auto *GV = dyn_cast<GlobalVariable>(Val: Ptr)) {
1329 if (GV->hasInitializer() || GV->isExternallyInitialized())
1330 return Error::success();
1331 Val = ZeroOrUndef(GV->getValueType());
1332 } else if (auto *AI = dyn_cast<AllocaInst>(Val: Ptr))
1333 Val = ZeroOrUndef(AI->getAllocatedType());
1334
1335 assert(Val && "Expected operand of lifetime intrinsic to be a global "
1336 "variable or alloca instruction");
1337 IRB.CreateStore(Val, Ptr, isVolatile: false);
1338
1339 CI->eraseFromParent();
1340 return Error::success();
1341 });
1342 }
1343
1344 [[nodiscard]] bool lowerIsFPClass(Function &F) {
1345 IRBuilder<> &IRB = OpBuilder.getIRB();
1346 Type *RetTy = IRB.getInt1Ty();
1347
1348 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1349 IRB.SetInsertPoint(CI);
1350 SmallVector<Value *> Args;
1351 Value *Fl = CI->getArgOperand(i: 0);
1352 Args.push_back(Elt: Fl);
1353
1354 dxil::OpCode OpCode;
1355 Value *T = CI->getArgOperand(i: 1);
1356 auto *TCI = dyn_cast<ConstantInt>(Val: T);
1357 switch (TCI->getZExtValue()) {
1358 case FPClassTest::fcInf:
1359 OpCode = dxil::OpCode::IsInf;
1360 break;
1361 case FPClassTest::fcNan:
1362 OpCode = dxil::OpCode::IsNaN;
1363 break;
1364 case FPClassTest::fcNormal:
1365 OpCode = dxil::OpCode::IsNormal;
1366 break;
1367 case FPClassTest::fcFinite:
1368 OpCode = dxil::OpCode::IsFinite;
1369 break;
1370 default:
1371 SmallString<128> Msg =
1372 formatv(Fmt: "Unsupported FPClassTest {0} for DXIL Op Lowering",
1373 Vals: TCI->getZExtValue());
1374 return make_error<StringError>(Args&: Msg, Args: inconvertibleErrorCode());
1375 }
1376
1377 Expected<CallInst *> OpCall =
1378 OpBuilder.tryCreateOp(Op: OpCode, Args, Name: CI->getName(), RetTy);
1379 if (Error E = OpCall.takeError())
1380 return E;
1381
1382 CI->replaceAllUsesWith(V: *OpCall);
1383 CI->eraseFromParent();
1384 return Error::success();
1385 });
1386 }
1387
1388 bool lowerIntrinsics() {
1389 bool Updated = false;
1390 bool HasErrors = false;
1391
1392 for (Function &F : make_early_inc_range(Range: M.functions())) {
1393 if (!F.isDeclaration())
1394 continue;
1395 Intrinsic::ID ID = F.getIntrinsicID();
1396 switch (ID) {
1397 // NOTE: Skip dx_resource_casthandle here. They are
1398 // resolved after this loop in cleanupHandleCasts.
1399 case Intrinsic::dx_resource_casthandle:
1400 // NOTE: llvm.dbg.value is supported as is in DXIL.
1401 case Intrinsic::dbg_value:
1402 case Intrinsic::not_intrinsic:
1403 if (F.use_empty())
1404 F.eraseFromParent();
1405 continue;
1406 default:
1407 if (F.use_empty())
1408 F.eraseFromParent();
1409 else {
1410 SmallString<128> Msg = formatv(
1411 Fmt: "Unsupported intrinsic {0} for DXIL lowering", Vals: F.getName());
1412 M.getContext().emitError(ErrorStr: Msg);
1413 HasErrors |= true;
1414 }
1415 break;
1416
1417#define DXIL_OP_INTRINSIC(OpCode, Intrin, ...) \
1418 case Intrin: \
1419 HasErrors |= replaceFunctionWithOp( \
1420 F, OpCode, ArrayRef<IntrinArgSelect>{__VA_ARGS__}); \
1421 break;
1422#include "DXILOperation.inc"
1423 case Intrinsic::dx_resource_handlefrombinding:
1424 HasErrors |= lowerHandleFromBinding(F);
1425 break;
1426 case Intrinsic::dx_resource_handlefromheap:
1427 HasErrors |= lowerHandleFromHeap(F);
1428 break;
1429 case Intrinsic::dx_resource_getbasepointer:
1430 case Intrinsic::dx_resource_getpointer:
1431 HasErrors |= lowerGetPointer(F);
1432 break;
1433 case Intrinsic::dx_resource_nonuniformindex:
1434 assert(!CleanupNURI &&
1435 "overloaded llvm.dx.resource.nonuniformindex intrinsics?");
1436 CleanupNURI = &F;
1437 break;
1438 case Intrinsic::dx_resource_load_typedbuffer:
1439 HasErrors |= lowerTypedBufferLoad(F, /*HasCheckBit=*/true);
1440 break;
1441 case Intrinsic::dx_resource_load_level:
1442 HasErrors |= lowerTextureLoad(F);
1443 break;
1444 case Intrinsic::dx_resource_sample:
1445 HasErrors |= lowerSample(F, /*HasClamp=*/false);
1446 break;
1447 case Intrinsic::dx_resource_sample_clamp:
1448 HasErrors |= lowerSample(F, /*HasClamp=*/true);
1449 break;
1450 case Intrinsic::dx_resource_samplebias:
1451 HasErrors |= lowerSampleBias(F, /*HasClamp=*/false);
1452 break;
1453 case Intrinsic::dx_resource_samplebias_clamp:
1454 HasErrors |= lowerSampleBias(F, /*HasClamp=*/true);
1455 break;
1456 case Intrinsic::dx_resource_samplelevel:
1457 HasErrors |= lowerSampleLevel(F);
1458 break;
1459 case Intrinsic::dx_resource_samplegrad:
1460 HasErrors |= lowerSampleGrad(F, /*HasClamp=*/false);
1461 break;
1462 case Intrinsic::dx_resource_samplegrad_clamp:
1463 HasErrors |= lowerSampleGrad(F, /*HasClamp=*/true);
1464 break;
1465 case Intrinsic::dx_resource_store_typedbuffer:
1466 HasErrors |= lowerBufferStore(F, /*IsRaw=*/false);
1467 break;
1468 case Intrinsic::dx_resource_store_texture:
1469 HasErrors |= lowerTextureStore(F);
1470 break;
1471 case Intrinsic::dx_resource_load_rawbuffer:
1472 HasErrors |= lowerRawBufferLoad(F);
1473 break;
1474 case Intrinsic::dx_resource_store_rawbuffer:
1475 HasErrors |= lowerBufferStore(F, /*IsRaw=*/true);
1476 break;
1477 case Intrinsic::dx_resource_load_cbufferrow_2:
1478 case Intrinsic::dx_resource_load_cbufferrow_4:
1479 case Intrinsic::dx_resource_load_cbufferrow_8:
1480 HasErrors |= lowerCBufferLoad(F);
1481 break;
1482 case Intrinsic::dx_resource_updatecounter:
1483 HasErrors |= lowerUpdateCounter(F);
1484 break;
1485 case Intrinsic::dx_barrier_by_memory_handle:
1486 HasErrors |= lowerBarrierByMemoryHandle(F);
1487 break;
1488 case Intrinsic::dx_resource_atomic_binop:
1489 HasErrors |= lowerResourceAtomicBinOp(F);
1490 break;
1491 case Intrinsic::dx_resource_atomic_compare_exchange:
1492 HasErrors |= lowerResourceAtomicCompareExchange(F);
1493 break;
1494 case Intrinsic::dx_resource_getdimensions_x:
1495 HasErrors |= lowerGetDimensionsX(F);
1496 break;
1497 case Intrinsic::ctpop:
1498 HasErrors |= lowerCtpopToCountBits(F);
1499 break;
1500 case Intrinsic::lifetime_start:
1501 case Intrinsic::lifetime_end:
1502 if (F.use_empty())
1503 F.eraseFromParent();
1504 else {
1505 if (MMDI.DXILVersion < VersionTuple(1, 6))
1506 HasErrors |= lowerLifetimeIntrinsic(F);
1507 else
1508 continue;
1509 }
1510 break;
1511 case Intrinsic::is_fpclass:
1512 HasErrors |= lowerIsFPClass(F);
1513 break;
1514 }
1515 Updated = true;
1516 }
1517 if (Updated && !HasErrors) {
1518 cleanupHandleCasts();
1519 cleanupNonUniformResourceIndexCalls();
1520 }
1521
1522 return Updated;
1523 }
1524};
1525} // namespace
1526
1527PreservedAnalyses DXILOpLoweringPass::run(Module &M,
1528 ModuleAnalysisManager &MAM) {
1529 DXILResourceMap &DRM = MAM.getResult<DXILResourceAnalysis>(IR&: M);
1530 DXILResourceTypeMap &DRTM = MAM.getResult<DXILResourceTypeAnalysis>(IR&: M);
1531 const ModuleMetadataInfo MMDI = MAM.getResult<DXILMetadataAnalysis>(IR&: M);
1532
1533 const bool MadeChanges = OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
1534 if (!MadeChanges)
1535 return PreservedAnalyses::all();
1536 PreservedAnalyses PA;
1537 PA.preserve<DXILResourceAnalysis>();
1538 PA.preserve<DXILMetadataAnalysis>();
1539 PA.preserve<ShaderFlagsAnalysis>();
1540 PA.preserve<RootSignatureAnalysis>();
1541 PA.preserve<SignatureAnalysis>();
1542 return PA;
1543}
1544
1545namespace {
1546class DXILOpLoweringLegacy : public ModulePass {
1547public:
1548 bool runOnModule(Module &M) override {
1549 DXILResourceMap &DRM =
1550 getAnalysis<DXILResourceWrapperPass>().getResourceMap();
1551 DXILResourceTypeMap &DRTM =
1552 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
1553 const ModuleMetadataInfo MMDI =
1554 getAnalysis<DXILMetadataAnalysisWrapperPass>().getModuleMetadata();
1555
1556 return OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
1557 }
1558 StringRef getPassName() const override { return "DXIL Op Lowering"; }
1559 DXILOpLoweringLegacy() : ModulePass(ID) {}
1560
1561 static char ID; // Pass identification.
1562 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
1563 AU.addRequired<DXILResourceTypeWrapperPass>();
1564 AU.addRequired<DXILResourceWrapperPass>();
1565 AU.addRequired<DXILMetadataAnalysisWrapperPass>();
1566 AU.addPreserved<DXILResourceWrapperPass>();
1567 AU.addPreserved<DXILMetadataAnalysisWrapperPass>();
1568 AU.addPreserved<ShaderFlagsAnalysisWrapper>();
1569 AU.addPreserved<RootSignatureAnalysisWrapper>();
1570 AU.addPreserved<SignatureAnalysisWrapper>();
1571 }
1572};
1573char DXILOpLoweringLegacy::ID = 0;
1574} // end anonymous namespace
1575
1576INITIALIZE_PASS_BEGIN(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering",
1577 false, false)
1578INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
1579INITIALIZE_PASS_DEPENDENCY(DXILResourceWrapperPass)
1580INITIALIZE_PASS_END(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering", false,
1581 false)
1582
1583ModulePass *llvm::createDXILOpLoweringLegacyPass() {
1584 return new DXILOpLoweringLegacy();
1585}
1586