1//===- DXILOpLowering.cpp - Lowering to DXIL operations -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DXILOpLowering.h"
10#include "DXILConstants.h"
11#include "DXILOpBuilder.h"
12#include "DXILRootSignature.h"
13#include "DXILShaderFlags.h"
14#include "DXILSignatureAnalysis.h"
15#include "DirectX.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/DXILMetadataAnalysis.h"
18#include "llvm/Analysis/DXILResource.h"
19#include "llvm/CodeGen/Passes.h"
20#include "llvm/IR/Constant.h"
21#include "llvm/IR/DiagnosticInfo.h"
22#include "llvm/IR/IRBuilder.h"
23#include "llvm/IR/Instruction.h"
24#include "llvm/IR/Instructions.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/IntrinsicsDirectX.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/PassManager.h"
29#include "llvm/IR/Use.h"
30#include "llvm/IR/ValueHandle.h"
31#include "llvm/InitializePasses.h"
32#include "llvm/Pass.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Support/FormatVariadic.h"
35
36#define DEBUG_TYPE "dxil-op-lower"
37
38using namespace llvm;
39using namespace llvm::dxil;
40
41/// Write mask covering all four components of a UAV element. Typed UAV stores
42/// (textures and typed buffers) must always use this mask - the DXIL validator
43/// rejects anything narrower. Only raw and / structured buffer stores may use a
44/// partial mask.
45static constexpr uint8_t TypedUAVStoreWriteMask = 0xF;
46
47namespace {
48class OpLowerer {
49 Module &M;
50 DXILOpBuilder OpBuilder;
51 DXILResourceMap &DRM;
52 DXILResourceTypeMap &DRTM;
53 const ModuleMetadataInfo &MMDI;
54 SmallVector<CallInst *> CleanupCasts;
55 Function *CleanupNURI = nullptr;
56
57public:
58 OpLowerer(Module &M, DXILResourceMap &DRM, DXILResourceTypeMap &DRTM,
59 const ModuleMetadataInfo &MMDI)
60 : M(M), OpBuilder(M), DRM(DRM), DRTM(DRTM), MMDI(MMDI) {}
61
62 /// Replace every call to \c F using \c ReplaceCall, and then erase \c F. If
63 /// there is an error replacing a call, we emit a diagnostic and return true.
64 [[nodiscard]] bool
65 replaceFunction(Function &F,
66 llvm::function_ref<Error(CallInst *CI)> ReplaceCall) {
67 for (User *U : make_early_inc_range(Range: F.users())) {
68 CallInst *CI = dyn_cast<CallInst>(Val: U);
69 if (!CI)
70 continue;
71
72 if (Error E = ReplaceCall(CI)) {
73 std::string Message(toString(E: std::move(E)));
74 M.getContext().diagnose(DI: DiagnosticInfoUnsupported(
75 *CI->getFunction(), Message, CI->getDebugLoc()));
76
77 return true;
78 }
79 }
80 if (F.user_empty())
81 F.eraseFromParent();
82 return false;
83 }
84
85 struct IntrinArgSelect {
86 enum class Type {
87#define DXIL_OP_INTRINSIC_ARG_SELECT_TYPE(name) name,
88#include "DXILOperation.inc"
89 };
90 Type Type;
91 int Value;
92 };
93
94 /// Replaces uses of a struct with uses of an equivalent named struct.
95 ///
96 /// DXIL operations that return structs give them well known names, so we need
97 /// to update uses when we switch from an LLVM intrinsic to an op.
98 Error replaceNamedStructUses(CallInst *Intrin, CallInst *DXILOp) {
99 auto *IntrinTy = cast<StructType>(Val: Intrin->getType());
100 auto *DXILOpTy = cast<StructType>(Val: DXILOp->getType());
101 if (!IntrinTy->isLayoutIdentical(Other: DXILOpTy))
102 return make_error<StringError>(
103 Args: "Type mismatch between intrinsic and DXIL op",
104 Args: inconvertibleErrorCode());
105
106 for (Use &U : make_early_inc_range(Range: Intrin->uses()))
107 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: U.getUser()))
108 EVI->setOperand(i_nocapture: 0, Val_nocapture: DXILOp);
109 else if (auto *IVI = dyn_cast<InsertValueInst>(Val: U.getUser()))
110 IVI->setOperand(i_nocapture: 0, Val_nocapture: DXILOp);
111 else
112 return make_error<StringError>(Args: "DXIL ops that return structs may only "
113 "be used by insert- and extractvalue",
114 Args: inconvertibleErrorCode());
115 return Error::success();
116 }
117
118 bool isFast(FastMathFlags Flags) {
119 // HLSL Fast Math doesn't enable AllowContract flag; This can be
120 // removed when we enable it in the future.
121 return Flags.allowReassoc() && Flags.noNaNs() && Flags.noInfs() &&
122 Flags.noSignedZeros() && Flags.allowReciprocal() &&
123 Flags.approxFunc();
124 }
125
126 void setDxPrecise(CallInst *CI) {
127 const StringRef Key = "dx.precise";
128 Module *M = CI->getModule();
129
130 LLVMContext &Ctx = M->getContext();
131 MDNode *One =
132 llvm::MDNode::get(Context&: Ctx, MDs: ConstantAsMetadata::get(C: ConstantInt::get(
133 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1)));
134
135 CI->setMetadata(Kind: Key, Node: One);
136 }
137
138 [[nodiscard]] bool
139 replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp,
140 ArrayRef<IntrinArgSelect> ArgSelects) {
141 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
142 OpBuilder.getIRB().SetInsertPoint(CI);
143 SmallVector<Value *> Args;
144 if (ArgSelects.size()) {
145 for (const IntrinArgSelect &A : ArgSelects) {
146 switch (A.Type) {
147 case IntrinArgSelect::Type::Index:
148 Args.push_back(Elt: CI->getArgOperand(i: A.Value));
149 break;
150 case IntrinArgSelect::Type::I8:
151 Args.push_back(Elt: OpBuilder.getIRB().getInt8(C: (uint8_t)A.Value));
152 break;
153 case IntrinArgSelect::Type::I32:
154 Args.push_back(Elt: OpBuilder.getIRB().getInt32(C: A.Value));
155 break;
156 }
157 }
158 } else {
159 Args.append(in_start: CI->arg_begin(), in_end: CI->arg_end());
160 }
161
162 Expected<CallInst *> OpCall =
163 OpBuilder.tryCreateOp(Op: DXILOp, Args, Name: CI->getName(), RetTy: F.getReturnType());
164 if (Error E = OpCall.takeError())
165 return E;
166
167 if (isa<FPMathOperator>(Val: CI) &&
168 !isFast(Flags: cast<FPMathOperator>(Val: CI)->getFastMathFlags()))
169 setDxPrecise(*OpCall);
170
171 if (isa<StructType>(Val: CI->getType())) {
172 if (Error E = replaceNamedStructUses(Intrin: CI, DXILOp: *OpCall))
173 return E;
174 } else
175 CI->replaceAllUsesWith(V: *OpCall);
176
177 CI->eraseFromParent();
178 return Error::success();
179 });
180 }
181
182 /// Create a cast between a `target("dx")` type and `dx.types.Handle`, which
183 /// is intended to be removed by the end of lowering. This is used to allow
184 /// lowering of ops which need to change their return or argument types in a
185 /// piecemeal way - we can add the casts in to avoid updating all of the uses
186 /// or defs, and by the end all of the casts will be redundant.
187 Value *createTmpHandleCast(Value *V, Type *Ty) {
188 CallInst *Cast = OpBuilder.getIRB().CreateIntrinsicWithoutFolding(
189 ID: Intrinsic::dx_resource_casthandle, OverloadTypes: {Ty, V->getType()}, Args: {V});
190 CleanupCasts.push_back(Elt: Cast);
191 return Cast;
192 }
193
194 void cleanupHandleCasts() {
195 SmallVector<CallInst *> ToRemove;
196 SmallVector<Function *> CastFns;
197
198 for (CallInst *Cast : CleanupCasts) {
199 // These casts were only put in to ease the move from `target("dx")` types
200 // to `dx.types.Handle in a piecemeal way. At this point, all of the
201 // non-cast uses should now be `dx.types.Handle`, and remaining casts
202 // should all form pairs to and from the now unused `target("dx")` type.
203 CastFns.push_back(Elt: Cast->getCalledFunction());
204
205 // If the cast is not to `dx.types.Handle`, it should be the first part of
206 // the pair. Keep track so we can remove it once it has no more uses.
207 if (Cast->getType() != OpBuilder.getHandleType()) {
208 ToRemove.push_back(Elt: Cast);
209 continue;
210 }
211 // Otherwise, we're the second handle in a pair. Forward the arguments and
212 // remove the (second) cast.
213 CallInst *Def = cast<CallInst>(Val: Cast->getOperand(i_nocapture: 0));
214 assert(Def->getIntrinsicID() == Intrinsic::dx_resource_casthandle &&
215 "Unbalanced pair of temporary handle casts");
216 Cast->replaceAllUsesWith(V: Def->getOperand(i_nocapture: 0));
217 Cast->eraseFromParent();
218 }
219 for (CallInst *Cast : ToRemove) {
220 assert(Cast->user_empty() && "Temporary handle cast still has users");
221 Cast->eraseFromParent();
222 }
223
224 // Deduplicate the cast functions so that we only erase each one once.
225 llvm::sort(C&: CastFns);
226 CastFns.erase(CS: llvm::unique(R&: CastFns), CE: CastFns.end());
227 for (Function *F : CastFns)
228 F->eraseFromParent();
229
230 CleanupCasts.clear();
231 }
232
233 void cleanupNonUniformResourceIndexCalls() {
234 // Replace all NonUniformResourceIndex calls with their argument.
235 if (!CleanupNURI)
236 return;
237 for (User *U : make_early_inc_range(Range: CleanupNURI->users())) {
238 CallInst *CI = dyn_cast<CallInst>(Val: U);
239 if (!CI)
240 continue;
241 CI->replaceAllUsesWith(V: CI->getArgOperand(i: 0));
242 CI->eraseFromParent();
243 }
244 CleanupNURI->eraseFromParent();
245 CleanupNURI = nullptr;
246 }
247
248 // Remove the resource global associated with the handleFromBinding call
249 // instruction and their uses as they aren't needed anymore.
250 // TODO: We should verify that all the globals get removed.
251 // It's expected we'll need a custom pass in the future that will eliminate
252 // the need for this here.
253 void removeResourceGlobals(CallInst *CI) {
254 for (User *User : make_early_inc_range(Range: CI->users())) {
255 if (StoreInst *Store = dyn_cast<StoreInst>(Val: User)) {
256 Value *V = Store->getOperand(i_nocapture: 1);
257 Store->eraseFromParent();
258 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: V))
259 if (GV->use_empty()) {
260 GV->removeDeadConstantUsers();
261 GV->eraseFromParent();
262 }
263 }
264 }
265 }
266
267 void replaceHandleFromBindingCall(CallInst *CI, Value *Replacement) {
268 assert(CI->getCalledFunction()->getIntrinsicID() ==
269 Intrinsic::dx_resource_handlefrombinding);
270
271 removeResourceGlobals(CI);
272
273 auto *NameGlobal = dyn_cast<llvm::GlobalVariable>(Val: CI->getArgOperand(i: 4));
274
275 CI->replaceAllUsesWith(V: Replacement);
276 CI->eraseFromParent();
277
278 if (NameGlobal && NameGlobal->use_empty())
279 NameGlobal->eraseFromParent();
280 }
281
282 bool hasNonUniformIndex(Value *IndexOp) {
283 if (isa<llvm::Constant>(Val: IndexOp))
284 return false;
285
286 SmallVector<Value *, 16> Worklist;
287 SmallPtrSet<Value *, 16> Visited;
288 Worklist.push_back(Elt: IndexOp);
289
290 while (!Worklist.empty()) {
291 Value *V = Worklist.pop_back_val();
292
293 if (isa<llvm::Constant>(Val: V))
294 continue;
295
296 if (!Visited.insert(Ptr: V).second)
297 continue;
298
299 if (auto *CI = dyn_cast<CallInst>(Val: V))
300 if (CI->getIntrinsicID() == Intrinsic::dx_resource_nonuniformindex)
301 return true;
302
303 // If it's a PHI node, check ALL incoming values —
304 // taint from ANY predecessor counts
305 if (auto *Phi = dyn_cast<PHINode>(Val: V)) {
306 for (Value *Incoming : Phi->incoming_values())
307 Worklist.push_back(Elt: Incoming);
308 continue;
309 }
310
311 if (auto *Inst = dyn_cast<Instruction>(Val: V))
312 if (Inst->getNumOperands() > 0 && !Inst->isTerminator())
313 for (Value *Op : Inst->operands())
314 Worklist.push_back(Elt: Op);
315 }
316 return false;
317 }
318
319 Error validateRawBufferElementIndex(Value *Resource, Value *ElementIndex) {
320 bool IsStructured =
321 cast<RawBufferExtType>(Val: Resource->getType())->isStructured();
322 bool IsPoison = isa<PoisonValue>(Val: ElementIndex);
323
324 if (IsStructured && IsPoison)
325 return make_error<StringError>(
326 Args: "Element index of structured buffer may not be poison",
327 Args: inconvertibleErrorCode());
328
329 if (!IsStructured && !IsPoison)
330 return make_error<StringError>(
331 Args: "Element index of raw buffer must be poison",
332 Args: inconvertibleErrorCode());
333
334 return Error::success();
335 }
336
337 [[nodiscard]] bool lowerToCreateHandle(Function &F) {
338 IRBuilder<> &IRB = OpBuilder.getIRB();
339 Type *Int8Ty = IRB.getInt8Ty();
340 Type *Int32Ty = IRB.getInt32Ty();
341 Type *Int1Ty = IRB.getInt1Ty();
342
343 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
344 IRB.SetInsertPoint(CI);
345
346 auto *It = DRM.find(Key: CI);
347 assert(It != DRM.end() && "Resource not in map?");
348 dxil::ResourceInfo &RI = *It;
349
350 const auto &Binding = RI.getBinding();
351 dxil::ResourceClass RC = DRTM[RI.getHandleTy()].getResourceClass();
352
353 Value *IndexOp = CI->getArgOperand(i: 3);
354 if (Binding.LowerBound != 0)
355 IndexOp = IRB.CreateAdd(LHS: IndexOp,
356 RHS: ConstantInt::get(Ty: Int32Ty, V: Binding.LowerBound));
357
358 bool HasNonUniformIndex =
359 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);
360 std::array<Value *, 4> Args{
361 ConstantInt::get(Ty: Int8Ty, V: llvm::to_underlying(E: RC)),
362 ConstantInt::get(Ty: Int32Ty, V: Binding.BindingID), IndexOp,
363 ConstantInt::get(Ty: Int1Ty, V: HasNonUniformIndex)};
364 Expected<CallInst *> OpCall =
365 OpBuilder.tryCreateOp(Op: OpCode::CreateHandle, Args, Name: CI->getName());
366 if (Error E = OpCall.takeError())
367 return E;
368
369 Value *Cast = createTmpHandleCast(V: *OpCall, Ty: CI->getType());
370 replaceHandleFromBindingCall(CI, Replacement: Cast);
371 return Error::success();
372 });
373 }
374
375 [[nodiscard]] bool lowerToBindAndAnnotateHandle(Function &F) {
376 IRBuilder<> &IRB = OpBuilder.getIRB();
377 Type *Int32Ty = IRB.getInt32Ty();
378 Type *Int1Ty = IRB.getInt1Ty();
379
380 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
381 IRB.SetInsertPoint(CI);
382
383 auto *It = DRM.find(Key: CI);
384 assert(It != DRM.end() && "Resource not in map?");
385 dxil::ResourceInfo &RI = *It;
386
387 const auto &Binding = RI.getBinding();
388 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];
389 dxil::ResourceClass RC = RTI.getResourceClass();
390
391 Value *IndexOp = CI->getArgOperand(i: 3);
392 if (Binding.LowerBound != 0)
393 IndexOp = IRB.CreateAdd(LHS: IndexOp,
394 RHS: ConstantInt::get(Ty: Int32Ty, V: Binding.LowerBound));
395
396 std::pair<uint32_t, uint32_t> Props =
397 RI.getAnnotateProps(M&: *F.getParent(), RTI);
398
399 // For `CreateHandleFromBinding` we need the upper bound rather than the
400 // size, so we need to be careful about the difference for "unbounded".
401 uint32_t UpperBound = Binding.Size == 0
402 ? std::numeric_limits<uint32_t>::max()
403 : Binding.LowerBound + Binding.Size - 1;
404 Constant *ResBind = OpBuilder.getResBind(LowerBound: Binding.LowerBound, UpperBound,
405 SpaceID: Binding.Space, RC);
406 bool NonUniformIndex =
407 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);
408 Constant *NonUniformOp = ConstantInt::get(Ty: Int1Ty, V: NonUniformIndex);
409 std::array<Value *, 3> BindArgs{ResBind, IndexOp, NonUniformOp};
410 Expected<CallInst *> OpBind = OpBuilder.tryCreateOp(
411 Op: OpCode::CreateHandleFromBinding, Args: BindArgs, Name: CI->getName());
412 if (Error E = OpBind.takeError())
413 return E;
414
415 std::array<Value *, 2> AnnotateArgs{
416 *OpBind, OpBuilder.getResProps(Word0: Props.first, Word1: Props.second)};
417 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(
418 Op: OpCode::AnnotateHandle, Args: AnnotateArgs,
419 Name: CI->hasName() ? CI->getName() + "_annot" : Twine());
420 if (Error E = OpAnnotate.takeError())
421 return E;
422
423 Value *Cast = createTmpHandleCast(V: *OpAnnotate, Ty: CI->getType());
424 replaceHandleFromBindingCall(CI, Replacement: Cast);
425 return Error::success();
426 });
427 }
428
429 /// Lower `dx.resource.handlefrombinding` intrinsics depending on the shader
430 /// model and taking into account binding information from
431 /// DXILResourceAnalysis.
432 bool lowerHandleFromBinding(Function &F) {
433 if (MMDI.DXILVersion < VersionTuple(1, 6))
434 return lowerToCreateHandle(F);
435 return lowerToBindAndAnnotateHandle(F);
436 }
437
438 bool lowerHandleFromHeap(Function &F) {
439 IRBuilder<> &IRB = OpBuilder.getIRB();
440
441 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
442 IRB.SetInsertPoint(CI);
443
444 auto *It = DRM.find(Key: CI);
445 assert(It != DRM.end() && "Resource not in map?");
446 dxil::ResourceInfo &RI = *It;
447 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];
448
449 Value *IndexOp = CI->getArgOperand(i: 0);
450 Value *IsSamplerHeap =
451 ConstantInt::getBool(Context&: IRB.getContext(), V: RTI.isSampler());
452
453 std::pair<uint32_t, uint32_t> Props =
454 RI.getAnnotateProps(M&: *F.getParent(), RTI);
455
456 bool NonUniformIndex = hasNonUniformIndex(IndexOp);
457 Value *NonUniformOp =
458 ConstantInt::getBool(Context&: IRB.getContext(), V: NonUniformIndex);
459
460 std::array<Value *, 3> Args{IndexOp, IsSamplerHeap, NonUniformOp};
461 Expected<CallInst *> OpCreateHandle = OpBuilder.tryCreateOp(
462 Op: OpCode::CreateHandleFromHeap, Args, Name: CI->getName());
463 if (Error E = OpCreateHandle.takeError())
464 return E;
465
466 std::array<Value *, 2> AnnotateArgs{
467 *OpCreateHandle, OpBuilder.getResProps(Word0: Props.first, Word1: Props.second)};
468 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(
469 Op: OpCode::AnnotateHandle, Args: AnnotateArgs,
470 Name: CI->hasName() ? CI->getName() + "_annot" : Twine());
471 if (Error E = OpAnnotate.takeError())
472 return E;
473
474 Value *Cast = createTmpHandleCast(V: *OpAnnotate, Ty: CI->getType());
475 CI->replaceAllUsesWith(V: Cast);
476 CI->eraseFromParent();
477 return Error::success();
478 });
479 }
480
481 /// Replace uses of \c Intrin with the values in the `dx.ResRet` of \c Op.
482 /// Since we expect to be post-scalarization, make an effort to avoid vectors.
483 Error replaceResRetUses(CallInst *Intrin, CallInst *Op, bool HasCheckBit) {
484 IRBuilder<> &IRB = OpBuilder.getIRB();
485
486 Instruction *OldResult = Intrin;
487 Type *OldTy = Intrin->getType();
488
489 if (HasCheckBit) {
490 auto *ST = cast<StructType>(Val: OldTy);
491
492 Value *CheckOp = nullptr;
493 Type *Int32Ty = IRB.getInt32Ty();
494 for (Use &U : make_early_inc_range(Range: OldResult->uses())) {
495 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: U.getUser())) {
496 ArrayRef<unsigned> Indices = EVI->getIndices();
497 assert(Indices.size() == 1);
498 // We're only interested in uses of the check bit for now.
499 if (Indices[0] != 1)
500 continue;
501 if (!CheckOp) {
502 Value *NewEVI = IRB.CreateExtractValue(Agg: Op, Idxs: 4);
503 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
504 Op: OpCode::CheckAccessFullyMapped, Args: {NewEVI},
505 Name: OldResult->hasName() ? OldResult->getName() + "_check"
506 : Twine(),
507 RetTy: Int32Ty);
508 if (Error E = OpCall.takeError())
509 return E;
510 CheckOp = *OpCall;
511 }
512 EVI->replaceAllUsesWith(V: CheckOp);
513 EVI->eraseFromParent();
514 }
515 }
516
517 if (OldResult->use_empty()) {
518 // Only the check bit was used, so we're done here.
519 OldResult->eraseFromParent();
520 return Error::success();
521 }
522
523 assert(OldResult->hasOneUse() &&
524 isa<ExtractValueInst>(*OldResult->user_begin()) &&
525 "Expected only use to be extract of first element");
526 OldResult = cast<Instruction>(Val: *OldResult->user_begin());
527 OldTy = ST->getElementType(N: 0);
528 }
529
530 // For scalars, we just extract the first element.
531 if (!isa<FixedVectorType>(Val: OldTy)) {
532 Value *EVI = IRB.CreateExtractValue(Agg: Op, Idxs: 0);
533 OldResult->replaceAllUsesWith(V: EVI);
534 OldResult->eraseFromParent();
535 if (OldResult != Intrin) {
536 assert(Intrin->use_empty() && "Intrinsic still has uses?");
537 Intrin->eraseFromParent();
538 }
539 return Error::success();
540 }
541
542 std::array<Value *, 4> Extracts = {};
543 SmallVector<ExtractElementInst *> DynamicAccesses;
544
545 // The users of the operation should all be scalarized, so we attempt to
546 // replace the extractelements with extractvalues directly.
547 for (Use &U : make_early_inc_range(Range: OldResult->uses())) {
548 if (auto *EEI = dyn_cast<ExtractElementInst>(Val: U.getUser())) {
549 if (auto *IndexOp = dyn_cast<ConstantInt>(Val: EEI->getIndexOperand())) {
550 size_t IndexVal = IndexOp->getZExtValue();
551 assert(IndexVal < 4 && "Index into buffer load out of range");
552 if (!Extracts[IndexVal])
553 Extracts[IndexVal] = IRB.CreateExtractValue(Agg: Op, Idxs: IndexVal);
554 EEI->replaceAllUsesWith(V: Extracts[IndexVal]);
555 EEI->eraseFromParent();
556 } else {
557 DynamicAccesses.push_back(Elt: EEI);
558 }
559 }
560 }
561
562 const auto *VecTy = cast<FixedVectorType>(Val: OldTy);
563 const unsigned N = VecTy->getNumElements();
564
565 // If there's a dynamic access we need to round trip through stack memory so
566 // that we don't leave vectors around.
567 if (!DynamicAccesses.empty()) {
568 Type *Int32Ty = IRB.getInt32Ty();
569 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
570
571 Type *ElTy = VecTy->getElementType();
572 Type *ArrayTy = ArrayType::get(ElementType: ElTy, NumElements: N);
573 Value *Alloca = IRB.CreateAlloca(Ty: ArrayTy);
574
575 for (int I = 0, E = N; I != E; ++I) {
576 if (!Extracts[I])
577 Extracts[I] = IRB.CreateExtractValue(Agg: Op, Idxs: I);
578 Value *GEP = GetElementPtrInst::CreateInBounds(
579 PointeeType: ArrayTy, Ptr: Alloca, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: I)}, NameStr: "",
580 InsertBefore: IRB.GetInsertPoint());
581 IRB.CreateStore(Val: Extracts[I], Ptr: GEP);
582 }
583
584 for (ExtractElementInst *EEI : DynamicAccesses) {
585 Value *GEP = GetElementPtrInst::CreateInBounds(
586 PointeeType: ArrayTy, Ptr: Alloca, IdxList: {Zero, EEI->getIndexOperand()}, NameStr: "",
587 InsertBefore: IRB.GetInsertPoint());
588 Value *Load = IRB.CreateLoad(Ty: ElTy, Ptr: GEP);
589 EEI->replaceAllUsesWith(V: Load);
590 EEI->eraseFromParent();
591 }
592 }
593
594 // If we still have uses, then we're not fully scalarized and need to
595 // recreate the vector. This should only happen for things like exported
596 // functions from libraries.
597 if (!OldResult->use_empty()) {
598 for (int I = 0, E = N; I != E; ++I)
599 if (!Extracts[I])
600 Extracts[I] = IRB.CreateExtractValue(Agg: Op, Idxs: I);
601
602 Value *Vec = PoisonValue::get(T: OldTy);
603 for (int I = 0, E = N; I != E; ++I)
604 Vec = IRB.CreateInsertElement(Vec, NewElt: Extracts[I], Idx: I);
605 OldResult->replaceAllUsesWith(V: Vec);
606 }
607
608 OldResult->eraseFromParent();
609 if (OldResult != Intrin) {
610 assert(Intrin->use_empty() && "Intrinsic still has uses?");
611 Intrin->eraseFromParent();
612 }
613
614 return Error::success();
615 }
616
617 [[nodiscard]] bool lowerTypedBufferLoad(Function &F, bool HasCheckBit) {
618 IRBuilder<> &IRB = OpBuilder.getIRB();
619 Type *Int32Ty = IRB.getInt32Ty();
620
621 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
622 IRB.SetInsertPoint(CI);
623
624 Value *Handle =
625 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
626 Value *Index0 = CI->getArgOperand(i: 1);
627 Value *Index1 = UndefValue::get(T: Int32Ty);
628
629 Type *OldTy = CI->getType();
630 if (HasCheckBit)
631 OldTy = cast<StructType>(Val: OldTy)->getElementType(N: 0);
632 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
633
634 std::array<Value *, 3> Args{Handle, Index0, Index1};
635 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
636 Op: OpCode::BufferLoad, Args, Name: CI->getName(), RetTy: NewRetTy);
637 if (Error E = OpCall.takeError())
638 return E;
639 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, HasCheckBit))
640 return E;
641
642 return Error::success();
643 });
644 }
645
646 /// Recover the scalar components of `Vec` from the `insertelement` chain that
647 /// built it. Since we run after the scalarizer, such a chain is usually just
648 /// a temporary gathered to pass the vector to a call.
649 static void collectInsertedElements(Value *Vec,
650 MutableArrayRef<Value *> Elements) {
651 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
652 assert(NumElts <= Elements.size() && "Not enough room for the components");
653
654 SmallVector<InsertElementInst *, 4> Chain;
655 for (auto *IEI = dyn_cast<InsertElementInst>(Val: Vec); IEI;
656 IEI = dyn_cast<InsertElementInst>(Val: IEI->getOperand(i_nocapture: 0))) {
657 if (!isa<ConstantInt>(Val: IEI->getOperand(i_nocapture: 2)))
658 break; // This break should never happen below SM6.9.
659 Chain.push_back(Elt: IEI);
660 }
661
662 // Replay element insertion from the innermost first, so that a repeated
663 // index ends up holding the live value.
664 while (!Chain.empty()) {
665 InsertElementInst *IEI = Chain.pop_back_val();
666 uint64_t IndexVal = cast<ConstantInt>(Val: IEI->getOperand(i_nocapture: 2))->getZExtValue();
667 if (IndexVal < NumElts)
668 Elements[IndexVal] = IEI->getOperand(i_nocapture: 1);
669 }
670 }
671
672 // Copies `Src` into `Args` starting at `ArgIdx`. If `Src` is a vector, its
673 // elements are placed in consecutive slots; otherwise `Src` is stored
674 // directly. At most `MaxElements` elements are expected.
675 static void extractElementsIntoArgs(IRBuilder<> &IRB,
676 MutableArrayRef<Value *> Args,
677 unsigned ArgIdx, Value *Src,
678 unsigned MaxElements) {
679 auto *VecTy = dyn_cast<FixedVectorType>(Val: Src->getType());
680 if (!VecTy) {
681 Args[ArgIdx] = Src;
682 return;
683 }
684
685 unsigned Count = VecTy->getNumElements();
686 assert(Count <= MaxElements && "Too many elements for the arg list");
687
688 SmallVector<Value *, 4> Elements(Count, nullptr);
689 collectInsertedElements(Vec: Src, Elements);
690
691 for (unsigned I = 0; I < Count; ++I)
692 Args[ArgIdx + I] = Elements[I]
693 ? Elements[I]
694 : IRB.CreateExtractElement(
695 Vec: Src, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: I));
696 }
697
698 /// Copy offsets into the argument list at the given index, unless
699 /// the offsets are known to be zero (i.e., a null constant).
700 static void extractNonZeroOffsets(IRBuilder<> &IRB,
701 MutableArrayRef<Value *> Args,
702 unsigned ArgIdx, Value *Offsets,
703 unsigned MaxElements) {
704 auto *COff = dyn_cast<Constant>(Val: Offsets);
705 bool OffsetsAreZero = COff && COff->isNullValue();
706 if (!OffsetsAreZero)
707 extractElementsIntoArgs(IRB, Args, ArgIdx, Src: Offsets, MaxElements);
708 }
709
710 [[nodiscard]] bool lowerTextureLoad(Function &F) {
711 IRBuilder<> &IRB = OpBuilder.getIRB();
712 Type *Int32Ty = IRB.getInt32Ty();
713
714 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
715 IRB.SetInsertPoint(CI);
716
717 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
718 Value *Handle =
719 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
720 Value *Coords = CI->getArgOperand(i: 1);
721 Value *MipLevel = CI->getArgOperand(i: 2);
722 Value *Offsets = CI->getArgOperand(i: 3);
723
724 // A UAV descriptor binds a single mip slice, so there is no mip to select
725 // in the case of a UAV. Multisampled UAVs are the exception: the slot
726 // carries a sample index and stays live.
727 auto *HandleTy = cast<TargetExtType>(Val: CI->getArgOperand(i: 0)->getType());
728 dxil::ResourceTypeInfo &RTI = DRTM[HandleTy];
729 dxil::ResourceKind Kind = RTI.getResourceKind();
730 if (RTI.isUAV() && Kind != dxil::ResourceKind::Texture2DMS &&
731 Kind != dxil::ResourceKind::Texture2DMSArray)
732 MipLevel = UndefValue::get(T: Int32Ty);
733
734 Type *OldTy = CI->getType();
735 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
736
737 Value *Undef = UndefValue::get(T: Int32Ty);
738 std::array<Value *, 8> Args{Handle, MipLevel, Undef, Undef,
739 Undef, Undef, Undef, Undef};
740
741 // Copy coordinates and offsets into Args.
742 extractElementsIntoArgs(IRB, Args, ArgIdx: 2, Src: Coords, MaxElements: 3);
743 extractNonZeroOffsets(IRB, Args, ArgIdx: 5, Offsets, MaxElements: 3);
744
745 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
746 Op: OpCode::TextureLoad, Args, Name: CI->getName(), RetTy: NewRetTy);
747 if (Error E = OpCall.takeError())
748 return E;
749 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/false))
750 return E;
751
752 eraseDeadInsertElementChains(Vectors: VectorArgs);
753
754 return Error::success();
755 });
756 }
757
758 /// Common helper for lowering sample operations (SampleBias, SampleGrad,
759 /// etc.) that share the same pattern: extract handle/sampler, unpack
760 /// coordinates and offsets, build the DXIL arg list, and replace uses.
761 [[nodiscard]] bool lowerSampleOp(
762 Function &F, OpCode Op, unsigned CoordsIdx, unsigned OffsetsIdx,
763 llvm::function_ref<void(IRBuilder<> &, CallInst *,
764 SmallVectorImpl<Value *> &)> EmitExtraArgs) {
765 IRBuilder<> &IRB = OpBuilder.getIRB();
766 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
767 IRB.SetInsertPoint(CI);
768
769 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
770 Value *Handle =
771 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
772 Value *Sampler =
773 createTmpHandleCast(V: CI->getArgOperand(i: 1), Ty: OpBuilder.getHandleType());
774 Value *Coords = CI->getArgOperand(i: CoordsIdx);
775 Value *Offsets = CI->getArgOperand(i: OffsetsIdx);
776
777 Type *OldTy = CI->getType();
778 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
779
780 Value *UndefF = UndefValue::get(T: IRB.getFloatTy());
781 Value *UndefI = UndefValue::get(T: IRB.getInt32Ty());
782 // Common prefix: Handle, Sampler, Coord0..3, Offset0..2
783 SmallVector<Value *, 17> Args{Handle, Sampler, UndefF, UndefF, UndefF,
784 UndefF, UndefI, UndefI, UndefI};
785
786 // Copy coordinates and offsets into Args.
787 extractElementsIntoArgs(IRB, Args, ArgIdx: 2, Src: Coords, MaxElements: 4);
788 extractNonZeroOffsets(IRB, Args, ArgIdx: 6, Offsets, MaxElements: 3);
789
790 // Emit op-specific trailing arguments (e.g. Bias+Clamp, DDX+DDY+Clamp).
791 EmitExtraArgs(IRB, CI, Args);
792
793 Expected<CallInst *> OpCall =
794 OpBuilder.tryCreateOp(Op, Args, Name: CI->getName(), RetTy: NewRetTy);
795 if (Error E = OpCall.takeError())
796 return E;
797 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/false))
798 return E;
799
800 eraseDeadInsertElementChains(Vectors: VectorArgs);
801
802 return Error::success();
803 });
804 }
805
806 [[nodiscard]] bool lowerSample(Function &F, bool HasClamp) {
807 return lowerSampleOp(F, Op: OpCode::Sample, /*CoordsIdx=*/2, /*OffsetsIdx=*/3,
808 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
809 SmallVectorImpl<Value *> &Args) {
810 // Clamp
811 Args.push_back(
812 Elt: HasClamp ? CI->getArgOperand(i: 4)
813 : UndefValue::get(T: IRB.getFloatTy()));
814 });
815 }
816
817 [[nodiscard]] bool lowerSampleBias(Function &F, bool HasClamp) {
818 return lowerSampleOp(
819 F, Op: OpCode::SampleBias, /*CoordsIdx=*/2, /*OffsetsIdx=*/4,
820 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
821 SmallVectorImpl<Value *> &Args) {
822 // Bias is operand 3.
823 Args.push_back(Elt: CI->getArgOperand(i: 3));
824 // Clamp
825 Args.push_back(Elt: HasClamp ? CI->getArgOperand(i: 5)
826 : UndefValue::get(T: IRB.getFloatTy()));
827 });
828 }
829
830 [[nodiscard]] bool lowerSampleLevel(Function &F) {
831 return lowerSampleOp(
832 F, Op: OpCode::SampleLevel, /*CoordsIdx=*/2, /*OffsetsIdx=*/4,
833 EmitExtraArgs: [](IRBuilder<> &, CallInst *CI, SmallVectorImpl<Value *> &Args) {
834 // LOD is operand 3.
835 Args.push_back(Elt: CI->getArgOperand(i: 3));
836 });
837 }
838
839 [[nodiscard]] bool lowerSampleGrad(Function &F, bool HasClamp) {
840 return lowerSampleOp(
841 F, Op: OpCode::SampleGrad, /*CoordsIdx=*/2, /*OffsetsIdx=*/5,
842 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
843 SmallVectorImpl<Value *> &Args) {
844 Value *DDX = CI->getArgOperand(i: 3);
845 Value *DDY = CI->getArgOperand(i: 4);
846 Value *UndefF = UndefValue::get(T: IRB.getFloatTy());
847 // DDX0..2
848 size_t DDXStart = Args.size();
849 Args.append(NumInputs: 3, Elt: UndefF);
850 extractElementsIntoArgs(IRB, Args, ArgIdx: DDXStart, Src: DDX, MaxElements: 3);
851 // DDY0..2
852 size_t DDYStart = Args.size();
853 Args.append(NumInputs: 3, Elt: UndefF);
854 extractElementsIntoArgs(IRB, Args, ArgIdx: DDYStart, Src: DDY, MaxElements: 3);
855 // Clamp
856 Args.push_back(Elt: HasClamp ? CI->getArgOperand(i: 6) : UndefF);
857 });
858 }
859
860 [[nodiscard]] bool lowerRawBufferLoad(Function &F) {
861 const DataLayout &DL = F.getDataLayout();
862 IRBuilder<> &IRB = OpBuilder.getIRB();
863 Type *Int8Ty = IRB.getInt8Ty();
864 Type *Int32Ty = IRB.getInt32Ty();
865
866 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
867 IRB.SetInsertPoint(CI);
868
869 Type *OldTy = cast<StructType>(Val: CI->getType())->getElementType(N: 0);
870 Type *ScalarTy = OldTy->getScalarType();
871 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: ScalarTy);
872
873 Value *Handle =
874 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
875 Value *Index0 = CI->getArgOperand(i: 1);
876 Value *Index1 = CI->getArgOperand(i: 2);
877 uint64_t NumElements =
878 DL.getTypeSizeInBits(Ty: OldTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
879 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: ~(~0U << NumElements));
880 Value *Align =
881 ConstantInt::get(Ty: Int32Ty, V: DL.getPrefTypeAlign(Ty: ScalarTy).value());
882
883 if (Error E = validateRawBufferElementIndex(Resource: CI->getOperand(i_nocapture: 0), ElementIndex: Index1))
884 return E;
885 if (isa<PoisonValue>(Val: Index1))
886 Index1 = UndefValue::get(T: Index1->getType());
887
888 Expected<CallInst *> OpCall =
889 MMDI.DXILVersion >= VersionTuple(1, 2)
890 ? OpBuilder.tryCreateOp(Op: OpCode::RawBufferLoad,
891 Args: {Handle, Index0, Index1, Mask, Align},
892 Name: CI->getName(), RetTy: NewRetTy)
893 : OpBuilder.tryCreateOp(Op: OpCode::BufferLoad,
894 Args: {Handle, Index0, Index1}, Name: CI->getName(),
895 RetTy: NewRetTy);
896 if (Error E = OpCall.takeError())
897 return E;
898 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/true))
899 return E;
900
901 return Error::success();
902 });
903 }
904
905 [[nodiscard]] bool lowerCBufferLoad(Function &F) {
906 IRBuilder<> &IRB = OpBuilder.getIRB();
907
908 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
909 IRB.SetInsertPoint(CI);
910
911 Type *OldTy = cast<StructType>(Val: CI->getType())->getElementType(N: 0);
912 Type *ScalarTy = OldTy->getScalarType();
913 Type *NewRetTy = OpBuilder.getCBufRetType(ElementTy: ScalarTy);
914
915 Value *Handle =
916 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
917 Value *Index = CI->getArgOperand(i: 1);
918
919 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
920 Op: OpCode::CBufferLoadLegacy, Args: {Handle, Index}, Name: CI->getName(), RetTy: NewRetTy);
921 if (Error E = OpCall.takeError())
922 return E;
923 if (Error E = replaceNamedStructUses(Intrin: CI, DXILOp: *OpCall))
924 return E;
925
926 CI->eraseFromParent();
927 return Error::success();
928 });
929 }
930
931 [[nodiscard]] bool lowerUpdateCounter(Function &F) {
932 IRBuilder<> &IRB = OpBuilder.getIRB();
933 Type *Int32Ty = IRB.getInt32Ty();
934
935 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
936 IRB.SetInsertPoint(CI);
937 Value *Handle =
938 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
939 Value *Op1 = CI->getArgOperand(i: 1);
940
941 std::array<Value *, 2> Args{Handle, Op1};
942
943 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
944 Op: OpCode::UpdateCounter, Args, Name: CI->getName(), RetTy: Int32Ty);
945
946 if (Error E = OpCall.takeError())
947 return E;
948
949 CI->replaceAllUsesWith(V: *OpCall);
950 CI->eraseFromParent();
951 return Error::success();
952 });
953 }
954
955 [[nodiscard]] bool lowerBarrierByMemoryHandle(Function &F) {
956 IRBuilder<> &IRB = OpBuilder.getIRB();
957
958 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
959 IRB.SetInsertPoint(CI);
960 Value *Handle =
961 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
962 Value *SemanticFlags = CI->getArgOperand(i: 1);
963
964 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
965 Op: OpCode::BarrierByMemoryHandle, Args: {Handle, SemanticFlags});
966 if (Error E = OpCall.takeError())
967 return E;
968
969 CI->eraseFromParent();
970 return Error::success();
971 });
972 }
973
974 [[nodiscard]] bool lowerGetDimensionsX(Function &F) {
975 IRBuilder<> &IRB = OpBuilder.getIRB();
976 Type *Int32Ty = IRB.getInt32Ty();
977
978 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
979 IRB.SetInsertPoint(CI);
980 Value *Handle =
981 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
982 Value *Undef = UndefValue::get(T: Int32Ty);
983
984 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
985 Op: OpCode::GetDimensions, Args: {Handle, Undef}, Name: CI->getName(), RetTy: Int32Ty);
986 if (Error E = OpCall.takeError())
987 return E;
988 Value *Dim = IRB.CreateExtractValue(Agg: *OpCall, Idxs: 0);
989
990 CI->replaceAllUsesWith(V: Dim);
991 CI->eraseFromParent();
992 return Error::success();
993 });
994 }
995
996 [[nodiscard]] bool lowerGetPointer(Function &F) {
997 // These should have already been handled in DXILResourceAccess, so we can
998 // just clean up the dead prototype.
999 assert(F.user_empty() && "getpointer operations should have been removed");
1000 F.eraseFromParent();
1001 return false;
1002 }
1003
1004 /// Splits the value operand of a resource store into its (at most four)
1005 /// scalar components. Slots beyond the length of `Data` are filled with
1006 /// `undef` when `FillWithUndef` is set (raw and structured buffers), or with
1007 /// the first component otherwise (typed UAVs, which must write all four
1008 /// components - repeating the first one matches DXC).
1009 static std::array<Value *, 4> splitStoreData(IRBuilder<> &IRB, Value *Data,
1010 uint64_t NumElements,
1011 bool FillWithUndef) {
1012 std::array<Value *, 4> DataElements{nullptr, nullptr, nullptr, nullptr};
1013 extractElementsIntoArgs(IRB, Args: DataElements, ArgIdx: 0, Src: Data, MaxElements: 4);
1014
1015 // For any elements beyond the length of the vector, we should fill it up
1016 // with undef - however, for typed UAVs we repeat the first element to
1017 // match DXC.
1018 for (uint64_t I = NumElements, E = 4; I < E; ++I)
1019 if (DataElements[I] == nullptr)
1020 DataElements[I] =
1021 FillWithUndef ? UndefValue::get(T: Data->getType()->getScalarType())
1022 : DataElements[0];
1023
1024 return DataElements;
1025 }
1026
1027 /// Erase the chain of `insertelement`s that only existed to build up a vector
1028 /// operand of an intrinsic we've just replaced.
1029 static void eraseDeadInsertElementChain(Value *Data) {
1030 auto *IEI = dyn_cast<InsertElementInst>(Val: Data);
1031 while (IEI && IEI->use_empty()) {
1032 InsertElementInst *Tmp = IEI;
1033 IEI = dyn_cast<InsertElementInst>(Val: IEI->getOperand(i_nocapture: 0));
1034 Tmp->eraseFromParent();
1035 }
1036 }
1037
1038 [[nodiscard]] bool lowerBufferStore(Function &F, bool IsRaw) {
1039 const DataLayout &DL = F.getDataLayout();
1040 IRBuilder<> &IRB = OpBuilder.getIRB();
1041 Type *Int8Ty = IRB.getInt8Ty();
1042 Type *Int32Ty = IRB.getInt32Ty();
1043
1044 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1045 IRB.SetInsertPoint(CI);
1046
1047 Value *Handle =
1048 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1049 Value *Index0 = CI->getArgOperand(i: 1);
1050 Value *Index1 = IsRaw ? CI->getArgOperand(i: 2) : UndefValue::get(T: Int32Ty);
1051
1052 if (IsRaw) {
1053 if (Error E = validateRawBufferElementIndex(Resource: CI->getOperand(i_nocapture: 0), ElementIndex: Index1))
1054 return E;
1055 if (isa<PoisonValue>(Val: Index1))
1056 Index1 = UndefValue::get(T: Index1->getType());
1057 }
1058
1059 Value *Data = CI->getArgOperand(i: IsRaw ? 3 : 2);
1060 Type *DataTy = Data->getType();
1061 Type *ScalarTy = DataTy->getScalarType();
1062
1063 uint64_t NumElements =
1064 DL.getTypeSizeInBits(Ty: DataTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
1065 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: IsRaw ? ~(~0U << NumElements)
1066 : TypedUAVStoreWriteMask);
1067
1068 // TODO: check that we only have vector or scalar...
1069 if (NumElements > 4)
1070 return make_error<StringError>(
1071 Args: "Buffer store data must have at most 4 elements",
1072 Args: inconvertibleErrorCode());
1073
1074 std::array<Value *, 4> DataElements =
1075 splitStoreData(IRB, Data, NumElements, /*FillWithUndef=*/IsRaw);
1076
1077 dxil::OpCode Op = OpCode::BufferStore;
1078 SmallVector<Value *, 9> Args{
1079 Handle, Index0, Index1, DataElements[0],
1080 DataElements[1], DataElements[2], DataElements[3], Mask};
1081 if (IsRaw && MMDI.DXILVersion >= VersionTuple(1, 2)) {
1082 Op = OpCode::RawBufferStore;
1083 // RawBufferStore requires the alignment
1084 Args.push_back(
1085 Elt: ConstantInt::get(Ty: Int32Ty, V: DL.getPrefTypeAlign(Ty: ScalarTy).value()));
1086 }
1087 Expected<CallInst *> OpCall =
1088 OpBuilder.tryCreateOp(Op, Args, Name: CI->getName());
1089 if (Error E = OpCall.takeError())
1090 return E;
1091
1092 CI->eraseFromParent();
1093 eraseDeadInsertElementChain(Data);
1094
1095 return Error::success();
1096 });
1097 }
1098
1099 /// Snapshot the vector-typed arguments of `CI` so their `insertelement`
1100 /// chains can be cleaned up once `CI` has been replaced. The handles are weak
1101 /// because two arguments can share an `insertelement` chain.
1102 static SmallVector<WeakTrackingVH, 4> collectVectorArgs(CallInst *CI) {
1103 SmallVector<WeakTrackingVH, 4> Vectors;
1104 for (Value *Arg : CI->args())
1105 if (isa<FixedVectorType>(Val: Arg->getType()))
1106 Vectors.emplace_back(Args&: Arg);
1107 return Vectors;
1108 }
1109
1110 static void eraseDeadInsertElementChains(ArrayRef<WeakTrackingVH> Vectors) {
1111 for (const WeakTrackingVH &VH : Vectors)
1112 if (Value *V = VH)
1113 eraseDeadInsertElementChain(Data: V);
1114 }
1115
1116 [[nodiscard]] bool lowerTextureStore(Function &F) {
1117 const DataLayout &DL = F.getDataLayout();
1118 IRBuilder<> &IRB = OpBuilder.getIRB();
1119 Type *Int8Ty = IRB.getInt8Ty();
1120 Type *Int32Ty = IRB.getInt32Ty();
1121
1122 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1123 IRB.SetInsertPoint(CI);
1124
1125 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
1126 Value *Handle =
1127 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1128 Value *Coords = CI->getArgOperand(i: 1);
1129 Value *Data = CI->getArgOperand(i: 2);
1130
1131 Type *DataTy = Data->getType();
1132 Type *ScalarTy = DataTy->getScalarType();
1133 uint64_t NumElements =
1134 DL.getTypeSizeInBits(Ty: DataTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
1135 if (NumElements > 4)
1136 return make_error<StringError>(
1137 Args: "Texture store data must have at most 4 elements",
1138 Args: inconvertibleErrorCode());
1139
1140 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: TypedUAVStoreWriteMask);
1141 std::array<Value *, 4> DataElements =
1142 splitStoreData(IRB, Data, NumElements, /*FillWithUndef=*/false);
1143
1144 Value *Undef = UndefValue::get(T: Int32Ty);
1145 std::array<Value *, 9> Args{
1146 Handle, Undef, Undef,
1147 Undef, DataElements[0], DataElements[1],
1148 DataElements[2], DataElements[3], Mask};
1149
1150 // Copy the coordinates into Args.
1151 extractElementsIntoArgs(IRB, Args, ArgIdx: 1, Src: Coords, MaxElements: 3);
1152
1153 Expected<CallInst *> OpCall =
1154 OpBuilder.tryCreateOp(Op: OpCode::TextureStore, Args, Name: CI->getName());
1155 if (Error E = OpCall.takeError())
1156 return E;
1157
1158 CI->eraseFromParent();
1159 eraseDeadInsertElementChains(Vectors: VectorArgs);
1160
1161 return Error::success();
1162 });
1163 }
1164
1165 [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) {
1166 IRBuilder<> &IRB = OpBuilder.getIRB();
1167
1168 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1169 IRB.SetInsertPoint(CI);
1170
1171 // Cast the target-extension typed handle to `%dx.types.Handle`, tracked
1172 // via CleanupCasts so the pair is reconciled by `cleanupHandleCasts`.
1173 Value *Handle =
1174 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1175 Value *BinOp = CI->getArgOperand(i: 1);
1176 Value *Coord0 = CI->getArgOperand(i: 2);
1177 Value *Coord1 = CI->getArgOperand(i: 3);
1178 Value *Coord2 = CI->getArgOperand(i: 4);
1179 Value *NewValue = CI->getArgOperand(i: 5);
1180
1181 std::array<Value *, 6> Args{Handle, BinOp, Coord0,
1182 Coord1, Coord2, NewValue};
1183 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
1184 Op: dxil::OpCode::AtomicBinOp, Args, Name: CI->getName(), RetTy: CI->getType());
1185 if (Error E = OpCall.takeError()) {
1186 // Preserve the DXIL op error text but attach it as a
1187 // DiagnosticInfoUnsupported so we don't crash with a dangling call.
1188 std::string Message(toString(E: std::move(E)));
1189 CI->getContext().diagnose(DI: DiagnosticInfoUnsupported(
1190 *CI->getFunction(), Message, CI->getDebugLoc()));
1191 CI->replaceAllUsesWith(V: PoisonValue::get(T: CI->getType()));
1192 CI->eraseFromParent();
1193 return Error::success();
1194 }
1195
1196 CI->replaceAllUsesWith(V: *OpCall);
1197 CI->eraseFromParent();
1198 return Error::success();
1199 });
1200 }
1201
1202 [[nodiscard]] bool lowerResourceAtomicCompareExchange(Function &F) {
1203 IRBuilder<> &IRB = OpBuilder.getIRB();
1204
1205 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1206 IRB.SetInsertPoint(CI);
1207
1208 // Cast the target-extension typed handle to `%dx.types.Handle`, tracked
1209 // via CleanupCasts so the pair is reconciled by `cleanupHandleCasts`.
1210 Value *Handle =
1211 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1212 Value *Coord0 = CI->getArgOperand(i: 1);
1213 Value *Coord1 = CI->getArgOperand(i: 2);
1214 Value *Coord2 = CI->getArgOperand(i: 3);
1215 Value *CompareValue = CI->getArgOperand(i: 4);
1216 Value *NewValue = CI->getArgOperand(i: 5);
1217
1218 std::array<Value *, 6> Args{Handle, Coord0, Coord1,
1219 Coord2, CompareValue, NewValue};
1220 Expected<CallInst *> OpCall =
1221 OpBuilder.tryCreateOp(Op: dxil::OpCode::AtomicCompareExchange, Args,
1222 Name: CI->getName(), RetTy: CI->getType());
1223 if (Error E = OpCall.takeError()) {
1224 // Preserve the DXIL op error text but attach it as a
1225 // DiagnosticInfoUnsupported so we don't crash with a dangling call.
1226 std::string Message(toString(E: std::move(E)));
1227 CI->getContext().diagnose(DI: DiagnosticInfoUnsupported(
1228 *CI->getFunction(), Message, CI->getDebugLoc()));
1229 CI->replaceAllUsesWith(V: PoisonValue::get(T: CI->getType()));
1230 CI->eraseFromParent();
1231 return Error::success();
1232 }
1233
1234 CI->replaceAllUsesWith(V: *OpCall);
1235 CI->eraseFromParent();
1236 return Error::success();
1237 });
1238 }
1239
1240 [[nodiscard]] bool lowerCtpopToCountBits(Function &F) {
1241 IRBuilder<> &IRB = OpBuilder.getIRB();
1242 Type *Int32Ty = IRB.getInt32Ty();
1243
1244 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1245 IRB.SetInsertPoint(CI);
1246 SmallVector<Value *> Args;
1247 Args.append(in_start: CI->arg_begin(), in_end: CI->arg_end());
1248
1249 Type *RetTy = Int32Ty;
1250 Type *FRT = F.getReturnType();
1251 if (const auto *VT = dyn_cast<VectorType>(Val: FRT))
1252 RetTy = VectorType::get(ElementType: RetTy, Other: VT);
1253
1254 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
1255 Op: dxil::OpCode::CountBits, Args, Name: CI->getName(), RetTy);
1256 if (Error E = OpCall.takeError())
1257 return E;
1258
1259 // If the result type is 32 bits we can do a direct replacement.
1260 if (FRT->isIntOrIntVectorTy(BitWidth: 32)) {
1261 CI->replaceAllUsesWith(V: *OpCall);
1262 CI->eraseFromParent();
1263 return Error::success();
1264 }
1265
1266 unsigned CastOp;
1267 unsigned CastOp2;
1268 if (FRT->isIntOrIntVectorTy(BitWidth: 16)) {
1269 CastOp = Instruction::ZExt;
1270 CastOp2 = Instruction::SExt;
1271 } else { // must be 64 bits
1272 assert(FRT->isIntOrIntVectorTy(64) &&
1273 "Currently only lowering 16, 32, or 64 bit ctpop to CountBits \
1274 is supported.");
1275 CastOp = Instruction::Trunc;
1276 CastOp2 = Instruction::Trunc;
1277 }
1278
1279 // It is correct to replace the ctpop with the dxil op and
1280 // remove all casts to i32
1281 bool NeedsCast = false;
1282 for (User *User : make_early_inc_range(Range: CI->users())) {
1283 Instruction *I = dyn_cast<Instruction>(Val: User);
1284 if (I && (I->getOpcode() == CastOp || I->getOpcode() == CastOp2) &&
1285 I->getType() == RetTy) {
1286 I->replaceAllUsesWith(V: *OpCall);
1287 I->eraseFromParent();
1288 } else
1289 NeedsCast = true;
1290 }
1291
1292 // It is correct to replace a ctpop with the dxil op and
1293 // a cast from i32 to the return type of the ctpop
1294 // the cast is emitted here if there is a non-cast to i32
1295 // instr which uses the ctpop
1296 if (NeedsCast) {
1297 Value *Cast =
1298 IRB.CreateZExtOrTrunc(V: *OpCall, DestTy: F.getReturnType(), Name: "ctpop.cast");
1299 CI->replaceAllUsesWith(V: Cast);
1300 }
1301
1302 CI->eraseFromParent();
1303 return Error::success();
1304 });
1305 }
1306
1307 [[nodiscard]] bool lowerLifetimeIntrinsic(Function &F) {
1308 IRBuilder<> &IRB = OpBuilder.getIRB();
1309 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1310 IRB.SetInsertPoint(CI);
1311 Value *Ptr = CI->getArgOperand(i: 0);
1312 assert(Ptr->getType()->isPointerTy() &&
1313 "Expected operand of lifetime intrinsic to be a pointer");
1314
1315 auto ZeroOrUndef = [&](Type *Ty) {
1316 return MMDI.ValidatorVersion < VersionTuple(1, 6)
1317 ? Constant::getNullValue(Ty)
1318 : UndefValue::get(T: Ty);
1319 };
1320
1321 Value *Val = nullptr;
1322 if (auto *GV = dyn_cast<GlobalVariable>(Val: Ptr)) {
1323 if (GV->hasInitializer() || GV->isExternallyInitialized())
1324 return Error::success();
1325 Val = ZeroOrUndef(GV->getValueType());
1326 } else if (auto *AI = dyn_cast<AllocaInst>(Val: Ptr))
1327 Val = ZeroOrUndef(AI->getAllocatedType());
1328
1329 assert(Val && "Expected operand of lifetime intrinsic to be a global "
1330 "variable or alloca instruction");
1331 IRB.CreateStore(Val, Ptr, isVolatile: false);
1332
1333 CI->eraseFromParent();
1334 return Error::success();
1335 });
1336 }
1337
1338 [[nodiscard]] bool lowerIsFPClass(Function &F) {
1339 IRBuilder<> &IRB = OpBuilder.getIRB();
1340 Type *RetTy = IRB.getInt1Ty();
1341
1342 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1343 IRB.SetInsertPoint(CI);
1344 SmallVector<Value *> Args;
1345 Value *Fl = CI->getArgOperand(i: 0);
1346 Args.push_back(Elt: Fl);
1347
1348 dxil::OpCode OpCode;
1349 Value *T = CI->getArgOperand(i: 1);
1350 auto *TCI = dyn_cast<ConstantInt>(Val: T);
1351 switch (TCI->getZExtValue()) {
1352 case FPClassTest::fcInf:
1353 OpCode = dxil::OpCode::IsInf;
1354 break;
1355 case FPClassTest::fcNan:
1356 OpCode = dxil::OpCode::IsNaN;
1357 break;
1358 case FPClassTest::fcNormal:
1359 OpCode = dxil::OpCode::IsNormal;
1360 break;
1361 case FPClassTest::fcFinite:
1362 OpCode = dxil::OpCode::IsFinite;
1363 break;
1364 default:
1365 SmallString<128> Msg =
1366 formatv(Fmt: "Unsupported FPClassTest {0} for DXIL Op Lowering",
1367 Vals: TCI->getZExtValue());
1368 return make_error<StringError>(Args&: Msg, Args: inconvertibleErrorCode());
1369 }
1370
1371 Expected<CallInst *> OpCall =
1372 OpBuilder.tryCreateOp(Op: OpCode, Args, Name: CI->getName(), RetTy);
1373 if (Error E = OpCall.takeError())
1374 return E;
1375
1376 CI->replaceAllUsesWith(V: *OpCall);
1377 CI->eraseFromParent();
1378 return Error::success();
1379 });
1380 }
1381
1382 bool lowerIntrinsics() {
1383 bool Updated = false;
1384 bool HasErrors = false;
1385
1386 for (Function &F : make_early_inc_range(Range: M.functions())) {
1387 if (!F.isDeclaration())
1388 continue;
1389 Intrinsic::ID ID = F.getIntrinsicID();
1390 switch (ID) {
1391 // NOTE: Skip dx_resource_casthandle here. They are
1392 // resolved after this loop in cleanupHandleCasts.
1393 case Intrinsic::dx_resource_casthandle:
1394 // NOTE: llvm.dbg.value is supported as is in DXIL.
1395 case Intrinsic::dbg_value:
1396 case Intrinsic::not_intrinsic:
1397 if (F.use_empty())
1398 F.eraseFromParent();
1399 continue;
1400 default:
1401 if (F.use_empty())
1402 F.eraseFromParent();
1403 else {
1404 SmallString<128> Msg = formatv(
1405 Fmt: "Unsupported intrinsic {0} for DXIL lowering", Vals: F.getName());
1406 M.getContext().emitError(ErrorStr: Msg);
1407 HasErrors |= true;
1408 }
1409 break;
1410
1411#define DXIL_OP_INTRINSIC(OpCode, Intrin, ...) \
1412 case Intrin: \
1413 HasErrors |= replaceFunctionWithOp( \
1414 F, OpCode, ArrayRef<IntrinArgSelect>{__VA_ARGS__}); \
1415 break;
1416#include "DXILOperation.inc"
1417 case Intrinsic::dx_resource_handlefrombinding:
1418 HasErrors |= lowerHandleFromBinding(F);
1419 break;
1420 case Intrinsic::dx_resource_handlefromheap:
1421 HasErrors |= lowerHandleFromHeap(F);
1422 break;
1423 case Intrinsic::dx_resource_getbasepointer:
1424 case Intrinsic::dx_resource_getpointer:
1425 HasErrors |= lowerGetPointer(F);
1426 break;
1427 case Intrinsic::dx_resource_nonuniformindex:
1428 assert(!CleanupNURI &&
1429 "overloaded llvm.dx.resource.nonuniformindex intrinsics?");
1430 CleanupNURI = &F;
1431 break;
1432 case Intrinsic::dx_resource_load_typedbuffer:
1433 HasErrors |= lowerTypedBufferLoad(F, /*HasCheckBit=*/true);
1434 break;
1435 case Intrinsic::dx_resource_load_level:
1436 HasErrors |= lowerTextureLoad(F);
1437 break;
1438 case Intrinsic::dx_resource_sample:
1439 HasErrors |= lowerSample(F, /*HasClamp=*/false);
1440 break;
1441 case Intrinsic::dx_resource_sample_clamp:
1442 HasErrors |= lowerSample(F, /*HasClamp=*/true);
1443 break;
1444 case Intrinsic::dx_resource_samplebias:
1445 HasErrors |= lowerSampleBias(F, /*HasClamp=*/false);
1446 break;
1447 case Intrinsic::dx_resource_samplebias_clamp:
1448 HasErrors |= lowerSampleBias(F, /*HasClamp=*/true);
1449 break;
1450 case Intrinsic::dx_resource_samplelevel:
1451 HasErrors |= lowerSampleLevel(F);
1452 break;
1453 case Intrinsic::dx_resource_samplegrad:
1454 HasErrors |= lowerSampleGrad(F, /*HasClamp=*/false);
1455 break;
1456 case Intrinsic::dx_resource_samplegrad_clamp:
1457 HasErrors |= lowerSampleGrad(F, /*HasClamp=*/true);
1458 break;
1459 case Intrinsic::dx_resource_store_typedbuffer:
1460 HasErrors |= lowerBufferStore(F, /*IsRaw=*/false);
1461 break;
1462 case Intrinsic::dx_resource_store_texture:
1463 HasErrors |= lowerTextureStore(F);
1464 break;
1465 case Intrinsic::dx_resource_load_rawbuffer:
1466 HasErrors |= lowerRawBufferLoad(F);
1467 break;
1468 case Intrinsic::dx_resource_store_rawbuffer:
1469 HasErrors |= lowerBufferStore(F, /*IsRaw=*/true);
1470 break;
1471 case Intrinsic::dx_resource_load_cbufferrow_2:
1472 case Intrinsic::dx_resource_load_cbufferrow_4:
1473 case Intrinsic::dx_resource_load_cbufferrow_8:
1474 HasErrors |= lowerCBufferLoad(F);
1475 break;
1476 case Intrinsic::dx_resource_updatecounter:
1477 HasErrors |= lowerUpdateCounter(F);
1478 break;
1479 case Intrinsic::dx_barrier_by_memory_handle:
1480 HasErrors |= lowerBarrierByMemoryHandle(F);
1481 break;
1482 case Intrinsic::dx_resource_atomic_binop:
1483 HasErrors |= lowerResourceAtomicBinOp(F);
1484 break;
1485 case Intrinsic::dx_resource_atomic_compare_exchange:
1486 HasErrors |= lowerResourceAtomicCompareExchange(F);
1487 break;
1488 case Intrinsic::dx_resource_getdimensions_x:
1489 HasErrors |= lowerGetDimensionsX(F);
1490 break;
1491 case Intrinsic::ctpop:
1492 HasErrors |= lowerCtpopToCountBits(F);
1493 break;
1494 case Intrinsic::lifetime_start:
1495 case Intrinsic::lifetime_end:
1496 if (F.use_empty())
1497 F.eraseFromParent();
1498 else {
1499 if (MMDI.DXILVersion < VersionTuple(1, 6))
1500 HasErrors |= lowerLifetimeIntrinsic(F);
1501 else
1502 continue;
1503 }
1504 break;
1505 case Intrinsic::is_fpclass:
1506 HasErrors |= lowerIsFPClass(F);
1507 break;
1508 }
1509 Updated = true;
1510 }
1511 if (Updated && !HasErrors) {
1512 cleanupHandleCasts();
1513 cleanupNonUniformResourceIndexCalls();
1514 }
1515
1516 return Updated;
1517 }
1518};
1519} // namespace
1520
1521PreservedAnalyses DXILOpLowering::run(Module &M, ModuleAnalysisManager &MAM) {
1522 DXILResourceMap &DRM = MAM.getResult<DXILResourceAnalysis>(IR&: M);
1523 DXILResourceTypeMap &DRTM = MAM.getResult<DXILResourceTypeAnalysis>(IR&: M);
1524 const ModuleMetadataInfo MMDI = MAM.getResult<DXILMetadataAnalysis>(IR&: M);
1525
1526 const bool MadeChanges = OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
1527 if (!MadeChanges)
1528 return PreservedAnalyses::all();
1529 PreservedAnalyses PA;
1530 PA.preserve<DXILResourceAnalysis>();
1531 PA.preserve<DXILMetadataAnalysis>();
1532 PA.preserve<ShaderFlagsAnalysis>();
1533 PA.preserve<RootSignatureAnalysis>();
1534 PA.preserve<SignatureAnalysis>();
1535 return PA;
1536}
1537
1538namespace {
1539class DXILOpLoweringLegacy : public ModulePass {
1540public:
1541 bool runOnModule(Module &M) override {
1542 DXILResourceMap &DRM =
1543 getAnalysis<DXILResourceWrapperPass>().getResourceMap();
1544 DXILResourceTypeMap &DRTM =
1545 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
1546 const ModuleMetadataInfo MMDI =
1547 getAnalysis<DXILMetadataAnalysisWrapperPass>().getModuleMetadata();
1548
1549 return OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
1550 }
1551 StringRef getPassName() const override { return "DXIL Op Lowering"; }
1552 DXILOpLoweringLegacy() : ModulePass(ID) {}
1553
1554 static char ID; // Pass identification.
1555 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
1556 AU.addRequired<DXILResourceTypeWrapperPass>();
1557 AU.addRequired<DXILResourceWrapperPass>();
1558 AU.addRequired<DXILMetadataAnalysisWrapperPass>();
1559 AU.addPreserved<DXILResourceWrapperPass>();
1560 AU.addPreserved<DXILMetadataAnalysisWrapperPass>();
1561 AU.addPreserved<ShaderFlagsAnalysisWrapper>();
1562 AU.addPreserved<RootSignatureAnalysisWrapper>();
1563 AU.addPreserved<SignatureAnalysisWrapper>();
1564 }
1565};
1566char DXILOpLoweringLegacy::ID = 0;
1567} // end anonymous namespace
1568
1569INITIALIZE_PASS_BEGIN(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering",
1570 false, false)
1571INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
1572INITIALIZE_PASS_DEPENDENCY(DXILResourceWrapperPass)
1573INITIALIZE_PASS_END(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering", false,
1574 false)
1575
1576ModulePass *llvm::createDXILOpLoweringLegacyPass() {
1577 return new DXILOpLoweringLegacy();
1578}
1579