1//===- DXILOpLowering.cpp - Lowering to DXIL operations -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DXILOpLowering.h"
10#include "DXILConstants.h"
11#include "DXILOpBuilder.h"
12#include "DXILRootSignature.h"
13#include "DXILShaderFlags.h"
14#include "DirectX.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/Analysis/DXILMetadataAnalysis.h"
17#include "llvm/Analysis/DXILResource.h"
18#include "llvm/CodeGen/Passes.h"
19#include "llvm/IR/Constant.h"
20#include "llvm/IR/DiagnosticInfo.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsDirectX.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PassManager.h"
28#include "llvm/IR/Use.h"
29#include "llvm/IR/ValueHandle.h"
30#include "llvm/InitializePasses.h"
31#include "llvm/Pass.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/FormatVariadic.h"
34
35#define DEBUG_TYPE "dxil-op-lower"
36
37using namespace llvm;
38using namespace llvm::dxil;
39
40/// Write mask covering all four components of a UAV element. Typed UAV stores
41/// (textures and typed buffers) must always use this mask - the DXIL validator
42/// rejects anything narrower. Only raw and / structured buffer stores may use a
43/// partial mask.
44static constexpr uint8_t TypedUAVStoreWriteMask = 0xF;
45
46namespace {
47class OpLowerer {
48 Module &M;
49 DXILOpBuilder OpBuilder;
50 DXILResourceMap &DRM;
51 DXILResourceTypeMap &DRTM;
52 const ModuleMetadataInfo &MMDI;
53 SmallVector<CallInst *> CleanupCasts;
54 Function *CleanupNURI = nullptr;
55
56public:
57 OpLowerer(Module &M, DXILResourceMap &DRM, DXILResourceTypeMap &DRTM,
58 const ModuleMetadataInfo &MMDI)
59 : M(M), OpBuilder(M), DRM(DRM), DRTM(DRTM), MMDI(MMDI) {}
60
61 /// Replace every call to \c F using \c ReplaceCall, and then erase \c F. If
62 /// there is an error replacing a call, we emit a diagnostic and return true.
63 [[nodiscard]] bool
64 replaceFunction(Function &F,
65 llvm::function_ref<Error(CallInst *CI)> ReplaceCall) {
66 for (User *U : make_early_inc_range(Range: F.users())) {
67 CallInst *CI = dyn_cast<CallInst>(Val: U);
68 if (!CI)
69 continue;
70
71 if (Error E = ReplaceCall(CI)) {
72 std::string Message(toString(E: std::move(E)));
73 M.getContext().diagnose(DI: DiagnosticInfoUnsupported(
74 *CI->getFunction(), Message, CI->getDebugLoc()));
75
76 return true;
77 }
78 }
79 if (F.user_empty())
80 F.eraseFromParent();
81 return false;
82 }
83
84 struct IntrinArgSelect {
85 enum class Type {
86#define DXIL_OP_INTRINSIC_ARG_SELECT_TYPE(name) name,
87#include "DXILOperation.inc"
88 };
89 Type Type;
90 int Value;
91 };
92
93 /// Replaces uses of a struct with uses of an equivalent named struct.
94 ///
95 /// DXIL operations that return structs give them well known names, so we need
96 /// to update uses when we switch from an LLVM intrinsic to an op.
97 Error replaceNamedStructUses(CallInst *Intrin, CallInst *DXILOp) {
98 auto *IntrinTy = cast<StructType>(Val: Intrin->getType());
99 auto *DXILOpTy = cast<StructType>(Val: DXILOp->getType());
100 if (!IntrinTy->isLayoutIdentical(Other: DXILOpTy))
101 return make_error<StringError>(
102 Args: "Type mismatch between intrinsic and DXIL op",
103 Args: inconvertibleErrorCode());
104
105 for (Use &U : make_early_inc_range(Range: Intrin->uses()))
106 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: U.getUser()))
107 EVI->setOperand(i_nocapture: 0, Val_nocapture: DXILOp);
108 else if (auto *IVI = dyn_cast<InsertValueInst>(Val: U.getUser()))
109 IVI->setOperand(i_nocapture: 0, Val_nocapture: DXILOp);
110 else
111 return make_error<StringError>(Args: "DXIL ops that return structs may only "
112 "be used by insert- and extractvalue",
113 Args: inconvertibleErrorCode());
114 return Error::success();
115 }
116
117 bool isFast(FastMathFlags Flags) {
118 // HLSL Fast Math doesn't enable AllowContract flag; This can be
119 // removed when we enable it in the future.
120 return Flags.allowReassoc() && Flags.noNaNs() && Flags.noInfs() &&
121 Flags.noSignedZeros() && Flags.allowReciprocal() &&
122 Flags.approxFunc();
123 }
124
125 void setDxPrecise(CallInst *CI) {
126 const StringRef Key = "dx.precise";
127 Module *M = CI->getModule();
128
129 LLVMContext &Ctx = M->getContext();
130 MDNode *One =
131 llvm::MDNode::get(Context&: Ctx, MDs: ConstantAsMetadata::get(C: ConstantInt::get(
132 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1)));
133
134 CI->setMetadata(Kind: Key, Node: One);
135 }
136
137 [[nodiscard]] bool
138 replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp,
139 ArrayRef<IntrinArgSelect> ArgSelects) {
140 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
141 OpBuilder.getIRB().SetInsertPoint(CI);
142 SmallVector<Value *> Args;
143 if (ArgSelects.size()) {
144 for (const IntrinArgSelect &A : ArgSelects) {
145 switch (A.Type) {
146 case IntrinArgSelect::Type::Index:
147 Args.push_back(Elt: CI->getArgOperand(i: A.Value));
148 break;
149 case IntrinArgSelect::Type::I8:
150 Args.push_back(Elt: OpBuilder.getIRB().getInt8(C: (uint8_t)A.Value));
151 break;
152 case IntrinArgSelect::Type::I32:
153 Args.push_back(Elt: OpBuilder.getIRB().getInt32(C: A.Value));
154 break;
155 }
156 }
157 } else {
158 Args.append(in_start: CI->arg_begin(), in_end: CI->arg_end());
159 }
160
161 Expected<CallInst *> OpCall =
162 OpBuilder.tryCreateOp(Op: DXILOp, Args, Name: CI->getName(), RetTy: F.getReturnType());
163 if (Error E = OpCall.takeError())
164 return E;
165
166 if (isa<FPMathOperator>(Val: CI) &&
167 !isFast(Flags: cast<FPMathOperator>(Val: CI)->getFastMathFlags()))
168 setDxPrecise(*OpCall);
169
170 if (isa<StructType>(Val: CI->getType())) {
171 if (Error E = replaceNamedStructUses(Intrin: CI, DXILOp: *OpCall))
172 return E;
173 } else
174 CI->replaceAllUsesWith(V: *OpCall);
175
176 CI->eraseFromParent();
177 return Error::success();
178 });
179 }
180
181 /// Create a cast between a `target("dx")` type and `dx.types.Handle`, which
182 /// is intended to be removed by the end of lowering. This is used to allow
183 /// lowering of ops which need to change their return or argument types in a
184 /// piecemeal way - we can add the casts in to avoid updating all of the uses
185 /// or defs, and by the end all of the casts will be redundant.
186 Value *createTmpHandleCast(Value *V, Type *Ty) {
187 CallInst *Cast = OpBuilder.getIRB().CreateIntrinsicWithoutFolding(
188 ID: Intrinsic::dx_resource_casthandle, OverloadTypes: {Ty, V->getType()}, Args: {V});
189 CleanupCasts.push_back(Elt: Cast);
190 return Cast;
191 }
192
193 void cleanupHandleCasts() {
194 SmallVector<CallInst *> ToRemove;
195 SmallVector<Function *> CastFns;
196
197 for (CallInst *Cast : CleanupCasts) {
198 // These casts were only put in to ease the move from `target("dx")` types
199 // to `dx.types.Handle in a piecemeal way. At this point, all of the
200 // non-cast uses should now be `dx.types.Handle`, and remaining casts
201 // should all form pairs to and from the now unused `target("dx")` type.
202 CastFns.push_back(Elt: Cast->getCalledFunction());
203
204 // If the cast is not to `dx.types.Handle`, it should be the first part of
205 // the pair. Keep track so we can remove it once it has no more uses.
206 if (Cast->getType() != OpBuilder.getHandleType()) {
207 ToRemove.push_back(Elt: Cast);
208 continue;
209 }
210 // Otherwise, we're the second handle in a pair. Forward the arguments and
211 // remove the (second) cast.
212 CallInst *Def = cast<CallInst>(Val: Cast->getOperand(i_nocapture: 0));
213 assert(Def->getIntrinsicID() == Intrinsic::dx_resource_casthandle &&
214 "Unbalanced pair of temporary handle casts");
215 Cast->replaceAllUsesWith(V: Def->getOperand(i_nocapture: 0));
216 Cast->eraseFromParent();
217 }
218 for (CallInst *Cast : ToRemove) {
219 assert(Cast->user_empty() && "Temporary handle cast still has users");
220 Cast->eraseFromParent();
221 }
222
223 // Deduplicate the cast functions so that we only erase each one once.
224 llvm::sort(C&: CastFns);
225 CastFns.erase(CS: llvm::unique(R&: CastFns), CE: CastFns.end());
226 for (Function *F : CastFns)
227 F->eraseFromParent();
228
229 CleanupCasts.clear();
230 }
231
232 void cleanupNonUniformResourceIndexCalls() {
233 // Replace all NonUniformResourceIndex calls with their argument.
234 if (!CleanupNURI)
235 return;
236 for (User *U : make_early_inc_range(Range: CleanupNURI->users())) {
237 CallInst *CI = dyn_cast<CallInst>(Val: U);
238 if (!CI)
239 continue;
240 CI->replaceAllUsesWith(V: CI->getArgOperand(i: 0));
241 CI->eraseFromParent();
242 }
243 CleanupNURI->eraseFromParent();
244 CleanupNURI = nullptr;
245 }
246
247 // Remove the resource global associated with the handleFromBinding call
248 // instruction and their uses as they aren't needed anymore.
249 // TODO: We should verify that all the globals get removed.
250 // It's expected we'll need a custom pass in the future that will eliminate
251 // the need for this here.
252 void removeResourceGlobals(CallInst *CI) {
253 for (User *User : make_early_inc_range(Range: CI->users())) {
254 if (StoreInst *Store = dyn_cast<StoreInst>(Val: User)) {
255 Value *V = Store->getOperand(i_nocapture: 1);
256 Store->eraseFromParent();
257 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: V))
258 if (GV->use_empty()) {
259 GV->removeDeadConstantUsers();
260 GV->eraseFromParent();
261 }
262 }
263 }
264 }
265
266 void replaceHandleFromBindingCall(CallInst *CI, Value *Replacement) {
267 assert(CI->getCalledFunction()->getIntrinsicID() ==
268 Intrinsic::dx_resource_handlefrombinding);
269
270 removeResourceGlobals(CI);
271
272 auto *NameGlobal = dyn_cast<llvm::GlobalVariable>(Val: CI->getArgOperand(i: 4));
273
274 CI->replaceAllUsesWith(V: Replacement);
275 CI->eraseFromParent();
276
277 if (NameGlobal && NameGlobal->use_empty())
278 NameGlobal->eraseFromParent();
279 }
280
281 bool hasNonUniformIndex(Value *IndexOp) {
282 if (isa<llvm::Constant>(Val: IndexOp))
283 return false;
284
285 SmallVector<Value *, 16> Worklist;
286 SmallPtrSet<Value *, 16> Visited;
287 Worklist.push_back(Elt: IndexOp);
288
289 while (!Worklist.empty()) {
290 Value *V = Worklist.pop_back_val();
291
292 if (isa<llvm::Constant>(Val: V))
293 continue;
294
295 if (!Visited.insert(Ptr: V).second)
296 continue;
297
298 if (auto *CI = dyn_cast<CallInst>(Val: V))
299 if (CI->getIntrinsicID() == Intrinsic::dx_resource_nonuniformindex)
300 return true;
301
302 // If it's a PHI node, check ALL incoming values —
303 // taint from ANY predecessor counts
304 if (auto *Phi = dyn_cast<PHINode>(Val: V)) {
305 for (Value *Incoming : Phi->incoming_values())
306 Worklist.push_back(Elt: Incoming);
307 continue;
308 }
309
310 if (auto *Inst = dyn_cast<Instruction>(Val: V))
311 if (Inst->getNumOperands() > 0 && !Inst->isTerminator())
312 for (Value *Op : Inst->operands())
313 Worklist.push_back(Elt: Op);
314 }
315 return false;
316 }
317
318 Error validateRawBufferElementIndex(Value *Resource, Value *ElementIndex) {
319 bool IsStructured =
320 cast<RawBufferExtType>(Val: Resource->getType())->isStructured();
321 bool IsPoison = isa<PoisonValue>(Val: ElementIndex);
322
323 if (IsStructured && IsPoison)
324 return make_error<StringError>(
325 Args: "Element index of structured buffer may not be poison",
326 Args: inconvertibleErrorCode());
327
328 if (!IsStructured && !IsPoison)
329 return make_error<StringError>(
330 Args: "Element index of raw buffer must be poison",
331 Args: inconvertibleErrorCode());
332
333 return Error::success();
334 }
335
336 [[nodiscard]] bool lowerToCreateHandle(Function &F) {
337 IRBuilder<> &IRB = OpBuilder.getIRB();
338 Type *Int8Ty = IRB.getInt8Ty();
339 Type *Int32Ty = IRB.getInt32Ty();
340 Type *Int1Ty = IRB.getInt1Ty();
341
342 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
343 IRB.SetInsertPoint(CI);
344
345 auto *It = DRM.find(Key: CI);
346 assert(It != DRM.end() && "Resource not in map?");
347 dxil::ResourceInfo &RI = *It;
348
349 const auto &Binding = RI.getBinding();
350 dxil::ResourceClass RC = DRTM[RI.getHandleTy()].getResourceClass();
351
352 Value *IndexOp = CI->getArgOperand(i: 3);
353 if (Binding.LowerBound != 0)
354 IndexOp = IRB.CreateAdd(LHS: IndexOp,
355 RHS: ConstantInt::get(Ty: Int32Ty, V: Binding.LowerBound));
356
357 bool HasNonUniformIndex =
358 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);
359 std::array<Value *, 4> Args{
360 ConstantInt::get(Ty: Int8Ty, V: llvm::to_underlying(E: RC)),
361 ConstantInt::get(Ty: Int32Ty, V: Binding.BindingID), IndexOp,
362 ConstantInt::get(Ty: Int1Ty, V: HasNonUniformIndex)};
363 Expected<CallInst *> OpCall =
364 OpBuilder.tryCreateOp(Op: OpCode::CreateHandle, Args, Name: CI->getName());
365 if (Error E = OpCall.takeError())
366 return E;
367
368 Value *Cast = createTmpHandleCast(V: *OpCall, Ty: CI->getType());
369 replaceHandleFromBindingCall(CI, Replacement: Cast);
370 return Error::success();
371 });
372 }
373
374 [[nodiscard]] bool lowerToBindAndAnnotateHandle(Function &F) {
375 IRBuilder<> &IRB = OpBuilder.getIRB();
376 Type *Int32Ty = IRB.getInt32Ty();
377 Type *Int1Ty = IRB.getInt1Ty();
378
379 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
380 IRB.SetInsertPoint(CI);
381
382 auto *It = DRM.find(Key: CI);
383 assert(It != DRM.end() && "Resource not in map?");
384 dxil::ResourceInfo &RI = *It;
385
386 const auto &Binding = RI.getBinding();
387 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];
388 dxil::ResourceClass RC = RTI.getResourceClass();
389
390 Value *IndexOp = CI->getArgOperand(i: 3);
391 if (Binding.LowerBound != 0)
392 IndexOp = IRB.CreateAdd(LHS: IndexOp,
393 RHS: ConstantInt::get(Ty: Int32Ty, V: Binding.LowerBound));
394
395 std::pair<uint32_t, uint32_t> Props =
396 RI.getAnnotateProps(M&: *F.getParent(), RTI);
397
398 // For `CreateHandleFromBinding` we need the upper bound rather than the
399 // size, so we need to be careful about the difference for "unbounded".
400 uint32_t UpperBound = Binding.Size == 0
401 ? std::numeric_limits<uint32_t>::max()
402 : Binding.LowerBound + Binding.Size - 1;
403 Constant *ResBind = OpBuilder.getResBind(LowerBound: Binding.LowerBound, UpperBound,
404 SpaceID: Binding.Space, RC);
405 bool NonUniformIndex =
406 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);
407 Constant *NonUniformOp = ConstantInt::get(Ty: Int1Ty, V: NonUniformIndex);
408 std::array<Value *, 3> BindArgs{ResBind, IndexOp, NonUniformOp};
409 Expected<CallInst *> OpBind = OpBuilder.tryCreateOp(
410 Op: OpCode::CreateHandleFromBinding, Args: BindArgs, Name: CI->getName());
411 if (Error E = OpBind.takeError())
412 return E;
413
414 std::array<Value *, 2> AnnotateArgs{
415 *OpBind, OpBuilder.getResProps(Word0: Props.first, Word1: Props.second)};
416 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(
417 Op: OpCode::AnnotateHandle, Args: AnnotateArgs,
418 Name: CI->hasName() ? CI->getName() + "_annot" : Twine());
419 if (Error E = OpAnnotate.takeError())
420 return E;
421
422 Value *Cast = createTmpHandleCast(V: *OpAnnotate, Ty: CI->getType());
423 replaceHandleFromBindingCall(CI, Replacement: Cast);
424 return Error::success();
425 });
426 }
427
428 /// Lower `dx.resource.handlefrombinding` intrinsics depending on the shader
429 /// model and taking into account binding information from
430 /// DXILResourceAnalysis.
431 bool lowerHandleFromBinding(Function &F) {
432 if (MMDI.DXILVersion < VersionTuple(1, 6))
433 return lowerToCreateHandle(F);
434 return lowerToBindAndAnnotateHandle(F);
435 }
436
437 bool lowerHandleFromHeap(Function &F) {
438 IRBuilder<> &IRB = OpBuilder.getIRB();
439
440 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
441 IRB.SetInsertPoint(CI);
442
443 auto *It = DRM.find(Key: CI);
444 assert(It != DRM.end() && "Resource not in map?");
445 dxil::ResourceInfo &RI = *It;
446 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];
447
448 Value *IndexOp = CI->getArgOperand(i: 0);
449 Value *IsSamplerHeap =
450 ConstantInt::getBool(Context&: IRB.getContext(), V: RTI.isSampler());
451
452 std::pair<uint32_t, uint32_t> Props =
453 RI.getAnnotateProps(M&: *F.getParent(), RTI);
454
455 bool NonUniformIndex = hasNonUniformIndex(IndexOp);
456 Value *NonUniformOp =
457 ConstantInt::getBool(Context&: IRB.getContext(), V: NonUniformIndex);
458
459 std::array<Value *, 3> Args{IndexOp, IsSamplerHeap, NonUniformOp};
460 Expected<CallInst *> OpCreateHandle = OpBuilder.tryCreateOp(
461 Op: OpCode::CreateHandleFromHeap, Args, Name: CI->getName());
462 if (Error E = OpCreateHandle.takeError())
463 return E;
464
465 std::array<Value *, 2> AnnotateArgs{
466 *OpCreateHandle, OpBuilder.getResProps(Word0: Props.first, Word1: Props.second)};
467 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(
468 Op: OpCode::AnnotateHandle, Args: AnnotateArgs,
469 Name: CI->hasName() ? CI->getName() + "_annot" : Twine());
470 if (Error E = OpAnnotate.takeError())
471 return E;
472
473 Value *Cast = createTmpHandleCast(V: *OpAnnotate, Ty: CI->getType());
474 CI->replaceAllUsesWith(V: Cast);
475 CI->eraseFromParent();
476 return Error::success();
477 });
478 }
479
480 /// Replace uses of \c Intrin with the values in the `dx.ResRet` of \c Op.
481 /// Since we expect to be post-scalarization, make an effort to avoid vectors.
482 Error replaceResRetUses(CallInst *Intrin, CallInst *Op, bool HasCheckBit) {
483 IRBuilder<> &IRB = OpBuilder.getIRB();
484
485 Instruction *OldResult = Intrin;
486 Type *OldTy = Intrin->getType();
487
488 if (HasCheckBit) {
489 auto *ST = cast<StructType>(Val: OldTy);
490
491 Value *CheckOp = nullptr;
492 Type *Int32Ty = IRB.getInt32Ty();
493 for (Use &U : make_early_inc_range(Range: OldResult->uses())) {
494 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: U.getUser())) {
495 ArrayRef<unsigned> Indices = EVI->getIndices();
496 assert(Indices.size() == 1);
497 // We're only interested in uses of the check bit for now.
498 if (Indices[0] != 1)
499 continue;
500 if (!CheckOp) {
501 Value *NewEVI = IRB.CreateExtractValue(Agg: Op, Idxs: 4);
502 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
503 Op: OpCode::CheckAccessFullyMapped, Args: {NewEVI},
504 Name: OldResult->hasName() ? OldResult->getName() + "_check"
505 : Twine(),
506 RetTy: Int32Ty);
507 if (Error E = OpCall.takeError())
508 return E;
509 CheckOp = *OpCall;
510 }
511 EVI->replaceAllUsesWith(V: CheckOp);
512 EVI->eraseFromParent();
513 }
514 }
515
516 if (OldResult->use_empty()) {
517 // Only the check bit was used, so we're done here.
518 OldResult->eraseFromParent();
519 return Error::success();
520 }
521
522 assert(OldResult->hasOneUse() &&
523 isa<ExtractValueInst>(*OldResult->user_begin()) &&
524 "Expected only use to be extract of first element");
525 OldResult = cast<Instruction>(Val: *OldResult->user_begin());
526 OldTy = ST->getElementType(N: 0);
527 }
528
529 // For scalars, we just extract the first element.
530 if (!isa<FixedVectorType>(Val: OldTy)) {
531 Value *EVI = IRB.CreateExtractValue(Agg: Op, Idxs: 0);
532 OldResult->replaceAllUsesWith(V: EVI);
533 OldResult->eraseFromParent();
534 if (OldResult != Intrin) {
535 assert(Intrin->use_empty() && "Intrinsic still has uses?");
536 Intrin->eraseFromParent();
537 }
538 return Error::success();
539 }
540
541 std::array<Value *, 4> Extracts = {};
542 SmallVector<ExtractElementInst *> DynamicAccesses;
543
544 // The users of the operation should all be scalarized, so we attempt to
545 // replace the extractelements with extractvalues directly.
546 for (Use &U : make_early_inc_range(Range: OldResult->uses())) {
547 if (auto *EEI = dyn_cast<ExtractElementInst>(Val: U.getUser())) {
548 if (auto *IndexOp = dyn_cast<ConstantInt>(Val: EEI->getIndexOperand())) {
549 size_t IndexVal = IndexOp->getZExtValue();
550 assert(IndexVal < 4 && "Index into buffer load out of range");
551 if (!Extracts[IndexVal])
552 Extracts[IndexVal] = IRB.CreateExtractValue(Agg: Op, Idxs: IndexVal);
553 EEI->replaceAllUsesWith(V: Extracts[IndexVal]);
554 EEI->eraseFromParent();
555 } else {
556 DynamicAccesses.push_back(Elt: EEI);
557 }
558 }
559 }
560
561 const auto *VecTy = cast<FixedVectorType>(Val: OldTy);
562 const unsigned N = VecTy->getNumElements();
563
564 // If there's a dynamic access we need to round trip through stack memory so
565 // that we don't leave vectors around.
566 if (!DynamicAccesses.empty()) {
567 Type *Int32Ty = IRB.getInt32Ty();
568 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
569
570 Type *ElTy = VecTy->getElementType();
571 Type *ArrayTy = ArrayType::get(ElementType: ElTy, NumElements: N);
572 Value *Alloca = IRB.CreateAlloca(Ty: ArrayTy);
573
574 for (int I = 0, E = N; I != E; ++I) {
575 if (!Extracts[I])
576 Extracts[I] = IRB.CreateExtractValue(Agg: Op, Idxs: I);
577 Value *GEP = GetElementPtrInst::CreateInBounds(
578 PointeeType: ArrayTy, Ptr: Alloca, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: I)}, NameStr: "",
579 InsertBefore: IRB.GetInsertPoint());
580 IRB.CreateStore(Val: Extracts[I], Ptr: GEP);
581 }
582
583 for (ExtractElementInst *EEI : DynamicAccesses) {
584 Value *GEP = GetElementPtrInst::CreateInBounds(
585 PointeeType: ArrayTy, Ptr: Alloca, IdxList: {Zero, EEI->getIndexOperand()}, NameStr: "",
586 InsertBefore: IRB.GetInsertPoint());
587 Value *Load = IRB.CreateLoad(Ty: ElTy, Ptr: GEP);
588 EEI->replaceAllUsesWith(V: Load);
589 EEI->eraseFromParent();
590 }
591 }
592
593 // If we still have uses, then we're not fully scalarized and need to
594 // recreate the vector. This should only happen for things like exported
595 // functions from libraries.
596 if (!OldResult->use_empty()) {
597 for (int I = 0, E = N; I != E; ++I)
598 if (!Extracts[I])
599 Extracts[I] = IRB.CreateExtractValue(Agg: Op, Idxs: I);
600
601 Value *Vec = PoisonValue::get(T: OldTy);
602 for (int I = 0, E = N; I != E; ++I)
603 Vec = IRB.CreateInsertElement(Vec, NewElt: Extracts[I], Idx: I);
604 OldResult->replaceAllUsesWith(V: Vec);
605 }
606
607 OldResult->eraseFromParent();
608 if (OldResult != Intrin) {
609 assert(Intrin->use_empty() && "Intrinsic still has uses?");
610 Intrin->eraseFromParent();
611 }
612
613 return Error::success();
614 }
615
616 [[nodiscard]] bool lowerTypedBufferLoad(Function &F, bool HasCheckBit) {
617 IRBuilder<> &IRB = OpBuilder.getIRB();
618 Type *Int32Ty = IRB.getInt32Ty();
619
620 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
621 IRB.SetInsertPoint(CI);
622
623 Value *Handle =
624 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
625 Value *Index0 = CI->getArgOperand(i: 1);
626 Value *Index1 = UndefValue::get(T: Int32Ty);
627
628 Type *OldTy = CI->getType();
629 if (HasCheckBit)
630 OldTy = cast<StructType>(Val: OldTy)->getElementType(N: 0);
631 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
632
633 std::array<Value *, 3> Args{Handle, Index0, Index1};
634 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
635 Op: OpCode::BufferLoad, Args, Name: CI->getName(), RetTy: NewRetTy);
636 if (Error E = OpCall.takeError())
637 return E;
638 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, HasCheckBit))
639 return E;
640
641 return Error::success();
642 });
643 }
644
645 /// Recover the scalar components of `Vec` from the `insertelement` chain that
646 /// built it. Since we run after the scalarizer, such a chain is usually just
647 /// a temporary gathered to pass the vector to a call.
648 static void collectInsertedElements(Value *Vec,
649 MutableArrayRef<Value *> Elements) {
650 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
651 assert(NumElts <= Elements.size() && "Not enough room for the components");
652
653 SmallVector<InsertElementInst *, 4> Chain;
654 for (auto *IEI = dyn_cast<InsertElementInst>(Val: Vec); IEI;
655 IEI = dyn_cast<InsertElementInst>(Val: IEI->getOperand(i_nocapture: 0))) {
656 if (!isa<ConstantInt>(Val: IEI->getOperand(i_nocapture: 2)))
657 break; // This break should never happen below SM6.9.
658 Chain.push_back(Elt: IEI);
659 }
660
661 // Replay element insertion from the innermost first, so that a repeated
662 // index ends up holding the live value.
663 while (!Chain.empty()) {
664 InsertElementInst *IEI = Chain.pop_back_val();
665 uint64_t IndexVal = cast<ConstantInt>(Val: IEI->getOperand(i_nocapture: 2))->getZExtValue();
666 if (IndexVal < NumElts)
667 Elements[IndexVal] = IEI->getOperand(i_nocapture: 1);
668 }
669 }
670
671 // Copies `Src` into `Args` starting at `ArgIdx`. If `Src` is a vector, its
672 // elements are placed in consecutive slots; otherwise `Src` is stored
673 // directly. At most `MaxElements` elements are expected.
674 static void extractElementsIntoArgs(IRBuilder<> &IRB,
675 MutableArrayRef<Value *> Args,
676 unsigned ArgIdx, Value *Src,
677 unsigned MaxElements) {
678 auto *VecTy = dyn_cast<FixedVectorType>(Val: Src->getType());
679 if (!VecTy) {
680 Args[ArgIdx] = Src;
681 return;
682 }
683
684 unsigned Count = VecTy->getNumElements();
685 assert(Count <= MaxElements && "Too many elements for the arg list");
686
687 SmallVector<Value *, 4> Elements(Count, nullptr);
688 collectInsertedElements(Vec: Src, Elements);
689
690 for (unsigned I = 0; I < Count; ++I)
691 Args[ArgIdx + I] = Elements[I]
692 ? Elements[I]
693 : IRB.CreateExtractElement(
694 Vec: Src, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: I));
695 }
696
697 /// Copy offsets into the argument list at the given index, unless
698 /// the offsets are known to be zero (i.e., a null constant).
699 static void extractNonZeroOffsets(IRBuilder<> &IRB,
700 MutableArrayRef<Value *> Args,
701 unsigned ArgIdx, Value *Offsets,
702 unsigned MaxElements) {
703 auto *COff = dyn_cast<Constant>(Val: Offsets);
704 bool OffsetsAreZero = COff && COff->isNullValue();
705 if (!OffsetsAreZero)
706 extractElementsIntoArgs(IRB, Args, ArgIdx, Src: Offsets, MaxElements);
707 }
708
709 [[nodiscard]] bool lowerTextureLoad(Function &F) {
710 IRBuilder<> &IRB = OpBuilder.getIRB();
711 Type *Int32Ty = IRB.getInt32Ty();
712
713 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
714 IRB.SetInsertPoint(CI);
715
716 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
717 Value *Handle =
718 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
719 Value *Coords = CI->getArgOperand(i: 1);
720 Value *MipLevel = CI->getArgOperand(i: 2);
721 Value *Offsets = CI->getArgOperand(i: 3);
722
723 // A UAV descriptor binds a single mip slice, so there is no mip to select
724 // in the case of a UAV. Multisampled UAVs are the exception: the slot
725 // carries a sample index and stays live.
726 auto *HandleTy = cast<TargetExtType>(Val: CI->getArgOperand(i: 0)->getType());
727 dxil::ResourceTypeInfo &RTI = DRTM[HandleTy];
728 dxil::ResourceKind Kind = RTI.getResourceKind();
729 if (RTI.isUAV() && Kind != dxil::ResourceKind::Texture2DMS &&
730 Kind != dxil::ResourceKind::Texture2DMSArray)
731 MipLevel = UndefValue::get(T: Int32Ty);
732
733 Type *OldTy = CI->getType();
734 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
735
736 Value *Undef = UndefValue::get(T: Int32Ty);
737 std::array<Value *, 8> Args{Handle, MipLevel, Undef, Undef,
738 Undef, Undef, Undef, Undef};
739
740 // Copy coordinates and offsets into Args.
741 extractElementsIntoArgs(IRB, Args, ArgIdx: 2, Src: Coords, MaxElements: 3);
742 extractNonZeroOffsets(IRB, Args, ArgIdx: 5, Offsets, MaxElements: 3);
743
744 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
745 Op: OpCode::TextureLoad, Args, Name: CI->getName(), RetTy: NewRetTy);
746 if (Error E = OpCall.takeError())
747 return E;
748 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/false))
749 return E;
750
751 eraseDeadInsertElementChains(Vectors: VectorArgs);
752
753 return Error::success();
754 });
755 }
756
757 /// Common helper for lowering sample operations (SampleBias, SampleGrad,
758 /// etc.) that share the same pattern: extract handle/sampler, unpack
759 /// coordinates and offsets, build the DXIL arg list, and replace uses.
760 [[nodiscard]] bool lowerSampleOp(
761 Function &F, OpCode Op, unsigned CoordsIdx, unsigned OffsetsIdx,
762 llvm::function_ref<void(IRBuilder<> &, CallInst *,
763 SmallVectorImpl<Value *> &)> EmitExtraArgs) {
764 IRBuilder<> &IRB = OpBuilder.getIRB();
765 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
766 IRB.SetInsertPoint(CI);
767
768 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
769 Value *Handle =
770 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
771 Value *Sampler =
772 createTmpHandleCast(V: CI->getArgOperand(i: 1), Ty: OpBuilder.getHandleType());
773 Value *Coords = CI->getArgOperand(i: CoordsIdx);
774 Value *Offsets = CI->getArgOperand(i: OffsetsIdx);
775
776 Type *OldTy = CI->getType();
777 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
778
779 Value *UndefF = UndefValue::get(T: IRB.getFloatTy());
780 Value *UndefI = UndefValue::get(T: IRB.getInt32Ty());
781 // Common prefix: Handle, Sampler, Coord0..3, Offset0..2
782 SmallVector<Value *, 17> Args{Handle, Sampler, UndefF, UndefF, UndefF,
783 UndefF, UndefI, UndefI, UndefI};
784
785 // Copy coordinates and offsets into Args.
786 extractElementsIntoArgs(IRB, Args, ArgIdx: 2, Src: Coords, MaxElements: 4);
787 extractNonZeroOffsets(IRB, Args, ArgIdx: 6, Offsets, MaxElements: 3);
788
789 // Emit op-specific trailing arguments (e.g. Bias+Clamp, DDX+DDY+Clamp).
790 EmitExtraArgs(IRB, CI, Args);
791
792 Expected<CallInst *> OpCall =
793 OpBuilder.tryCreateOp(Op, Args, Name: CI->getName(), RetTy: NewRetTy);
794 if (Error E = OpCall.takeError())
795 return E;
796 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/false))
797 return E;
798
799 eraseDeadInsertElementChains(Vectors: VectorArgs);
800
801 return Error::success();
802 });
803 }
804
805 [[nodiscard]] bool lowerSample(Function &F, bool HasClamp) {
806 return lowerSampleOp(F, Op: OpCode::Sample, /*CoordsIdx=*/2, /*OffsetsIdx=*/3,
807 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
808 SmallVectorImpl<Value *> &Args) {
809 // Clamp
810 Args.push_back(
811 Elt: HasClamp ? CI->getArgOperand(i: 4)
812 : UndefValue::get(T: IRB.getFloatTy()));
813 });
814 }
815
816 [[nodiscard]] bool lowerSampleBias(Function &F, bool HasClamp) {
817 return lowerSampleOp(
818 F, Op: OpCode::SampleBias, /*CoordsIdx=*/2, /*OffsetsIdx=*/4,
819 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
820 SmallVectorImpl<Value *> &Args) {
821 // Bias is operand 3.
822 Args.push_back(Elt: CI->getArgOperand(i: 3));
823 // Clamp
824 Args.push_back(Elt: HasClamp ? CI->getArgOperand(i: 5)
825 : UndefValue::get(T: IRB.getFloatTy()));
826 });
827 }
828
829 [[nodiscard]] bool lowerSampleLevel(Function &F) {
830 return lowerSampleOp(
831 F, Op: OpCode::SampleLevel, /*CoordsIdx=*/2, /*OffsetsIdx=*/4,
832 EmitExtraArgs: [](IRBuilder<> &, CallInst *CI, SmallVectorImpl<Value *> &Args) {
833 // LOD is operand 3.
834 Args.push_back(Elt: CI->getArgOperand(i: 3));
835 });
836 }
837
838 [[nodiscard]] bool lowerSampleGrad(Function &F, bool HasClamp) {
839 return lowerSampleOp(
840 F, Op: OpCode::SampleGrad, /*CoordsIdx=*/2, /*OffsetsIdx=*/5,
841 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
842 SmallVectorImpl<Value *> &Args) {
843 Value *DDX = CI->getArgOperand(i: 3);
844 Value *DDY = CI->getArgOperand(i: 4);
845 Value *UndefF = UndefValue::get(T: IRB.getFloatTy());
846 // DDX0..2
847 size_t DDXStart = Args.size();
848 Args.append(NumInputs: 3, Elt: UndefF);
849 extractElementsIntoArgs(IRB, Args, ArgIdx: DDXStart, Src: DDX, MaxElements: 3);
850 // DDY0..2
851 size_t DDYStart = Args.size();
852 Args.append(NumInputs: 3, Elt: UndefF);
853 extractElementsIntoArgs(IRB, Args, ArgIdx: DDYStart, Src: DDY, MaxElements: 3);
854 // Clamp
855 Args.push_back(Elt: HasClamp ? CI->getArgOperand(i: 6) : UndefF);
856 });
857 }
858
859 [[nodiscard]] bool lowerRawBufferLoad(Function &F) {
860 const DataLayout &DL = F.getDataLayout();
861 IRBuilder<> &IRB = OpBuilder.getIRB();
862 Type *Int8Ty = IRB.getInt8Ty();
863 Type *Int32Ty = IRB.getInt32Ty();
864
865 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
866 IRB.SetInsertPoint(CI);
867
868 Type *OldTy = cast<StructType>(Val: CI->getType())->getElementType(N: 0);
869 Type *ScalarTy = OldTy->getScalarType();
870 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: ScalarTy);
871
872 Value *Handle =
873 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
874 Value *Index0 = CI->getArgOperand(i: 1);
875 Value *Index1 = CI->getArgOperand(i: 2);
876 uint64_t NumElements =
877 DL.getTypeSizeInBits(Ty: OldTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
878 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: ~(~0U << NumElements));
879 Value *Align =
880 ConstantInt::get(Ty: Int32Ty, V: DL.getPrefTypeAlign(Ty: ScalarTy).value());
881
882 if (Error E = validateRawBufferElementIndex(Resource: CI->getOperand(i_nocapture: 0), ElementIndex: Index1))
883 return E;
884 if (isa<PoisonValue>(Val: Index1))
885 Index1 = UndefValue::get(T: Index1->getType());
886
887 Expected<CallInst *> OpCall =
888 MMDI.DXILVersion >= VersionTuple(1, 2)
889 ? OpBuilder.tryCreateOp(Op: OpCode::RawBufferLoad,
890 Args: {Handle, Index0, Index1, Mask, Align},
891 Name: CI->getName(), RetTy: NewRetTy)
892 : OpBuilder.tryCreateOp(Op: OpCode::BufferLoad,
893 Args: {Handle, Index0, Index1}, Name: CI->getName(),
894 RetTy: NewRetTy);
895 if (Error E = OpCall.takeError())
896 return E;
897 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/true))
898 return E;
899
900 return Error::success();
901 });
902 }
903
904 [[nodiscard]] bool lowerCBufferLoad(Function &F) {
905 IRBuilder<> &IRB = OpBuilder.getIRB();
906
907 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
908 IRB.SetInsertPoint(CI);
909
910 Type *OldTy = cast<StructType>(Val: CI->getType())->getElementType(N: 0);
911 Type *ScalarTy = OldTy->getScalarType();
912 Type *NewRetTy = OpBuilder.getCBufRetType(ElementTy: ScalarTy);
913
914 Value *Handle =
915 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
916 Value *Index = CI->getArgOperand(i: 1);
917
918 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
919 Op: OpCode::CBufferLoadLegacy, Args: {Handle, Index}, Name: CI->getName(), RetTy: NewRetTy);
920 if (Error E = OpCall.takeError())
921 return E;
922 if (Error E = replaceNamedStructUses(Intrin: CI, DXILOp: *OpCall))
923 return E;
924
925 CI->eraseFromParent();
926 return Error::success();
927 });
928 }
929
930 [[nodiscard]] bool lowerUpdateCounter(Function &F) {
931 IRBuilder<> &IRB = OpBuilder.getIRB();
932 Type *Int32Ty = IRB.getInt32Ty();
933
934 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
935 IRB.SetInsertPoint(CI);
936 Value *Handle =
937 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
938 Value *Op1 = CI->getArgOperand(i: 1);
939
940 std::array<Value *, 2> Args{Handle, Op1};
941
942 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
943 Op: OpCode::UpdateCounter, Args, Name: CI->getName(), RetTy: Int32Ty);
944
945 if (Error E = OpCall.takeError())
946 return E;
947
948 CI->replaceAllUsesWith(V: *OpCall);
949 CI->eraseFromParent();
950 return Error::success();
951 });
952 }
953
954 [[nodiscard]] bool lowerBarrierByMemoryHandle(Function &F) {
955 IRBuilder<> &IRB = OpBuilder.getIRB();
956
957 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
958 IRB.SetInsertPoint(CI);
959 Value *Handle =
960 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
961 Value *SemanticFlags = CI->getArgOperand(i: 1);
962
963 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
964 Op: OpCode::BarrierByMemoryHandle, Args: {Handle, SemanticFlags});
965 if (Error E = OpCall.takeError())
966 return E;
967
968 CI->eraseFromParent();
969 return Error::success();
970 });
971 }
972
973 [[nodiscard]] bool lowerGetDimensionsX(Function &F) {
974 IRBuilder<> &IRB = OpBuilder.getIRB();
975 Type *Int32Ty = IRB.getInt32Ty();
976
977 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
978 IRB.SetInsertPoint(CI);
979 Value *Handle =
980 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
981 Value *Undef = UndefValue::get(T: Int32Ty);
982
983 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
984 Op: OpCode::GetDimensions, Args: {Handle, Undef}, Name: CI->getName(), RetTy: Int32Ty);
985 if (Error E = OpCall.takeError())
986 return E;
987 Value *Dim = IRB.CreateExtractValue(Agg: *OpCall, Idxs: 0);
988
989 CI->replaceAllUsesWith(V: Dim);
990 CI->eraseFromParent();
991 return Error::success();
992 });
993 }
994
995 [[nodiscard]] bool lowerGetPointer(Function &F) {
996 // These should have already been handled in DXILResourceAccess, so we can
997 // just clean up the dead prototype.
998 assert(F.user_empty() && "getpointer operations should have been removed");
999 F.eraseFromParent();
1000 return false;
1001 }
1002
1003 /// Splits the value operand of a resource store into its (at most four)
1004 /// scalar components. Slots beyond the length of `Data` are filled with
1005 /// `undef` when `FillWithUndef` is set (raw and structured buffers), or with
1006 /// the first component otherwise (typed UAVs, which must write all four
1007 /// components - repeating the first one matches DXC).
1008 static std::array<Value *, 4> splitStoreData(IRBuilder<> &IRB, Value *Data,
1009 uint64_t NumElements,
1010 bool FillWithUndef) {
1011 std::array<Value *, 4> DataElements{nullptr, nullptr, nullptr, nullptr};
1012 extractElementsIntoArgs(IRB, Args: DataElements, ArgIdx: 0, Src: Data, MaxElements: 4);
1013
1014 // For any elements beyond the length of the vector, we should fill it up
1015 // with undef - however, for typed UAVs we repeat the first element to
1016 // match DXC.
1017 for (uint64_t I = NumElements, E = 4; I < E; ++I)
1018 if (DataElements[I] == nullptr)
1019 DataElements[I] =
1020 FillWithUndef ? UndefValue::get(T: Data->getType()->getScalarType())
1021 : DataElements[0];
1022
1023 return DataElements;
1024 }
1025
1026 /// Erase the chain of `insertelement`s that only existed to build up a vector
1027 /// operand of an intrinsic we've just replaced.
1028 static void eraseDeadInsertElementChain(Value *Data) {
1029 auto *IEI = dyn_cast<InsertElementInst>(Val: Data);
1030 while (IEI && IEI->use_empty()) {
1031 InsertElementInst *Tmp = IEI;
1032 IEI = dyn_cast<InsertElementInst>(Val: IEI->getOperand(i_nocapture: 0));
1033 Tmp->eraseFromParent();
1034 }
1035 }
1036
1037 [[nodiscard]] bool lowerBufferStore(Function &F, bool IsRaw) {
1038 const DataLayout &DL = F.getDataLayout();
1039 IRBuilder<> &IRB = OpBuilder.getIRB();
1040 Type *Int8Ty = IRB.getInt8Ty();
1041 Type *Int32Ty = IRB.getInt32Ty();
1042
1043 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1044 IRB.SetInsertPoint(CI);
1045
1046 Value *Handle =
1047 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1048 Value *Index0 = CI->getArgOperand(i: 1);
1049 Value *Index1 = IsRaw ? CI->getArgOperand(i: 2) : UndefValue::get(T: Int32Ty);
1050
1051 if (IsRaw) {
1052 if (Error E = validateRawBufferElementIndex(Resource: CI->getOperand(i_nocapture: 0), ElementIndex: Index1))
1053 return E;
1054 if (isa<PoisonValue>(Val: Index1))
1055 Index1 = UndefValue::get(T: Index1->getType());
1056 }
1057
1058 Value *Data = CI->getArgOperand(i: IsRaw ? 3 : 2);
1059 Type *DataTy = Data->getType();
1060 Type *ScalarTy = DataTy->getScalarType();
1061
1062 uint64_t NumElements =
1063 DL.getTypeSizeInBits(Ty: DataTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
1064 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: IsRaw ? ~(~0U << NumElements)
1065 : TypedUAVStoreWriteMask);
1066
1067 // TODO: check that we only have vector or scalar...
1068 if (NumElements > 4)
1069 return make_error<StringError>(
1070 Args: "Buffer store data must have at most 4 elements",
1071 Args: inconvertibleErrorCode());
1072
1073 std::array<Value *, 4> DataElements =
1074 splitStoreData(IRB, Data, NumElements, /*FillWithUndef=*/IsRaw);
1075
1076 dxil::OpCode Op = OpCode::BufferStore;
1077 SmallVector<Value *, 9> Args{
1078 Handle, Index0, Index1, DataElements[0],
1079 DataElements[1], DataElements[2], DataElements[3], Mask};
1080 if (IsRaw && MMDI.DXILVersion >= VersionTuple(1, 2)) {
1081 Op = OpCode::RawBufferStore;
1082 // RawBufferStore requires the alignment
1083 Args.push_back(
1084 Elt: ConstantInt::get(Ty: Int32Ty, V: DL.getPrefTypeAlign(Ty: ScalarTy).value()));
1085 }
1086 Expected<CallInst *> OpCall =
1087 OpBuilder.tryCreateOp(Op, Args, Name: CI->getName());
1088 if (Error E = OpCall.takeError())
1089 return E;
1090
1091 CI->eraseFromParent();
1092 eraseDeadInsertElementChain(Data);
1093
1094 return Error::success();
1095 });
1096 }
1097
1098 /// Snapshot the vector-typed arguments of `CI` so their `insertelement`
1099 /// chains can be cleaned up once `CI` has been replaced. The handles are weak
1100 /// because two arguments can share an `insertelement` chain.
1101 static SmallVector<WeakTrackingVH, 4> collectVectorArgs(CallInst *CI) {
1102 SmallVector<WeakTrackingVH, 4> Vectors;
1103 for (Value *Arg : CI->args())
1104 if (isa<FixedVectorType>(Val: Arg->getType()))
1105 Vectors.emplace_back(Args&: Arg);
1106 return Vectors;
1107 }
1108
1109 static void eraseDeadInsertElementChains(ArrayRef<WeakTrackingVH> Vectors) {
1110 for (const WeakTrackingVH &VH : Vectors)
1111 if (Value *V = VH)
1112 eraseDeadInsertElementChain(Data: V);
1113 }
1114
1115 [[nodiscard]] bool lowerTextureStore(Function &F) {
1116 const DataLayout &DL = F.getDataLayout();
1117 IRBuilder<> &IRB = OpBuilder.getIRB();
1118 Type *Int8Ty = IRB.getInt8Ty();
1119 Type *Int32Ty = IRB.getInt32Ty();
1120
1121 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1122 IRB.SetInsertPoint(CI);
1123
1124 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
1125 Value *Handle =
1126 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1127 Value *Coords = CI->getArgOperand(i: 1);
1128 Value *Data = CI->getArgOperand(i: 2);
1129
1130 Type *DataTy = Data->getType();
1131 Type *ScalarTy = DataTy->getScalarType();
1132 uint64_t NumElements =
1133 DL.getTypeSizeInBits(Ty: DataTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
1134 if (NumElements > 4)
1135 return make_error<StringError>(
1136 Args: "Texture store data must have at most 4 elements",
1137 Args: inconvertibleErrorCode());
1138
1139 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: TypedUAVStoreWriteMask);
1140 std::array<Value *, 4> DataElements =
1141 splitStoreData(IRB, Data, NumElements, /*FillWithUndef=*/false);
1142
1143 Value *Undef = UndefValue::get(T: Int32Ty);
1144 std::array<Value *, 9> Args{
1145 Handle, Undef, Undef,
1146 Undef, DataElements[0], DataElements[1],
1147 DataElements[2], DataElements[3], Mask};
1148
1149 // Copy the coordinates into Args.
1150 extractElementsIntoArgs(IRB, Args, ArgIdx: 1, Src: Coords, MaxElements: 3);
1151
1152 Expected<CallInst *> OpCall =
1153 OpBuilder.tryCreateOp(Op: OpCode::TextureStore, Args, Name: CI->getName());
1154 if (Error E = OpCall.takeError())
1155 return E;
1156
1157 CI->eraseFromParent();
1158 eraseDeadInsertElementChains(Vectors: VectorArgs);
1159
1160 return Error::success();
1161 });
1162 }
1163
1164 [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) {
1165 IRBuilder<> &IRB = OpBuilder.getIRB();
1166
1167 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1168 IRB.SetInsertPoint(CI);
1169
1170 // Cast the target-extension typed handle to `%dx.types.Handle`, tracked
1171 // via CleanupCasts so the pair is reconciled by `cleanupHandleCasts`.
1172 Value *Handle =
1173 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1174 Value *BinOp = CI->getArgOperand(i: 1);
1175 Value *Coord0 = CI->getArgOperand(i: 2);
1176 Value *Coord1 = CI->getArgOperand(i: 3);
1177 Value *Coord2 = CI->getArgOperand(i: 4);
1178 Value *NewValue = CI->getArgOperand(i: 5);
1179
1180 std::array<Value *, 6> Args{Handle, BinOp, Coord0,
1181 Coord1, Coord2, NewValue};
1182 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
1183 Op: dxil::OpCode::AtomicBinOp, Args, Name: CI->getName(), RetTy: CI->getType());
1184 if (Error E = OpCall.takeError()) {
1185 // Preserve the DXIL op error text but attach it as a
1186 // DiagnosticInfoUnsupported so we don't crash with a dangling call.
1187 std::string Message(toString(E: std::move(E)));
1188 CI->getContext().diagnose(DI: DiagnosticInfoUnsupported(
1189 *CI->getFunction(), Message, CI->getDebugLoc()));
1190 CI->replaceAllUsesWith(V: PoisonValue::get(T: CI->getType()));
1191 CI->eraseFromParent();
1192 return Error::success();
1193 }
1194
1195 CI->replaceAllUsesWith(V: *OpCall);
1196 CI->eraseFromParent();
1197 return Error::success();
1198 });
1199 }
1200
1201 [[nodiscard]] bool lowerResourceAtomicCompareExchange(Function &F) {
1202 IRBuilder<> &IRB = OpBuilder.getIRB();
1203
1204 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1205 IRB.SetInsertPoint(CI);
1206
1207 // Cast the target-extension typed handle to `%dx.types.Handle`, tracked
1208 // via CleanupCasts so the pair is reconciled by `cleanupHandleCasts`.
1209 Value *Handle =
1210 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1211 Value *Coord0 = CI->getArgOperand(i: 1);
1212 Value *Coord1 = CI->getArgOperand(i: 2);
1213 Value *Coord2 = CI->getArgOperand(i: 3);
1214 Value *CompareValue = CI->getArgOperand(i: 4);
1215 Value *NewValue = CI->getArgOperand(i: 5);
1216
1217 std::array<Value *, 6> Args{Handle, Coord0, Coord1,
1218 Coord2, CompareValue, NewValue};
1219 Expected<CallInst *> OpCall =
1220 OpBuilder.tryCreateOp(Op: dxil::OpCode::AtomicCompareExchange, Args,
1221 Name: CI->getName(), RetTy: CI->getType());
1222 if (Error E = OpCall.takeError()) {
1223 // Preserve the DXIL op error text but attach it as a
1224 // DiagnosticInfoUnsupported so we don't crash with a dangling call.
1225 std::string Message(toString(E: std::move(E)));
1226 CI->getContext().diagnose(DI: DiagnosticInfoUnsupported(
1227 *CI->getFunction(), Message, CI->getDebugLoc()));
1228 CI->replaceAllUsesWith(V: PoisonValue::get(T: CI->getType()));
1229 CI->eraseFromParent();
1230 return Error::success();
1231 }
1232
1233 CI->replaceAllUsesWith(V: *OpCall);
1234 CI->eraseFromParent();
1235 return Error::success();
1236 });
1237 }
1238
1239 [[nodiscard]] bool lowerCtpopToCountBits(Function &F) {
1240 IRBuilder<> &IRB = OpBuilder.getIRB();
1241 Type *Int32Ty = IRB.getInt32Ty();
1242
1243 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1244 IRB.SetInsertPoint(CI);
1245 SmallVector<Value *> Args;
1246 Args.append(in_start: CI->arg_begin(), in_end: CI->arg_end());
1247
1248 Type *RetTy = Int32Ty;
1249 Type *FRT = F.getReturnType();
1250 if (const auto *VT = dyn_cast<VectorType>(Val: FRT))
1251 RetTy = VectorType::get(ElementType: RetTy, Other: VT);
1252
1253 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
1254 Op: dxil::OpCode::CountBits, Args, Name: CI->getName(), RetTy);
1255 if (Error E = OpCall.takeError())
1256 return E;
1257
1258 // If the result type is 32 bits we can do a direct replacement.
1259 if (FRT->isIntOrIntVectorTy(BitWidth: 32)) {
1260 CI->replaceAllUsesWith(V: *OpCall);
1261 CI->eraseFromParent();
1262 return Error::success();
1263 }
1264
1265 unsigned CastOp;
1266 unsigned CastOp2;
1267 if (FRT->isIntOrIntVectorTy(BitWidth: 16)) {
1268 CastOp = Instruction::ZExt;
1269 CastOp2 = Instruction::SExt;
1270 } else { // must be 64 bits
1271 assert(FRT->isIntOrIntVectorTy(64) &&
1272 "Currently only lowering 16, 32, or 64 bit ctpop to CountBits \
1273 is supported.");
1274 CastOp = Instruction::Trunc;
1275 CastOp2 = Instruction::Trunc;
1276 }
1277
1278 // It is correct to replace the ctpop with the dxil op and
1279 // remove all casts to i32
1280 bool NeedsCast = false;
1281 for (User *User : make_early_inc_range(Range: CI->users())) {
1282 Instruction *I = dyn_cast<Instruction>(Val: User);
1283 if (I && (I->getOpcode() == CastOp || I->getOpcode() == CastOp2) &&
1284 I->getType() == RetTy) {
1285 I->replaceAllUsesWith(V: *OpCall);
1286 I->eraseFromParent();
1287 } else
1288 NeedsCast = true;
1289 }
1290
1291 // It is correct to replace a ctpop with the dxil op and
1292 // a cast from i32 to the return type of the ctpop
1293 // the cast is emitted here if there is a non-cast to i32
1294 // instr which uses the ctpop
1295 if (NeedsCast) {
1296 Value *Cast =
1297 IRB.CreateZExtOrTrunc(V: *OpCall, DestTy: F.getReturnType(), Name: "ctpop.cast");
1298 CI->replaceAllUsesWith(V: Cast);
1299 }
1300
1301 CI->eraseFromParent();
1302 return Error::success();
1303 });
1304 }
1305
1306 [[nodiscard]] bool lowerLifetimeIntrinsic(Function &F) {
1307 IRBuilder<> &IRB = OpBuilder.getIRB();
1308 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1309 IRB.SetInsertPoint(CI);
1310 Value *Ptr = CI->getArgOperand(i: 0);
1311 assert(Ptr->getType()->isPointerTy() &&
1312 "Expected operand of lifetime intrinsic to be a pointer");
1313
1314 auto ZeroOrUndef = [&](Type *Ty) {
1315 return MMDI.ValidatorVersion < VersionTuple(1, 6)
1316 ? Constant::getNullValue(Ty)
1317 : UndefValue::get(T: Ty);
1318 };
1319
1320 Value *Val = nullptr;
1321 if (auto *GV = dyn_cast<GlobalVariable>(Val: Ptr)) {
1322 if (GV->hasInitializer() || GV->isExternallyInitialized())
1323 return Error::success();
1324 Val = ZeroOrUndef(GV->getValueType());
1325 } else if (auto *AI = dyn_cast<AllocaInst>(Val: Ptr))
1326 Val = ZeroOrUndef(AI->getAllocatedType());
1327
1328 assert(Val && "Expected operand of lifetime intrinsic to be a global "
1329 "variable or alloca instruction");
1330 IRB.CreateStore(Val, Ptr, isVolatile: false);
1331
1332 CI->eraseFromParent();
1333 return Error::success();
1334 });
1335 }
1336
1337 [[nodiscard]] bool lowerIsFPClass(Function &F) {
1338 IRBuilder<> &IRB = OpBuilder.getIRB();
1339 Type *RetTy = IRB.getInt1Ty();
1340
1341 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1342 IRB.SetInsertPoint(CI);
1343 SmallVector<Value *> Args;
1344 Value *Fl = CI->getArgOperand(i: 0);
1345 Args.push_back(Elt: Fl);
1346
1347 dxil::OpCode OpCode;
1348 Value *T = CI->getArgOperand(i: 1);
1349 auto *TCI = dyn_cast<ConstantInt>(Val: T);
1350 switch (TCI->getZExtValue()) {
1351 case FPClassTest::fcInf:
1352 OpCode = dxil::OpCode::IsInf;
1353 break;
1354 case FPClassTest::fcNan:
1355 OpCode = dxil::OpCode::IsNaN;
1356 break;
1357 case FPClassTest::fcNormal:
1358 OpCode = dxil::OpCode::IsNormal;
1359 break;
1360 case FPClassTest::fcFinite:
1361 OpCode = dxil::OpCode::IsFinite;
1362 break;
1363 default:
1364 SmallString<128> Msg =
1365 formatv(Fmt: "Unsupported FPClassTest {0} for DXIL Op Lowering",
1366 Vals: TCI->getZExtValue());
1367 return make_error<StringError>(Args&: Msg, Args: inconvertibleErrorCode());
1368 }
1369
1370 Expected<CallInst *> OpCall =
1371 OpBuilder.tryCreateOp(Op: OpCode, Args, Name: CI->getName(), RetTy);
1372 if (Error E = OpCall.takeError())
1373 return E;
1374
1375 CI->replaceAllUsesWith(V: *OpCall);
1376 CI->eraseFromParent();
1377 return Error::success();
1378 });
1379 }
1380
1381 bool lowerIntrinsics() {
1382 bool Updated = false;
1383 bool HasErrors = false;
1384
1385 for (Function &F : make_early_inc_range(Range: M.functions())) {
1386 if (!F.isDeclaration())
1387 continue;
1388 Intrinsic::ID ID = F.getIntrinsicID();
1389 switch (ID) {
1390 // NOTE: Skip dx_resource_casthandle here. They are
1391 // resolved after this loop in cleanupHandleCasts.
1392 case Intrinsic::dx_resource_casthandle:
1393 // NOTE: llvm.dbg.value is supported as is in DXIL.
1394 case Intrinsic::dbg_value:
1395 case Intrinsic::not_intrinsic:
1396 if (F.use_empty())
1397 F.eraseFromParent();
1398 continue;
1399 default:
1400 if (F.use_empty())
1401 F.eraseFromParent();
1402 else {
1403 SmallString<128> Msg = formatv(
1404 Fmt: "Unsupported intrinsic {0} for DXIL lowering", Vals: F.getName());
1405 M.getContext().emitError(ErrorStr: Msg);
1406 HasErrors |= true;
1407 }
1408 break;
1409
1410#define DXIL_OP_INTRINSIC(OpCode, Intrin, ...) \
1411 case Intrin: \
1412 HasErrors |= replaceFunctionWithOp( \
1413 F, OpCode, ArrayRef<IntrinArgSelect>{__VA_ARGS__}); \
1414 break;
1415#include "DXILOperation.inc"
1416 case Intrinsic::dx_resource_handlefrombinding:
1417 HasErrors |= lowerHandleFromBinding(F);
1418 break;
1419 case Intrinsic::dx_resource_handlefromheap:
1420 HasErrors |= lowerHandleFromHeap(F);
1421 break;
1422 case Intrinsic::dx_resource_getbasepointer:
1423 case Intrinsic::dx_resource_getpointer:
1424 HasErrors |= lowerGetPointer(F);
1425 break;
1426 case Intrinsic::dx_resource_nonuniformindex:
1427 assert(!CleanupNURI &&
1428 "overloaded llvm.dx.resource.nonuniformindex intrinsics?");
1429 CleanupNURI = &F;
1430 break;
1431 case Intrinsic::dx_resource_load_typedbuffer:
1432 HasErrors |= lowerTypedBufferLoad(F, /*HasCheckBit=*/true);
1433 break;
1434 case Intrinsic::dx_resource_load_level:
1435 HasErrors |= lowerTextureLoad(F);
1436 break;
1437 case Intrinsic::dx_resource_sample:
1438 HasErrors |= lowerSample(F, /*HasClamp=*/false);
1439 break;
1440 case Intrinsic::dx_resource_sample_clamp:
1441 HasErrors |= lowerSample(F, /*HasClamp=*/true);
1442 break;
1443 case Intrinsic::dx_resource_samplebias:
1444 HasErrors |= lowerSampleBias(F, /*HasClamp=*/false);
1445 break;
1446 case Intrinsic::dx_resource_samplebias_clamp:
1447 HasErrors |= lowerSampleBias(F, /*HasClamp=*/true);
1448 break;
1449 case Intrinsic::dx_resource_samplelevel:
1450 HasErrors |= lowerSampleLevel(F);
1451 break;
1452 case Intrinsic::dx_resource_samplegrad:
1453 HasErrors |= lowerSampleGrad(F, /*HasClamp=*/false);
1454 break;
1455 case Intrinsic::dx_resource_samplegrad_clamp:
1456 HasErrors |= lowerSampleGrad(F, /*HasClamp=*/true);
1457 break;
1458 case Intrinsic::dx_resource_store_typedbuffer:
1459 HasErrors |= lowerBufferStore(F, /*IsRaw=*/false);
1460 break;
1461 case Intrinsic::dx_resource_store_texture:
1462 HasErrors |= lowerTextureStore(F);
1463 break;
1464 case Intrinsic::dx_resource_load_rawbuffer:
1465 HasErrors |= lowerRawBufferLoad(F);
1466 break;
1467 case Intrinsic::dx_resource_store_rawbuffer:
1468 HasErrors |= lowerBufferStore(F, /*IsRaw=*/true);
1469 break;
1470 case Intrinsic::dx_resource_load_cbufferrow_2:
1471 case Intrinsic::dx_resource_load_cbufferrow_4:
1472 case Intrinsic::dx_resource_load_cbufferrow_8:
1473 HasErrors |= lowerCBufferLoad(F);
1474 break;
1475 case Intrinsic::dx_resource_updatecounter:
1476 HasErrors |= lowerUpdateCounter(F);
1477 break;
1478 case Intrinsic::dx_barrier_by_memory_handle:
1479 HasErrors |= lowerBarrierByMemoryHandle(F);
1480 break;
1481 case Intrinsic::dx_resource_atomic_binop:
1482 HasErrors |= lowerResourceAtomicBinOp(F);
1483 break;
1484 case Intrinsic::dx_resource_atomic_compare_exchange:
1485 HasErrors |= lowerResourceAtomicCompareExchange(F);
1486 break;
1487 case Intrinsic::dx_resource_getdimensions_x:
1488 HasErrors |= lowerGetDimensionsX(F);
1489 break;
1490 case Intrinsic::ctpop:
1491 HasErrors |= lowerCtpopToCountBits(F);
1492 break;
1493 case Intrinsic::lifetime_start:
1494 case Intrinsic::lifetime_end:
1495 if (F.use_empty())
1496 F.eraseFromParent();
1497 else {
1498 if (MMDI.DXILVersion < VersionTuple(1, 6))
1499 HasErrors |= lowerLifetimeIntrinsic(F);
1500 else
1501 continue;
1502 }
1503 break;
1504 case Intrinsic::is_fpclass:
1505 HasErrors |= lowerIsFPClass(F);
1506 break;
1507 }
1508 Updated = true;
1509 }
1510 if (Updated && !HasErrors) {
1511 cleanupHandleCasts();
1512 cleanupNonUniformResourceIndexCalls();
1513 }
1514
1515 return Updated;
1516 }
1517};
1518} // namespace
1519
1520PreservedAnalyses DXILOpLowering::run(Module &M, ModuleAnalysisManager &MAM) {
1521 DXILResourceMap &DRM = MAM.getResult<DXILResourceAnalysis>(IR&: M);
1522 DXILResourceTypeMap &DRTM = MAM.getResult<DXILResourceTypeAnalysis>(IR&: M);
1523 const ModuleMetadataInfo MMDI = MAM.getResult<DXILMetadataAnalysis>(IR&: M);
1524
1525 const bool MadeChanges = OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
1526 if (!MadeChanges)
1527 return PreservedAnalyses::all();
1528 PreservedAnalyses PA;
1529 PA.preserve<DXILResourceAnalysis>();
1530 PA.preserve<DXILMetadataAnalysis>();
1531 PA.preserve<ShaderFlagsAnalysis>();
1532 PA.preserve<RootSignatureAnalysis>();
1533 return PA;
1534}
1535
1536namespace {
1537class DXILOpLoweringLegacy : public ModulePass {
1538public:
1539 bool runOnModule(Module &M) override {
1540 DXILResourceMap &DRM =
1541 getAnalysis<DXILResourceWrapperPass>().getResourceMap();
1542 DXILResourceTypeMap &DRTM =
1543 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
1544 const ModuleMetadataInfo MMDI =
1545 getAnalysis<DXILMetadataAnalysisWrapperPass>().getModuleMetadata();
1546
1547 return OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
1548 }
1549 StringRef getPassName() const override { return "DXIL Op Lowering"; }
1550 DXILOpLoweringLegacy() : ModulePass(ID) {}
1551
1552 static char ID; // Pass identification.
1553 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
1554 AU.addRequired<DXILResourceTypeWrapperPass>();
1555 AU.addRequired<DXILResourceWrapperPass>();
1556 AU.addRequired<DXILMetadataAnalysisWrapperPass>();
1557 AU.addPreserved<DXILResourceWrapperPass>();
1558 AU.addPreserved<DXILMetadataAnalysisWrapperPass>();
1559 AU.addPreserved<ShaderFlagsAnalysisWrapper>();
1560 AU.addPreserved<RootSignatureAnalysisWrapper>();
1561 }
1562};
1563char DXILOpLoweringLegacy::ID = 0;
1564} // end anonymous namespace
1565
1566INITIALIZE_PASS_BEGIN(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering",
1567 false, false)
1568INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
1569INITIALIZE_PASS_DEPENDENCY(DXILResourceWrapperPass)
1570INITIALIZE_PASS_END(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering", false,
1571 false)
1572
1573ModulePass *llvm::createDXILOpLoweringLegacyPass() {
1574 return new DXILOpLoweringLegacy();
1575}
1576