1//===- DXILOpLowering.cpp - Lowering to DXIL operations -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DXILOpLowering.h"
10#include "DXILConstants.h"
11#include "DXILOpBuilder.h"
12#include "DXILRootSignature.h"
13#include "DXILShaderFlags.h"
14#include "DirectX.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/Analysis/DXILMetadataAnalysis.h"
17#include "llvm/Analysis/DXILResource.h"
18#include "llvm/CodeGen/Passes.h"
19#include "llvm/IR/Constant.h"
20#include "llvm/IR/DiagnosticInfo.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsDirectX.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PassManager.h"
28#include "llvm/IR/Use.h"
29#include "llvm/IR/ValueHandle.h"
30#include "llvm/InitializePasses.h"
31#include "llvm/Pass.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/FormatVariadic.h"
34
35#define DEBUG_TYPE "dxil-op-lower"
36
37using namespace llvm;
38using namespace llvm::dxil;
39
40/// Write mask covering all four components of a UAV element. Typed UAV stores
41/// (textures and typed buffers) must always use this mask - the DXIL validator
42/// rejects anything narrower. Only raw and / structured buffer stores may use a
43/// partial mask.
44static constexpr uint8_t TypedUAVStoreWriteMask = 0xF;
45
46namespace {
47class OpLowerer {
48 Module &M;
49 DXILOpBuilder OpBuilder;
50 DXILResourceMap &DRM;
51 DXILResourceTypeMap &DRTM;
52 const ModuleMetadataInfo &MMDI;
53 SmallVector<CallInst *> CleanupCasts;
54 Function *CleanupNURI = nullptr;
55
56public:
57 OpLowerer(Module &M, DXILResourceMap &DRM, DXILResourceTypeMap &DRTM,
58 const ModuleMetadataInfo &MMDI)
59 : M(M), OpBuilder(M), DRM(DRM), DRTM(DRTM), MMDI(MMDI) {}
60
61 /// Replace every call to \c F using \c ReplaceCall, and then erase \c F. If
62 /// there is an error replacing a call, we emit a diagnostic and return true.
63 [[nodiscard]] bool
64 replaceFunction(Function &F,
65 llvm::function_ref<Error(CallInst *CI)> ReplaceCall) {
66 for (User *U : make_early_inc_range(Range: F.users())) {
67 CallInst *CI = dyn_cast<CallInst>(Val: U);
68 if (!CI)
69 continue;
70
71 if (Error E = ReplaceCall(CI)) {
72 std::string Message(toString(E: std::move(E)));
73 M.getContext().diagnose(DI: DiagnosticInfoUnsupported(
74 *CI->getFunction(), Message, CI->getDebugLoc()));
75
76 return true;
77 }
78 }
79 if (F.user_empty())
80 F.eraseFromParent();
81 return false;
82 }
83
84 struct IntrinArgSelect {
85 enum class Type {
86#define DXIL_OP_INTRINSIC_ARG_SELECT_TYPE(name) name,
87#include "DXILOperation.inc"
88 };
89 Type Type;
90 int Value;
91 };
92
93 /// Replaces uses of a struct with uses of an equivalent named struct.
94 ///
95 /// DXIL operations that return structs give them well known names, so we need
96 /// to update uses when we switch from an LLVM intrinsic to an op.
97 Error replaceNamedStructUses(CallInst *Intrin, CallInst *DXILOp) {
98 auto *IntrinTy = cast<StructType>(Val: Intrin->getType());
99 auto *DXILOpTy = cast<StructType>(Val: DXILOp->getType());
100 if (!IntrinTy->isLayoutIdentical(Other: DXILOpTy))
101 return make_error<StringError>(
102 Args: "Type mismatch between intrinsic and DXIL op",
103 Args: inconvertibleErrorCode());
104
105 for (Use &U : make_early_inc_range(Range: Intrin->uses()))
106 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: U.getUser()))
107 EVI->setOperand(i_nocapture: 0, Val_nocapture: DXILOp);
108 else if (auto *IVI = dyn_cast<InsertValueInst>(Val: U.getUser()))
109 IVI->setOperand(i_nocapture: 0, Val_nocapture: DXILOp);
110 else
111 return make_error<StringError>(Args: "DXIL ops that return structs may only "
112 "be used by insert- and extractvalue",
113 Args: inconvertibleErrorCode());
114 return Error::success();
115 }
116
117 bool isFast(FastMathFlags Flags) {
118 // HLSL Fast Math doesn't enable AllowContract flag; This can be
119 // removed when we enable it in the future.
120 return Flags.allowReassoc() && Flags.noNaNs() && Flags.noInfs() &&
121 Flags.noSignedZeros() && Flags.allowReciprocal() &&
122 Flags.approxFunc();
123 }
124
125 void setDxPrecise(CallInst *CI) {
126 const StringRef Key = "dx.precise";
127 Module *M = CI->getModule();
128
129 LLVMContext &Ctx = M->getContext();
130 MDNode *One =
131 llvm::MDNode::get(Context&: Ctx, MDs: ConstantAsMetadata::get(C: ConstantInt::get(
132 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1)));
133
134 CI->setMetadata(Kind: Key, Node: One);
135 }
136
137 [[nodiscard]] bool
138 replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp,
139 ArrayRef<IntrinArgSelect> ArgSelects) {
140 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
141 OpBuilder.getIRB().SetInsertPoint(CI);
142 SmallVector<Value *> Args;
143 if (ArgSelects.size()) {
144 for (const IntrinArgSelect &A : ArgSelects) {
145 switch (A.Type) {
146 case IntrinArgSelect::Type::Index:
147 Args.push_back(Elt: CI->getArgOperand(i: A.Value));
148 break;
149 case IntrinArgSelect::Type::I8:
150 Args.push_back(Elt: OpBuilder.getIRB().getInt8(C: (uint8_t)A.Value));
151 break;
152 case IntrinArgSelect::Type::I32:
153 Args.push_back(Elt: OpBuilder.getIRB().getInt32(C: A.Value));
154 break;
155 }
156 }
157 } else {
158 Args.append(in_start: CI->arg_begin(), in_end: CI->arg_end());
159 }
160
161 Expected<CallInst *> OpCall =
162 OpBuilder.tryCreateOp(Op: DXILOp, Args, Name: CI->getName(), RetTy: F.getReturnType());
163 if (Error E = OpCall.takeError())
164 return E;
165
166 if (isa<FPMathOperator>(Val: CI) &&
167 !isFast(Flags: cast<FPMathOperator>(Val: CI)->getFastMathFlags()))
168 setDxPrecise(*OpCall);
169
170 if (isa<StructType>(Val: CI->getType())) {
171 if (Error E = replaceNamedStructUses(Intrin: CI, DXILOp: *OpCall))
172 return E;
173 } else
174 CI->replaceAllUsesWith(V: *OpCall);
175
176 CI->eraseFromParent();
177 return Error::success();
178 });
179 }
180
181 /// Create a cast between a `target("dx")` type and `dx.types.Handle`, which
182 /// is intended to be removed by the end of lowering. This is used to allow
183 /// lowering of ops which need to change their return or argument types in a
184 /// piecemeal way - we can add the casts in to avoid updating all of the uses
185 /// or defs, and by the end all of the casts will be redundant.
186 Value *createTmpHandleCast(Value *V, Type *Ty) {
187 CallInst *Cast = OpBuilder.getIRB().CreateIntrinsicWithoutFolding(
188 ID: Intrinsic::dx_resource_casthandle, OverloadTypes: {Ty, V->getType()}, Args: {V});
189 CleanupCasts.push_back(Elt: Cast);
190 return Cast;
191 }
192
193 void cleanupHandleCasts() {
194 SmallVector<CallInst *> ToRemove;
195 SmallVector<Function *> CastFns;
196
197 for (CallInst *Cast : CleanupCasts) {
198 // These casts were only put in to ease the move from `target("dx")` types
199 // to `dx.types.Handle in a piecemeal way. At this point, all of the
200 // non-cast uses should now be `dx.types.Handle`, and remaining casts
201 // should all form pairs to and from the now unused `target("dx")` type.
202 CastFns.push_back(Elt: Cast->getCalledFunction());
203
204 // If the cast is not to `dx.types.Handle`, it should be the first part of
205 // the pair. Keep track so we can remove it once it has no more uses.
206 if (Cast->getType() != OpBuilder.getHandleType()) {
207 ToRemove.push_back(Elt: Cast);
208 continue;
209 }
210 // Otherwise, we're the second handle in a pair. Forward the arguments and
211 // remove the (second) cast.
212 CallInst *Def = cast<CallInst>(Val: Cast->getOperand(i_nocapture: 0));
213 assert(Def->getIntrinsicID() == Intrinsic::dx_resource_casthandle &&
214 "Unbalanced pair of temporary handle casts");
215 Cast->replaceAllUsesWith(V: Def->getOperand(i_nocapture: 0));
216 Cast->eraseFromParent();
217 }
218 for (CallInst *Cast : ToRemove) {
219 assert(Cast->user_empty() && "Temporary handle cast still has users");
220 Cast->eraseFromParent();
221 }
222
223 // Deduplicate the cast functions so that we only erase each one once.
224 llvm::sort(C&: CastFns);
225 CastFns.erase(CS: llvm::unique(R&: CastFns), CE: CastFns.end());
226 for (Function *F : CastFns)
227 F->eraseFromParent();
228
229 CleanupCasts.clear();
230 }
231
232 void cleanupNonUniformResourceIndexCalls() {
233 // Replace all NonUniformResourceIndex calls with their argument.
234 if (!CleanupNURI)
235 return;
236 for (User *U : make_early_inc_range(Range: CleanupNURI->users())) {
237 CallInst *CI = dyn_cast<CallInst>(Val: U);
238 if (!CI)
239 continue;
240 CI->replaceAllUsesWith(V: CI->getArgOperand(i: 0));
241 CI->eraseFromParent();
242 }
243 CleanupNURI->eraseFromParent();
244 CleanupNURI = nullptr;
245 }
246
247 // Remove the resource global associated with the handleFromBinding call
248 // instruction and their uses as they aren't needed anymore.
249 // TODO: We should verify that all the globals get removed.
250 // It's expected we'll need a custom pass in the future that will eliminate
251 // the need for this here.
252 void removeResourceGlobals(CallInst *CI) {
253 for (User *User : make_early_inc_range(Range: CI->users())) {
254 if (StoreInst *Store = dyn_cast<StoreInst>(Val: User)) {
255 Value *V = Store->getOperand(i_nocapture: 1);
256 Store->eraseFromParent();
257 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: V))
258 if (GV->use_empty()) {
259 GV->removeDeadConstantUsers();
260 GV->eraseFromParent();
261 }
262 }
263 }
264 }
265
266 void replaceHandleFromBindingCall(CallInst *CI, Value *Replacement) {
267 assert(CI->getCalledFunction()->getIntrinsicID() ==
268 Intrinsic::dx_resource_handlefrombinding);
269
270 removeResourceGlobals(CI);
271
272 auto *NameGlobal = dyn_cast<llvm::GlobalVariable>(Val: CI->getArgOperand(i: 4));
273
274 CI->replaceAllUsesWith(V: Replacement);
275 CI->eraseFromParent();
276
277 if (NameGlobal && NameGlobal->use_empty())
278 NameGlobal->eraseFromParent();
279 }
280
281 bool hasNonUniformIndex(Value *IndexOp) {
282 if (isa<llvm::Constant>(Val: IndexOp))
283 return false;
284
285 SmallVector<Value *, 16> Worklist;
286 SmallPtrSet<Value *, 16> Visited;
287 Worklist.push_back(Elt: IndexOp);
288
289 while (!Worklist.empty()) {
290 Value *V = Worklist.pop_back_val();
291
292 if (isa<llvm::Constant>(Val: V))
293 continue;
294
295 if (!Visited.insert(Ptr: V).second)
296 continue;
297
298 if (auto *CI = dyn_cast<CallInst>(Val: V))
299 if (CI->getIntrinsicID() == Intrinsic::dx_resource_nonuniformindex)
300 return true;
301
302 // If it's a PHI node, check ALL incoming values —
303 // taint from ANY predecessor counts
304 if (auto *Phi = dyn_cast<PHINode>(Val: V)) {
305 for (Value *Incoming : Phi->incoming_values())
306 Worklist.push_back(Elt: Incoming);
307 continue;
308 }
309
310 if (auto *Inst = dyn_cast<Instruction>(Val: V))
311 if (Inst->getNumOperands() > 0 && !Inst->isTerminator())
312 for (Value *Op : Inst->operands())
313 Worklist.push_back(Elt: Op);
314 }
315 return false;
316 }
317
318 Error validateRawBufferElementIndex(Value *Resource, Value *ElementIndex) {
319 bool IsStructured =
320 cast<RawBufferExtType>(Val: Resource->getType())->isStructured();
321 bool IsPoison = isa<PoisonValue>(Val: ElementIndex);
322
323 if (IsStructured && IsPoison)
324 return make_error<StringError>(
325 Args: "Element index of structured buffer may not be poison",
326 Args: inconvertibleErrorCode());
327
328 if (!IsStructured && !IsPoison)
329 return make_error<StringError>(
330 Args: "Element index of raw buffer must be poison",
331 Args: inconvertibleErrorCode());
332
333 return Error::success();
334 }
335
336 [[nodiscard]] bool lowerToCreateHandle(Function &F) {
337 IRBuilder<> &IRB = OpBuilder.getIRB();
338 Type *Int8Ty = IRB.getInt8Ty();
339 Type *Int32Ty = IRB.getInt32Ty();
340 Type *Int1Ty = IRB.getInt1Ty();
341
342 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
343 IRB.SetInsertPoint(CI);
344
345 auto *It = DRM.find(Key: CI);
346 assert(It != DRM.end() && "Resource not in map?");
347 dxil::ResourceInfo &RI = *It;
348
349 const auto &Binding = RI.getBinding();
350 dxil::ResourceClass RC = DRTM[RI.getHandleTy()].getResourceClass();
351
352 Value *IndexOp = CI->getArgOperand(i: 3);
353 if (Binding.LowerBound != 0)
354 IndexOp = IRB.CreateAdd(LHS: IndexOp,
355 RHS: ConstantInt::get(Ty: Int32Ty, V: Binding.LowerBound));
356
357 bool HasNonUniformIndex =
358 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);
359 std::array<Value *, 4> Args{
360 ConstantInt::get(Ty: Int8Ty, V: llvm::to_underlying(E: RC)),
361 ConstantInt::get(Ty: Int32Ty, V: Binding.BindingID), IndexOp,
362 ConstantInt::get(Ty: Int1Ty, V: HasNonUniformIndex)};
363 Expected<CallInst *> OpCall =
364 OpBuilder.tryCreateOp(Op: OpCode::CreateHandle, Args, Name: CI->getName());
365 if (Error E = OpCall.takeError())
366 return E;
367
368 Value *Cast = createTmpHandleCast(V: *OpCall, Ty: CI->getType());
369 replaceHandleFromBindingCall(CI, Replacement: Cast);
370 return Error::success();
371 });
372 }
373
374 [[nodiscard]] bool lowerToBindAndAnnotateHandle(Function &F) {
375 IRBuilder<> &IRB = OpBuilder.getIRB();
376 Type *Int32Ty = IRB.getInt32Ty();
377 Type *Int1Ty = IRB.getInt1Ty();
378
379 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
380 IRB.SetInsertPoint(CI);
381
382 auto *It = DRM.find(Key: CI);
383 assert(It != DRM.end() && "Resource not in map?");
384 dxil::ResourceInfo &RI = *It;
385
386 const auto &Binding = RI.getBinding();
387 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];
388 dxil::ResourceClass RC = RTI.getResourceClass();
389
390 Value *IndexOp = CI->getArgOperand(i: 3);
391 if (Binding.LowerBound != 0)
392 IndexOp = IRB.CreateAdd(LHS: IndexOp,
393 RHS: ConstantInt::get(Ty: Int32Ty, V: Binding.LowerBound));
394
395 std::pair<uint32_t, uint32_t> Props =
396 RI.getAnnotateProps(M&: *F.getParent(), RTI);
397
398 // For `CreateHandleFromBinding` we need the upper bound rather than the
399 // size, so we need to be careful about the difference for "unbounded".
400 uint32_t UpperBound = Binding.Size == 0
401 ? std::numeric_limits<uint32_t>::max()
402 : Binding.LowerBound + Binding.Size - 1;
403 Constant *ResBind = OpBuilder.getResBind(LowerBound: Binding.LowerBound, UpperBound,
404 SpaceID: Binding.Space, RC);
405 bool NonUniformIndex =
406 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);
407 Constant *NonUniformOp = ConstantInt::get(Ty: Int1Ty, V: NonUniformIndex);
408 std::array<Value *, 3> BindArgs{ResBind, IndexOp, NonUniformOp};
409 Expected<CallInst *> OpBind = OpBuilder.tryCreateOp(
410 Op: OpCode::CreateHandleFromBinding, Args: BindArgs, Name: CI->getName());
411 if (Error E = OpBind.takeError())
412 return E;
413
414 std::array<Value *, 2> AnnotateArgs{
415 *OpBind, OpBuilder.getResProps(Word0: Props.first, Word1: Props.second)};
416 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(
417 Op: OpCode::AnnotateHandle, Args: AnnotateArgs,
418 Name: CI->hasName() ? CI->getName() + "_annot" : Twine());
419 if (Error E = OpAnnotate.takeError())
420 return E;
421
422 Value *Cast = createTmpHandleCast(V: *OpAnnotate, Ty: CI->getType());
423 replaceHandleFromBindingCall(CI, Replacement: Cast);
424 return Error::success();
425 });
426 }
427
428 /// Lower `dx.resource.handlefrombinding` intrinsics depending on the shader
429 /// model and taking into account binding information from
430 /// DXILResourceAnalysis.
431 bool lowerHandleFromBinding(Function &F) {
432 if (MMDI.DXILVersion < VersionTuple(1, 6))
433 return lowerToCreateHandle(F);
434 return lowerToBindAndAnnotateHandle(F);
435 }
436
437 bool lowerHandleFromHeap(Function &F) {
438 IRBuilder<> &IRB = OpBuilder.getIRB();
439
440 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
441 IRB.SetInsertPoint(CI);
442
443 auto *It = DRM.find(Key: CI);
444 assert(It != DRM.end() && "Resource not in map?");
445 dxil::ResourceInfo &RI = *It;
446 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];
447
448 Value *IndexOp = CI->getArgOperand(i: 0);
449 Value *IsSamplerHeap =
450 ConstantInt::getBool(Context&: IRB.getContext(), V: RTI.isSampler());
451
452 std::pair<uint32_t, uint32_t> Props =
453 RI.getAnnotateProps(M&: *F.getParent(), RTI);
454
455 bool NonUniformIndex = hasNonUniformIndex(IndexOp);
456 Value *NonUniformOp =
457 ConstantInt::getBool(Context&: IRB.getContext(), V: NonUniformIndex);
458
459 std::array<Value *, 3> Args{IndexOp, IsSamplerHeap, NonUniformOp};
460 Expected<CallInst *> OpCreateHandle = OpBuilder.tryCreateOp(
461 Op: OpCode::CreateHandleFromHeap, Args, Name: CI->getName());
462 if (Error E = OpCreateHandle.takeError())
463 return E;
464
465 std::array<Value *, 2> AnnotateArgs{
466 *OpCreateHandle, OpBuilder.getResProps(Word0: Props.first, Word1: Props.second)};
467 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(
468 Op: OpCode::AnnotateHandle, Args: AnnotateArgs,
469 Name: CI->hasName() ? CI->getName() + "_annot" : Twine());
470 if (Error E = OpAnnotate.takeError())
471 return E;
472
473 Value *Cast = createTmpHandleCast(V: *OpAnnotate, Ty: CI->getType());
474 CI->replaceAllUsesWith(V: Cast);
475 CI->eraseFromParent();
476 return Error::success();
477 });
478 }
479
480 /// Replace uses of \c Intrin with the values in the `dx.ResRet` of \c Op.
481 /// Since we expect to be post-scalarization, make an effort to avoid vectors.
482 Error replaceResRetUses(CallInst *Intrin, CallInst *Op, bool HasCheckBit) {
483 IRBuilder<> &IRB = OpBuilder.getIRB();
484
485 Instruction *OldResult = Intrin;
486 Type *OldTy = Intrin->getType();
487
488 if (HasCheckBit) {
489 auto *ST = cast<StructType>(Val: OldTy);
490
491 Value *CheckOp = nullptr;
492 Type *Int32Ty = IRB.getInt32Ty();
493 for (Use &U : make_early_inc_range(Range: OldResult->uses())) {
494 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: U.getUser())) {
495 ArrayRef<unsigned> Indices = EVI->getIndices();
496 assert(Indices.size() == 1);
497 // We're only interested in uses of the check bit for now.
498 if (Indices[0] != 1)
499 continue;
500 if (!CheckOp) {
501 Value *NewEVI = IRB.CreateExtractValue(Agg: Op, Idxs: 4);
502 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
503 Op: OpCode::CheckAccessFullyMapped, Args: {NewEVI},
504 Name: OldResult->hasName() ? OldResult->getName() + "_check"
505 : Twine(),
506 RetTy: Int32Ty);
507 if (Error E = OpCall.takeError())
508 return E;
509 CheckOp = *OpCall;
510 }
511 EVI->replaceAllUsesWith(V: CheckOp);
512 EVI->eraseFromParent();
513 }
514 }
515
516 if (OldResult->use_empty()) {
517 // Only the check bit was used, so we're done here.
518 OldResult->eraseFromParent();
519 return Error::success();
520 }
521
522 assert(OldResult->hasOneUse() &&
523 isa<ExtractValueInst>(*OldResult->user_begin()) &&
524 "Expected only use to be extract of first element");
525 OldResult = cast<Instruction>(Val: *OldResult->user_begin());
526 OldTy = ST->getElementType(N: 0);
527 }
528
529 // For scalars, we just extract the first element.
530 if (!isa<FixedVectorType>(Val: OldTy)) {
531 Value *EVI = IRB.CreateExtractValue(Agg: Op, Idxs: 0);
532 OldResult->replaceAllUsesWith(V: EVI);
533 OldResult->eraseFromParent();
534 if (OldResult != Intrin) {
535 assert(Intrin->use_empty() && "Intrinsic still has uses?");
536 Intrin->eraseFromParent();
537 }
538 return Error::success();
539 }
540
541 std::array<Value *, 4> Extracts = {};
542 SmallVector<ExtractElementInst *> DynamicAccesses;
543
544 // The users of the operation should all be scalarized, so we attempt to
545 // replace the extractelements with extractvalues directly.
546 for (Use &U : make_early_inc_range(Range: OldResult->uses())) {
547 if (auto *EEI = dyn_cast<ExtractElementInst>(Val: U.getUser())) {
548 if (auto *IndexOp = dyn_cast<ConstantInt>(Val: EEI->getIndexOperand())) {
549 size_t IndexVal = IndexOp->getZExtValue();
550 assert(IndexVal < 4 && "Index into buffer load out of range");
551 if (!Extracts[IndexVal])
552 Extracts[IndexVal] = IRB.CreateExtractValue(Agg: Op, Idxs: IndexVal);
553 EEI->replaceAllUsesWith(V: Extracts[IndexVal]);
554 EEI->eraseFromParent();
555 } else {
556 DynamicAccesses.push_back(Elt: EEI);
557 }
558 }
559 }
560
561 const auto *VecTy = cast<FixedVectorType>(Val: OldTy);
562 const unsigned N = VecTy->getNumElements();
563
564 // If there's a dynamic access we need to round trip through stack memory so
565 // that we don't leave vectors around.
566 if (!DynamicAccesses.empty()) {
567 Type *Int32Ty = IRB.getInt32Ty();
568 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
569
570 Type *ElTy = VecTy->getElementType();
571 Type *ArrayTy = ArrayType::get(ElementType: ElTy, NumElements: N);
572 Value *Alloca = IRB.CreateAlloca(Ty: ArrayTy);
573
574 for (int I = 0, E = N; I != E; ++I) {
575 if (!Extracts[I])
576 Extracts[I] = IRB.CreateExtractValue(Agg: Op, Idxs: I);
577 Value *GEP = IRB.CreateInBoundsGEP(
578 Ty: ArrayTy, Ptr: Alloca, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: I)});
579 IRB.CreateStore(Val: Extracts[I], Ptr: GEP);
580 }
581
582 for (ExtractElementInst *EEI : DynamicAccesses) {
583 Value *GEP = IRB.CreateInBoundsGEP(Ty: ArrayTy, Ptr: Alloca,
584 IdxList: {Zero, EEI->getIndexOperand()});
585 Value *Load = IRB.CreateLoad(Ty: ElTy, Ptr: GEP);
586 EEI->replaceAllUsesWith(V: Load);
587 EEI->eraseFromParent();
588 }
589 }
590
591 // If we still have uses, then we're not fully scalarized and need to
592 // recreate the vector. This should only happen for things like exported
593 // functions from libraries.
594 if (!OldResult->use_empty()) {
595 for (int I = 0, E = N; I != E; ++I)
596 if (!Extracts[I])
597 Extracts[I] = IRB.CreateExtractValue(Agg: Op, Idxs: I);
598
599 Value *Vec = PoisonValue::get(T: OldTy);
600 for (int I = 0, E = N; I != E; ++I)
601 Vec = IRB.CreateInsertElement(Vec, NewElt: Extracts[I], Idx: I);
602 OldResult->replaceAllUsesWith(V: Vec);
603 }
604
605 OldResult->eraseFromParent();
606 if (OldResult != Intrin) {
607 assert(Intrin->use_empty() && "Intrinsic still has uses?");
608 Intrin->eraseFromParent();
609 }
610
611 return Error::success();
612 }
613
614 [[nodiscard]] bool lowerTypedBufferLoad(Function &F, bool HasCheckBit) {
615 IRBuilder<> &IRB = OpBuilder.getIRB();
616 Type *Int32Ty = IRB.getInt32Ty();
617
618 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
619 IRB.SetInsertPoint(CI);
620
621 Value *Handle =
622 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
623 Value *Index0 = CI->getArgOperand(i: 1);
624 Value *Index1 = UndefValue::get(T: Int32Ty);
625
626 Type *OldTy = CI->getType();
627 if (HasCheckBit)
628 OldTy = cast<StructType>(Val: OldTy)->getElementType(N: 0);
629 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
630
631 std::array<Value *, 3> Args{Handle, Index0, Index1};
632 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
633 Op: OpCode::BufferLoad, Args, Name: CI->getName(), RetTy: NewRetTy);
634 if (Error E = OpCall.takeError())
635 return E;
636 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, HasCheckBit))
637 return E;
638
639 return Error::success();
640 });
641 }
642
643 /// Recover the scalar components of `Vec` from the `insertelement` chain that
644 /// built it. Since we run after the scalarizer, such a chain is usually just
645 /// a temporary gathered to pass the vector to a call.
646 static void collectInsertedElements(Value *Vec,
647 MutableArrayRef<Value *> Elements) {
648 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
649 assert(NumElts <= Elements.size() && "Not enough room for the components");
650
651 SmallVector<InsertElementInst *, 4> Chain;
652 for (auto *IEI = dyn_cast<InsertElementInst>(Val: Vec); IEI;
653 IEI = dyn_cast<InsertElementInst>(Val: IEI->getOperand(i_nocapture: 0))) {
654 if (!isa<ConstantInt>(Val: IEI->getOperand(i_nocapture: 2)))
655 break; // This break should never happen below SM6.9.
656 Chain.push_back(Elt: IEI);
657 }
658
659 // Replay element insertion from the innermost first, so that a repeated
660 // index ends up holding the live value.
661 while (!Chain.empty()) {
662 InsertElementInst *IEI = Chain.pop_back_val();
663 uint64_t IndexVal = cast<ConstantInt>(Val: IEI->getOperand(i_nocapture: 2))->getZExtValue();
664 if (IndexVal < NumElts)
665 Elements[IndexVal] = IEI->getOperand(i_nocapture: 1);
666 }
667 }
668
669 // Copies `Src` into `Args` starting at `ArgIdx`. If `Src` is a vector, its
670 // elements are placed in consecutive slots; otherwise `Src` is stored
671 // directly. At most `MaxElements` elements are expected.
672 static void extractElementsIntoArgs(IRBuilder<> &IRB,
673 MutableArrayRef<Value *> Args,
674 unsigned ArgIdx, Value *Src,
675 unsigned MaxElements) {
676 auto *VecTy = dyn_cast<FixedVectorType>(Val: Src->getType());
677 if (!VecTy) {
678 Args[ArgIdx] = Src;
679 return;
680 }
681
682 unsigned Count = VecTy->getNumElements();
683 assert(Count <= MaxElements && "Too many elements for the arg list");
684
685 SmallVector<Value *, 4> Elements(Count, nullptr);
686 collectInsertedElements(Vec: Src, Elements);
687
688 for (unsigned I = 0; I < Count; ++I)
689 Args[ArgIdx + I] = Elements[I]
690 ? Elements[I]
691 : IRB.CreateExtractElement(
692 Vec: Src, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: I));
693 }
694
695 /// Copy offsets into the argument list at the given index, unless
696 /// the offsets are known to be zero (i.e., a null constant).
697 static void extractNonZeroOffsets(IRBuilder<> &IRB,
698 MutableArrayRef<Value *> Args,
699 unsigned ArgIdx, Value *Offsets,
700 unsigned MaxElements) {
701 auto *COff = dyn_cast<Constant>(Val: Offsets);
702 bool OffsetsAreZero = COff && COff->isNullValue();
703 if (!OffsetsAreZero)
704 extractElementsIntoArgs(IRB, Args, ArgIdx, Src: Offsets, MaxElements);
705 }
706
707 [[nodiscard]] bool lowerTextureLoad(Function &F) {
708 IRBuilder<> &IRB = OpBuilder.getIRB();
709 Type *Int32Ty = IRB.getInt32Ty();
710
711 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
712 IRB.SetInsertPoint(CI);
713
714 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
715 Value *Handle =
716 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
717 Value *Coords = CI->getArgOperand(i: 1);
718 Value *MipLevel = CI->getArgOperand(i: 2);
719 Value *Offsets = CI->getArgOperand(i: 3);
720
721 // A UAV descriptor binds a single mip slice, so there is no mip to select
722 // in the case of a UAV. Multisampled UAVs are the exception: the slot
723 // carries a sample index and stays live.
724 auto *HandleTy = cast<TargetExtType>(Val: CI->getArgOperand(i: 0)->getType());
725 dxil::ResourceTypeInfo &RTI = DRTM[HandleTy];
726 dxil::ResourceKind Kind = RTI.getResourceKind();
727 if (RTI.isUAV() && Kind != dxil::ResourceKind::Texture2DMS &&
728 Kind != dxil::ResourceKind::Texture2DMSArray)
729 MipLevel = UndefValue::get(T: Int32Ty);
730
731 Type *OldTy = CI->getType();
732 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
733
734 Value *Undef = UndefValue::get(T: Int32Ty);
735 std::array<Value *, 8> Args{Handle, MipLevel, Undef, Undef,
736 Undef, Undef, Undef, Undef};
737
738 // Copy coordinates and offsets into Args.
739 extractElementsIntoArgs(IRB, Args, ArgIdx: 2, Src: Coords, MaxElements: 3);
740 extractNonZeroOffsets(IRB, Args, ArgIdx: 5, Offsets, MaxElements: 3);
741
742 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
743 Op: OpCode::TextureLoad, Args, Name: CI->getName(), RetTy: NewRetTy);
744 if (Error E = OpCall.takeError())
745 return E;
746 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/false))
747 return E;
748
749 eraseDeadInsertElementChains(Vectors: VectorArgs);
750
751 return Error::success();
752 });
753 }
754
755 /// Common helper for lowering sample operations (SampleBias, SampleGrad,
756 /// etc.) that share the same pattern: extract handle/sampler, unpack
757 /// coordinates and offsets, build the DXIL arg list, and replace uses.
758 [[nodiscard]] bool lowerSampleOp(
759 Function &F, OpCode Op, unsigned CoordsIdx, unsigned OffsetsIdx,
760 llvm::function_ref<void(IRBuilder<> &, CallInst *,
761 SmallVectorImpl<Value *> &)> EmitExtraArgs) {
762 IRBuilder<> &IRB = OpBuilder.getIRB();
763 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
764 IRB.SetInsertPoint(CI);
765
766 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
767 Value *Handle =
768 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
769 Value *Sampler =
770 createTmpHandleCast(V: CI->getArgOperand(i: 1), Ty: OpBuilder.getHandleType());
771 Value *Coords = CI->getArgOperand(i: CoordsIdx);
772 Value *Offsets = CI->getArgOperand(i: OffsetsIdx);
773
774 Type *OldTy = CI->getType();
775 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: OldTy->getScalarType());
776
777 Value *UndefF = UndefValue::get(T: IRB.getFloatTy());
778 Value *UndefI = UndefValue::get(T: IRB.getInt32Ty());
779 // Common prefix: Handle, Sampler, Coord0..3, Offset0..2
780 SmallVector<Value *, 17> Args{Handle, Sampler, UndefF, UndefF, UndefF,
781 UndefF, UndefI, UndefI, UndefI};
782
783 // Copy coordinates and offsets into Args.
784 extractElementsIntoArgs(IRB, Args, ArgIdx: 2, Src: Coords, MaxElements: 4);
785 extractNonZeroOffsets(IRB, Args, ArgIdx: 6, Offsets, MaxElements: 3);
786
787 // Emit op-specific trailing arguments (e.g. Bias+Clamp, DDX+DDY+Clamp).
788 EmitExtraArgs(IRB, CI, Args);
789
790 Expected<CallInst *> OpCall =
791 OpBuilder.tryCreateOp(Op, Args, Name: CI->getName(), RetTy: NewRetTy);
792 if (Error E = OpCall.takeError())
793 return E;
794 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/false))
795 return E;
796
797 eraseDeadInsertElementChains(Vectors: VectorArgs);
798
799 return Error::success();
800 });
801 }
802
803 [[nodiscard]] bool lowerSample(Function &F, bool HasClamp) {
804 return lowerSampleOp(F, Op: OpCode::Sample, /*CoordsIdx=*/2, /*OffsetsIdx=*/3,
805 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
806 SmallVectorImpl<Value *> &Args) {
807 // Clamp
808 Args.push_back(
809 Elt: HasClamp ? CI->getArgOperand(i: 4)
810 : UndefValue::get(T: IRB.getFloatTy()));
811 });
812 }
813
814 [[nodiscard]] bool lowerSampleBias(Function &F, bool HasClamp) {
815 return lowerSampleOp(
816 F, Op: OpCode::SampleBias, /*CoordsIdx=*/2, /*OffsetsIdx=*/4,
817 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
818 SmallVectorImpl<Value *> &Args) {
819 // Bias is operand 3.
820 Args.push_back(Elt: CI->getArgOperand(i: 3));
821 // Clamp
822 Args.push_back(Elt: HasClamp ? CI->getArgOperand(i: 5)
823 : UndefValue::get(T: IRB.getFloatTy()));
824 });
825 }
826
827 [[nodiscard]] bool lowerSampleLevel(Function &F) {
828 return lowerSampleOp(
829 F, Op: OpCode::SampleLevel, /*CoordsIdx=*/2, /*OffsetsIdx=*/4,
830 EmitExtraArgs: [](IRBuilder<> &, CallInst *CI, SmallVectorImpl<Value *> &Args) {
831 // LOD is operand 3.
832 Args.push_back(Elt: CI->getArgOperand(i: 3));
833 });
834 }
835
836 [[nodiscard]] bool lowerSampleGrad(Function &F, bool HasClamp) {
837 return lowerSampleOp(
838 F, Op: OpCode::SampleGrad, /*CoordsIdx=*/2, /*OffsetsIdx=*/5,
839 EmitExtraArgs: [HasClamp](IRBuilder<> &IRB, CallInst *CI,
840 SmallVectorImpl<Value *> &Args) {
841 Value *DDX = CI->getArgOperand(i: 3);
842 Value *DDY = CI->getArgOperand(i: 4);
843 Value *UndefF = UndefValue::get(T: IRB.getFloatTy());
844 // DDX0..2
845 size_t DDXStart = Args.size();
846 Args.append(NumInputs: 3, Elt: UndefF);
847 extractElementsIntoArgs(IRB, Args, ArgIdx: DDXStart, Src: DDX, MaxElements: 3);
848 // DDY0..2
849 size_t DDYStart = Args.size();
850 Args.append(NumInputs: 3, Elt: UndefF);
851 extractElementsIntoArgs(IRB, Args, ArgIdx: DDYStart, Src: DDY, MaxElements: 3);
852 // Clamp
853 Args.push_back(Elt: HasClamp ? CI->getArgOperand(i: 6) : UndefF);
854 });
855 }
856
857 [[nodiscard]] bool lowerRawBufferLoad(Function &F) {
858 const DataLayout &DL = F.getDataLayout();
859 IRBuilder<> &IRB = OpBuilder.getIRB();
860 Type *Int8Ty = IRB.getInt8Ty();
861 Type *Int32Ty = IRB.getInt32Ty();
862
863 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
864 IRB.SetInsertPoint(CI);
865
866 Type *OldTy = cast<StructType>(Val: CI->getType())->getElementType(N: 0);
867 Type *ScalarTy = OldTy->getScalarType();
868 Type *NewRetTy = OpBuilder.getResRetType(ElementTy: ScalarTy);
869
870 Value *Handle =
871 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
872 Value *Index0 = CI->getArgOperand(i: 1);
873 Value *Index1 = CI->getArgOperand(i: 2);
874 uint64_t NumElements =
875 DL.getTypeSizeInBits(Ty: OldTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
876 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: ~(~0U << NumElements));
877 Value *Align =
878 ConstantInt::get(Ty: Int32Ty, V: DL.getPrefTypeAlign(Ty: ScalarTy).value());
879
880 if (Error E = validateRawBufferElementIndex(Resource: CI->getOperand(i_nocapture: 0), ElementIndex: Index1))
881 return E;
882 if (isa<PoisonValue>(Val: Index1))
883 Index1 = UndefValue::get(T: Index1->getType());
884
885 Expected<CallInst *> OpCall =
886 MMDI.DXILVersion >= VersionTuple(1, 2)
887 ? OpBuilder.tryCreateOp(Op: OpCode::RawBufferLoad,
888 Args: {Handle, Index0, Index1, Mask, Align},
889 Name: CI->getName(), RetTy: NewRetTy)
890 : OpBuilder.tryCreateOp(Op: OpCode::BufferLoad,
891 Args: {Handle, Index0, Index1}, Name: CI->getName(),
892 RetTy: NewRetTy);
893 if (Error E = OpCall.takeError())
894 return E;
895 if (Error E = replaceResRetUses(Intrin: CI, Op: *OpCall, /*HasCheckBit=*/true))
896 return E;
897
898 return Error::success();
899 });
900 }
901
902 [[nodiscard]] bool lowerCBufferLoad(Function &F) {
903 IRBuilder<> &IRB = OpBuilder.getIRB();
904
905 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
906 IRB.SetInsertPoint(CI);
907
908 Type *OldTy = cast<StructType>(Val: CI->getType())->getElementType(N: 0);
909 Type *ScalarTy = OldTy->getScalarType();
910 Type *NewRetTy = OpBuilder.getCBufRetType(ElementTy: ScalarTy);
911
912 Value *Handle =
913 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
914 Value *Index = CI->getArgOperand(i: 1);
915
916 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
917 Op: OpCode::CBufferLoadLegacy, Args: {Handle, Index}, Name: CI->getName(), RetTy: NewRetTy);
918 if (Error E = OpCall.takeError())
919 return E;
920 if (Error E = replaceNamedStructUses(Intrin: CI, DXILOp: *OpCall))
921 return E;
922
923 CI->eraseFromParent();
924 return Error::success();
925 });
926 }
927
928 [[nodiscard]] bool lowerUpdateCounter(Function &F) {
929 IRBuilder<> &IRB = OpBuilder.getIRB();
930 Type *Int32Ty = IRB.getInt32Ty();
931
932 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
933 IRB.SetInsertPoint(CI);
934 Value *Handle =
935 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
936 Value *Op1 = CI->getArgOperand(i: 1);
937
938 std::array<Value *, 2> Args{Handle, Op1};
939
940 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
941 Op: OpCode::UpdateCounter, Args, Name: CI->getName(), RetTy: Int32Ty);
942
943 if (Error E = OpCall.takeError())
944 return E;
945
946 CI->replaceAllUsesWith(V: *OpCall);
947 CI->eraseFromParent();
948 return Error::success();
949 });
950 }
951
952 [[nodiscard]] bool lowerGetDimensionsX(Function &F) {
953 IRBuilder<> &IRB = OpBuilder.getIRB();
954 Type *Int32Ty = IRB.getInt32Ty();
955
956 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
957 IRB.SetInsertPoint(CI);
958 Value *Handle =
959 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
960 Value *Undef = UndefValue::get(T: Int32Ty);
961
962 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
963 Op: OpCode::GetDimensions, Args: {Handle, Undef}, Name: CI->getName(), RetTy: Int32Ty);
964 if (Error E = OpCall.takeError())
965 return E;
966 Value *Dim = IRB.CreateExtractValue(Agg: *OpCall, Idxs: 0);
967
968 CI->replaceAllUsesWith(V: Dim);
969 CI->eraseFromParent();
970 return Error::success();
971 });
972 }
973
974 [[nodiscard]] bool lowerGetPointer(Function &F) {
975 // These should have already been handled in DXILResourceAccess, so we can
976 // just clean up the dead prototype.
977 assert(F.user_empty() && "getpointer operations should have been removed");
978 F.eraseFromParent();
979 return false;
980 }
981
982 /// Splits the value operand of a resource store into its (at most four)
983 /// scalar components. Slots beyond the length of `Data` are filled with
984 /// `undef` when `FillWithUndef` is set (raw and structured buffers), or with
985 /// the first component otherwise (typed UAVs, which must write all four
986 /// components - repeating the first one matches DXC).
987 static std::array<Value *, 4> splitStoreData(IRBuilder<> &IRB, Value *Data,
988 uint64_t NumElements,
989 bool FillWithUndef) {
990 std::array<Value *, 4> DataElements{nullptr, nullptr, nullptr, nullptr};
991 extractElementsIntoArgs(IRB, Args: DataElements, ArgIdx: 0, Src: Data, MaxElements: 4);
992
993 // For any elements beyond the length of the vector, we should fill it up
994 // with undef - however, for typed UAVs we repeat the first element to
995 // match DXC.
996 for (uint64_t I = NumElements, E = 4; I < E; ++I)
997 if (DataElements[I] == nullptr)
998 DataElements[I] =
999 FillWithUndef ? UndefValue::get(T: Data->getType()->getScalarType())
1000 : DataElements[0];
1001
1002 return DataElements;
1003 }
1004
1005 /// Erase the chain of `insertelement`s that only existed to build up a vector
1006 /// operand of an intrinsic we've just replaced.
1007 static void eraseDeadInsertElementChain(Value *Data) {
1008 auto *IEI = dyn_cast<InsertElementInst>(Val: Data);
1009 while (IEI && IEI->use_empty()) {
1010 InsertElementInst *Tmp = IEI;
1011 IEI = dyn_cast<InsertElementInst>(Val: IEI->getOperand(i_nocapture: 0));
1012 Tmp->eraseFromParent();
1013 }
1014 }
1015
1016 [[nodiscard]] bool lowerBufferStore(Function &F, bool IsRaw) {
1017 const DataLayout &DL = F.getDataLayout();
1018 IRBuilder<> &IRB = OpBuilder.getIRB();
1019 Type *Int8Ty = IRB.getInt8Ty();
1020 Type *Int32Ty = IRB.getInt32Ty();
1021
1022 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1023 IRB.SetInsertPoint(CI);
1024
1025 Value *Handle =
1026 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1027 Value *Index0 = CI->getArgOperand(i: 1);
1028 Value *Index1 = IsRaw ? CI->getArgOperand(i: 2) : UndefValue::get(T: Int32Ty);
1029
1030 if (IsRaw) {
1031 if (Error E = validateRawBufferElementIndex(Resource: CI->getOperand(i_nocapture: 0), ElementIndex: Index1))
1032 return E;
1033 if (isa<PoisonValue>(Val: Index1))
1034 Index1 = UndefValue::get(T: Index1->getType());
1035 }
1036
1037 Value *Data = CI->getArgOperand(i: IsRaw ? 3 : 2);
1038 Type *DataTy = Data->getType();
1039 Type *ScalarTy = DataTy->getScalarType();
1040
1041 uint64_t NumElements =
1042 DL.getTypeSizeInBits(Ty: DataTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
1043 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: IsRaw ? ~(~0U << NumElements)
1044 : TypedUAVStoreWriteMask);
1045
1046 // TODO: check that we only have vector or scalar...
1047 if (NumElements > 4)
1048 return make_error<StringError>(
1049 Args: "Buffer store data must have at most 4 elements",
1050 Args: inconvertibleErrorCode());
1051
1052 std::array<Value *, 4> DataElements =
1053 splitStoreData(IRB, Data, NumElements, /*FillWithUndef=*/IsRaw);
1054
1055 dxil::OpCode Op = OpCode::BufferStore;
1056 SmallVector<Value *, 9> Args{
1057 Handle, Index0, Index1, DataElements[0],
1058 DataElements[1], DataElements[2], DataElements[3], Mask};
1059 if (IsRaw && MMDI.DXILVersion >= VersionTuple(1, 2)) {
1060 Op = OpCode::RawBufferStore;
1061 // RawBufferStore requires the alignment
1062 Args.push_back(
1063 Elt: ConstantInt::get(Ty: Int32Ty, V: DL.getPrefTypeAlign(Ty: ScalarTy).value()));
1064 }
1065 Expected<CallInst *> OpCall =
1066 OpBuilder.tryCreateOp(Op, Args, Name: CI->getName());
1067 if (Error E = OpCall.takeError())
1068 return E;
1069
1070 CI->eraseFromParent();
1071 eraseDeadInsertElementChain(Data);
1072
1073 return Error::success();
1074 });
1075 }
1076
1077 /// Snapshot the vector-typed arguments of `CI` so their `insertelement`
1078 /// chains can be cleaned up once `CI` has been replaced. The handles are weak
1079 /// because two arguments can share an `insertelement` chain.
1080 static SmallVector<WeakTrackingVH, 4> collectVectorArgs(CallInst *CI) {
1081 SmallVector<WeakTrackingVH, 4> Vectors;
1082 for (Value *Arg : CI->args())
1083 if (isa<FixedVectorType>(Val: Arg->getType()))
1084 Vectors.emplace_back(Args&: Arg);
1085 return Vectors;
1086 }
1087
1088 static void eraseDeadInsertElementChains(ArrayRef<WeakTrackingVH> Vectors) {
1089 for (const WeakTrackingVH &VH : Vectors)
1090 if (Value *V = VH)
1091 eraseDeadInsertElementChain(Data: V);
1092 }
1093
1094 [[nodiscard]] bool lowerTextureStore(Function &F) {
1095 const DataLayout &DL = F.getDataLayout();
1096 IRBuilder<> &IRB = OpBuilder.getIRB();
1097 Type *Int8Ty = IRB.getInt8Ty();
1098 Type *Int32Ty = IRB.getInt32Ty();
1099
1100 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1101 IRB.SetInsertPoint(CI);
1102
1103 SmallVector<WeakTrackingVH, 4> VectorArgs = collectVectorArgs(CI);
1104 Value *Handle =
1105 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1106 Value *Coords = CI->getArgOperand(i: 1);
1107 Value *Data = CI->getArgOperand(i: 2);
1108
1109 Type *DataTy = Data->getType();
1110 Type *ScalarTy = DataTy->getScalarType();
1111 uint64_t NumElements =
1112 DL.getTypeSizeInBits(Ty: DataTy) / DL.getTypeSizeInBits(Ty: ScalarTy);
1113 if (NumElements > 4)
1114 return make_error<StringError>(
1115 Args: "Texture store data must have at most 4 elements",
1116 Args: inconvertibleErrorCode());
1117
1118 Value *Mask = ConstantInt::get(Ty: Int8Ty, V: TypedUAVStoreWriteMask);
1119 std::array<Value *, 4> DataElements =
1120 splitStoreData(IRB, Data, NumElements, /*FillWithUndef=*/false);
1121
1122 Value *Undef = UndefValue::get(T: Int32Ty);
1123 std::array<Value *, 9> Args{
1124 Handle, Undef, Undef,
1125 Undef, DataElements[0], DataElements[1],
1126 DataElements[2], DataElements[3], Mask};
1127
1128 // Copy the coordinates into Args.
1129 extractElementsIntoArgs(IRB, Args, ArgIdx: 1, Src: Coords, MaxElements: 3);
1130
1131 Expected<CallInst *> OpCall =
1132 OpBuilder.tryCreateOp(Op: OpCode::TextureStore, Args, Name: CI->getName());
1133 if (Error E = OpCall.takeError())
1134 return E;
1135
1136 CI->eraseFromParent();
1137 eraseDeadInsertElementChains(Vectors: VectorArgs);
1138
1139 return Error::success();
1140 });
1141 }
1142
1143 [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) {
1144 IRBuilder<> &IRB = OpBuilder.getIRB();
1145
1146 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1147 IRB.SetInsertPoint(CI);
1148
1149 // Cast the target-extension typed handle to `%dx.types.Handle`, tracked
1150 // via CleanupCasts so the pair is reconciled by `cleanupHandleCasts`.
1151 Value *Handle =
1152 createTmpHandleCast(V: CI->getArgOperand(i: 0), Ty: OpBuilder.getHandleType());
1153 Value *BinOp = CI->getArgOperand(i: 1);
1154 Value *Coord0 = CI->getArgOperand(i: 2);
1155 Value *Coord1 = CI->getArgOperand(i: 3);
1156 Value *Coord2 = CI->getArgOperand(i: 4);
1157 Value *NewValue = CI->getArgOperand(i: 5);
1158
1159 std::array<Value *, 6> Args{Handle, BinOp, Coord0,
1160 Coord1, Coord2, NewValue};
1161 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
1162 Op: dxil::OpCode::AtomicBinOp, Args, Name: CI->getName(), RetTy: CI->getType());
1163 if (Error E = OpCall.takeError()) {
1164 // Preserve the DXIL op error text but attach it as a
1165 // DiagnosticInfoUnsupported so we don't crash with a dangling call.
1166 std::string Message(toString(E: std::move(E)));
1167 CI->getContext().diagnose(DI: DiagnosticInfoUnsupported(
1168 *CI->getFunction(), Message, CI->getDebugLoc()));
1169 CI->replaceAllUsesWith(V: PoisonValue::get(T: CI->getType()));
1170 CI->eraseFromParent();
1171 return Error::success();
1172 }
1173
1174 CI->replaceAllUsesWith(V: *OpCall);
1175 CI->eraseFromParent();
1176 return Error::success();
1177 });
1178 }
1179
1180 [[nodiscard]] bool lowerCtpopToCountBits(Function &F) {
1181 IRBuilder<> &IRB = OpBuilder.getIRB();
1182 Type *Int32Ty = IRB.getInt32Ty();
1183
1184 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1185 IRB.SetInsertPoint(CI);
1186 SmallVector<Value *> Args;
1187 Args.append(in_start: CI->arg_begin(), in_end: CI->arg_end());
1188
1189 Type *RetTy = Int32Ty;
1190 Type *FRT = F.getReturnType();
1191 if (const auto *VT = dyn_cast<VectorType>(Val: FRT))
1192 RetTy = VectorType::get(ElementType: RetTy, Other: VT);
1193
1194 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(
1195 Op: dxil::OpCode::CountBits, Args, Name: CI->getName(), RetTy);
1196 if (Error E = OpCall.takeError())
1197 return E;
1198
1199 // If the result type is 32 bits we can do a direct replacement.
1200 if (FRT->isIntOrIntVectorTy(BitWidth: 32)) {
1201 CI->replaceAllUsesWith(V: *OpCall);
1202 CI->eraseFromParent();
1203 return Error::success();
1204 }
1205
1206 unsigned CastOp;
1207 unsigned CastOp2;
1208 if (FRT->isIntOrIntVectorTy(BitWidth: 16)) {
1209 CastOp = Instruction::ZExt;
1210 CastOp2 = Instruction::SExt;
1211 } else { // must be 64 bits
1212 assert(FRT->isIntOrIntVectorTy(64) &&
1213 "Currently only lowering 16, 32, or 64 bit ctpop to CountBits \
1214 is supported.");
1215 CastOp = Instruction::Trunc;
1216 CastOp2 = Instruction::Trunc;
1217 }
1218
1219 // It is correct to replace the ctpop with the dxil op and
1220 // remove all casts to i32
1221 bool NeedsCast = false;
1222 for (User *User : make_early_inc_range(Range: CI->users())) {
1223 Instruction *I = dyn_cast<Instruction>(Val: User);
1224 if (I && (I->getOpcode() == CastOp || I->getOpcode() == CastOp2) &&
1225 I->getType() == RetTy) {
1226 I->replaceAllUsesWith(V: *OpCall);
1227 I->eraseFromParent();
1228 } else
1229 NeedsCast = true;
1230 }
1231
1232 // It is correct to replace a ctpop with the dxil op and
1233 // a cast from i32 to the return type of the ctpop
1234 // the cast is emitted here if there is a non-cast to i32
1235 // instr which uses the ctpop
1236 if (NeedsCast) {
1237 Value *Cast =
1238 IRB.CreateZExtOrTrunc(V: *OpCall, DestTy: F.getReturnType(), Name: "ctpop.cast");
1239 CI->replaceAllUsesWith(V: Cast);
1240 }
1241
1242 CI->eraseFromParent();
1243 return Error::success();
1244 });
1245 }
1246
1247 [[nodiscard]] bool lowerLifetimeIntrinsic(Function &F) {
1248 IRBuilder<> &IRB = OpBuilder.getIRB();
1249 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1250 IRB.SetInsertPoint(CI);
1251 Value *Ptr = CI->getArgOperand(i: 0);
1252 assert(Ptr->getType()->isPointerTy() &&
1253 "Expected operand of lifetime intrinsic to be a pointer");
1254
1255 auto ZeroOrUndef = [&](Type *Ty) {
1256 return MMDI.ValidatorVersion < VersionTuple(1, 6)
1257 ? Constant::getNullValue(Ty)
1258 : UndefValue::get(T: Ty);
1259 };
1260
1261 Value *Val = nullptr;
1262 if (auto *GV = dyn_cast<GlobalVariable>(Val: Ptr)) {
1263 if (GV->hasInitializer() || GV->isExternallyInitialized())
1264 return Error::success();
1265 Val = ZeroOrUndef(GV->getValueType());
1266 } else if (auto *AI = dyn_cast<AllocaInst>(Val: Ptr))
1267 Val = ZeroOrUndef(AI->getAllocatedType());
1268
1269 assert(Val && "Expected operand of lifetime intrinsic to be a global "
1270 "variable or alloca instruction");
1271 IRB.CreateStore(Val, Ptr, isVolatile: false);
1272
1273 CI->eraseFromParent();
1274 return Error::success();
1275 });
1276 }
1277
1278 [[nodiscard]] bool lowerIsFPClass(Function &F) {
1279 IRBuilder<> &IRB = OpBuilder.getIRB();
1280 Type *RetTy = IRB.getInt1Ty();
1281
1282 return replaceFunction(F, ReplaceCall: [&](CallInst *CI) -> Error {
1283 IRB.SetInsertPoint(CI);
1284 SmallVector<Value *> Args;
1285 Value *Fl = CI->getArgOperand(i: 0);
1286 Args.push_back(Elt: Fl);
1287
1288 dxil::OpCode OpCode;
1289 Value *T = CI->getArgOperand(i: 1);
1290 auto *TCI = dyn_cast<ConstantInt>(Val: T);
1291 switch (TCI->getZExtValue()) {
1292 case FPClassTest::fcInf:
1293 OpCode = dxil::OpCode::IsInf;
1294 break;
1295 case FPClassTest::fcNan:
1296 OpCode = dxil::OpCode::IsNaN;
1297 break;
1298 case FPClassTest::fcNormal:
1299 OpCode = dxil::OpCode::IsNormal;
1300 break;
1301 case FPClassTest::fcFinite:
1302 OpCode = dxil::OpCode::IsFinite;
1303 break;
1304 default:
1305 SmallString<128> Msg =
1306 formatv(Fmt: "Unsupported FPClassTest {0} for DXIL Op Lowering",
1307 Vals: TCI->getZExtValue());
1308 return make_error<StringError>(Args&: Msg, Args: inconvertibleErrorCode());
1309 }
1310
1311 Expected<CallInst *> OpCall =
1312 OpBuilder.tryCreateOp(Op: OpCode, Args, Name: CI->getName(), RetTy);
1313 if (Error E = OpCall.takeError())
1314 return E;
1315
1316 CI->replaceAllUsesWith(V: *OpCall);
1317 CI->eraseFromParent();
1318 return Error::success();
1319 });
1320 }
1321
1322 bool lowerIntrinsics() {
1323 bool Updated = false;
1324 bool HasErrors = false;
1325
1326 for (Function &F : make_early_inc_range(Range: M.functions())) {
1327 if (!F.isDeclaration())
1328 continue;
1329 Intrinsic::ID ID = F.getIntrinsicID();
1330 switch (ID) {
1331 // NOTE: Skip dx_resource_casthandle here. They are
1332 // resolved after this loop in cleanupHandleCasts.
1333 case Intrinsic::dx_resource_casthandle:
1334 // NOTE: llvm.dbg.value is supported as is in DXIL.
1335 case Intrinsic::dbg_value:
1336 case Intrinsic::not_intrinsic:
1337 if (F.use_empty())
1338 F.eraseFromParent();
1339 continue;
1340 default:
1341 if (F.use_empty())
1342 F.eraseFromParent();
1343 else {
1344 SmallString<128> Msg = formatv(
1345 Fmt: "Unsupported intrinsic {0} for DXIL lowering", Vals: F.getName());
1346 M.getContext().emitError(ErrorStr: Msg);
1347 HasErrors |= true;
1348 }
1349 break;
1350
1351#define DXIL_OP_INTRINSIC(OpCode, Intrin, ...) \
1352 case Intrin: \
1353 HasErrors |= replaceFunctionWithOp( \
1354 F, OpCode, ArrayRef<IntrinArgSelect>{__VA_ARGS__}); \
1355 break;
1356#include "DXILOperation.inc"
1357 case Intrinsic::dx_resource_handlefrombinding:
1358 HasErrors |= lowerHandleFromBinding(F);
1359 break;
1360 case Intrinsic::dx_resource_handlefromheap:
1361 HasErrors |= lowerHandleFromHeap(F);
1362 break;
1363 case Intrinsic::dx_resource_getbasepointer:
1364 case Intrinsic::dx_resource_getpointer:
1365 HasErrors |= lowerGetPointer(F);
1366 break;
1367 case Intrinsic::dx_resource_nonuniformindex:
1368 assert(!CleanupNURI &&
1369 "overloaded llvm.dx.resource.nonuniformindex intrinsics?");
1370 CleanupNURI = &F;
1371 break;
1372 case Intrinsic::dx_resource_load_typedbuffer:
1373 HasErrors |= lowerTypedBufferLoad(F, /*HasCheckBit=*/true);
1374 break;
1375 case Intrinsic::dx_resource_load_level:
1376 HasErrors |= lowerTextureLoad(F);
1377 break;
1378 case Intrinsic::dx_resource_sample:
1379 HasErrors |= lowerSample(F, /*HasClamp=*/false);
1380 break;
1381 case Intrinsic::dx_resource_sample_clamp:
1382 HasErrors |= lowerSample(F, /*HasClamp=*/true);
1383 break;
1384 case Intrinsic::dx_resource_samplebias:
1385 HasErrors |= lowerSampleBias(F, /*HasClamp=*/false);
1386 break;
1387 case Intrinsic::dx_resource_samplebias_clamp:
1388 HasErrors |= lowerSampleBias(F, /*HasClamp=*/true);
1389 break;
1390 case Intrinsic::dx_resource_samplelevel:
1391 HasErrors |= lowerSampleLevel(F);
1392 break;
1393 case Intrinsic::dx_resource_samplegrad:
1394 HasErrors |= lowerSampleGrad(F, /*HasClamp=*/false);
1395 break;
1396 case Intrinsic::dx_resource_samplegrad_clamp:
1397 HasErrors |= lowerSampleGrad(F, /*HasClamp=*/true);
1398 break;
1399 case Intrinsic::dx_resource_store_typedbuffer:
1400 HasErrors |= lowerBufferStore(F, /*IsRaw=*/false);
1401 break;
1402 case Intrinsic::dx_resource_store_texture:
1403 HasErrors |= lowerTextureStore(F);
1404 break;
1405 case Intrinsic::dx_resource_load_rawbuffer:
1406 HasErrors |= lowerRawBufferLoad(F);
1407 break;
1408 case Intrinsic::dx_resource_store_rawbuffer:
1409 HasErrors |= lowerBufferStore(F, /*IsRaw=*/true);
1410 break;
1411 case Intrinsic::dx_resource_load_cbufferrow_2:
1412 case Intrinsic::dx_resource_load_cbufferrow_4:
1413 case Intrinsic::dx_resource_load_cbufferrow_8:
1414 HasErrors |= lowerCBufferLoad(F);
1415 break;
1416 case Intrinsic::dx_resource_updatecounter:
1417 HasErrors |= lowerUpdateCounter(F);
1418 break;
1419 case Intrinsic::dx_resource_atomic_binop:
1420 HasErrors |= lowerResourceAtomicBinOp(F);
1421 break;
1422 case Intrinsic::dx_resource_getdimensions_x:
1423 HasErrors |= lowerGetDimensionsX(F);
1424 break;
1425 case Intrinsic::ctpop:
1426 HasErrors |= lowerCtpopToCountBits(F);
1427 break;
1428 case Intrinsic::lifetime_start:
1429 case Intrinsic::lifetime_end:
1430 if (F.use_empty())
1431 F.eraseFromParent();
1432 else {
1433 if (MMDI.DXILVersion < VersionTuple(1, 6))
1434 HasErrors |= lowerLifetimeIntrinsic(F);
1435 else
1436 continue;
1437 }
1438 break;
1439 case Intrinsic::is_fpclass:
1440 HasErrors |= lowerIsFPClass(F);
1441 break;
1442 }
1443 Updated = true;
1444 }
1445 if (Updated && !HasErrors) {
1446 cleanupHandleCasts();
1447 cleanupNonUniformResourceIndexCalls();
1448 }
1449
1450 return Updated;
1451 }
1452};
1453} // namespace
1454
1455PreservedAnalyses DXILOpLowering::run(Module &M, ModuleAnalysisManager &MAM) {
1456 DXILResourceMap &DRM = MAM.getResult<DXILResourceAnalysis>(IR&: M);
1457 DXILResourceTypeMap &DRTM = MAM.getResult<DXILResourceTypeAnalysis>(IR&: M);
1458 const ModuleMetadataInfo MMDI = MAM.getResult<DXILMetadataAnalysis>(IR&: M);
1459
1460 const bool MadeChanges = OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
1461 if (!MadeChanges)
1462 return PreservedAnalyses::all();
1463 PreservedAnalyses PA;
1464 PA.preserve<DXILResourceAnalysis>();
1465 PA.preserve<DXILMetadataAnalysis>();
1466 PA.preserve<ShaderFlagsAnalysis>();
1467 PA.preserve<RootSignatureAnalysis>();
1468 return PA;
1469}
1470
1471namespace {
1472class DXILOpLoweringLegacy : public ModulePass {
1473public:
1474 bool runOnModule(Module &M) override {
1475 DXILResourceMap &DRM =
1476 getAnalysis<DXILResourceWrapperPass>().getResourceMap();
1477 DXILResourceTypeMap &DRTM =
1478 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
1479 const ModuleMetadataInfo MMDI =
1480 getAnalysis<DXILMetadataAnalysisWrapperPass>().getModuleMetadata();
1481
1482 return OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
1483 }
1484 StringRef getPassName() const override { return "DXIL Op Lowering"; }
1485 DXILOpLoweringLegacy() : ModulePass(ID) {}
1486
1487 static char ID; // Pass identification.
1488 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
1489 AU.addRequired<DXILResourceTypeWrapperPass>();
1490 AU.addRequired<DXILResourceWrapperPass>();
1491 AU.addRequired<DXILMetadataAnalysisWrapperPass>();
1492 AU.addPreserved<DXILResourceWrapperPass>();
1493 AU.addPreserved<DXILMetadataAnalysisWrapperPass>();
1494 AU.addPreserved<ShaderFlagsAnalysisWrapper>();
1495 AU.addPreserved<RootSignatureAnalysisWrapper>();
1496 }
1497};
1498char DXILOpLoweringLegacy::ID = 0;
1499} // end anonymous namespace
1500
1501INITIALIZE_PASS_BEGIN(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering",
1502 false, false)
1503INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
1504INITIALIZE_PASS_DEPENDENCY(DXILResourceWrapperPass)
1505INITIALIZE_PASS_END(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering", false,
1506 false)
1507
1508ModulePass *llvm::createDXILOpLoweringLegacyPass() {
1509 return new DXILOpLoweringLegacy();
1510}
1511