1//===- DXILIntrinsicExpansion.cpp - Prepare LLVM Module for DXIL encoding--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file This file contains DXIL intrinsic expansions for those that don't have
10// opcodes in DirectX Intermediate Language (DXIL).
11//===----------------------------------------------------------------------===//
12
13#include "DirectX.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/CodeGen/Passes.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/InstrTypes.h"
21#include "llvm/IR/Instruction.h"
22#include "llvm/IR/Instructions.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/IntrinsicsDirectX.h"
25#include "llvm/IR/MatrixBuilder.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PassManager.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Pass.h"
30#include "llvm/Support/Casting.h"
31#include "llvm/Support/ErrorHandling.h"
32#include "llvm/Support/MathExtras.h"
33
34#define DEBUG_TYPE "dxil-intrinsic-expansion"
35
36using namespace llvm;
37
38class DXILIntrinsicExpansionLegacy : public ModulePass {
39
40public:
41 bool runOnModule(Module &M) override;
42 DXILIntrinsicExpansionLegacy() : ModulePass(ID) {}
43
44 static char ID; // Pass identification.
45};
46
47static bool resourceAccessNeeds64BitExpansion(Module *M, Type *OverloadTy,
48 bool IsRaw) {
49 if (IsRaw && M->getTargetTriple().getDXILVersion() > VersionTuple(1, 2))
50 return false;
51
52 Type *ScalarTy = OverloadTy->getScalarType();
53 return ScalarTy->isDoubleTy() || ScalarTy->isIntegerTy(BitWidth: 64);
54}
55
56static Value *expand16BitIsInf(CallInst *Orig) {
57 Module *M = Orig->getModule();
58 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
59 return nullptr;
60
61 Value *Val = Orig->getOperand(i_nocapture: 0);
62 Type *ValTy = Val->getType();
63 if (!ValTy->getScalarType()->isHalfTy())
64 return nullptr;
65
66 IRBuilder<> Builder(Orig);
67 Type *IType = Type::getInt16Ty(C&: M->getContext());
68 Constant *PosInf =
69 ValTy->isVectorTy()
70 ? ConstantVector::getSplat(
71 EC: ElementCount::getFixed(
72 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
73 Elt: ConstantInt::get(Ty: IType, V: 0x7c00))
74 : ConstantInt::get(Ty: IType, V: 0x7c00);
75
76 Constant *NegInf =
77 ValTy->isVectorTy()
78 ? ConstantVector::getSplat(
79 EC: ElementCount::getFixed(
80 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
81 Elt: ConstantInt::get(Ty: IType, V: 0xfc00))
82 : ConstantInt::get(Ty: IType, V: 0xfc00);
83
84 Value *IVal = Builder.CreateBitCast(V: Val, DestTy: PosInf->getType());
85 Value *B1 = Builder.CreateICmpEQ(LHS: IVal, RHS: PosInf);
86 Value *B2 = Builder.CreateICmpEQ(LHS: IVal, RHS: NegInf);
87 Value *B3 = Builder.CreateOr(LHS: B1, RHS: B2);
88 return B3;
89}
90
91static Value *expand16BitIsNaN(CallInst *Orig) {
92 Module *M = Orig->getModule();
93 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
94 return nullptr;
95
96 Value *Val = Orig->getOperand(i_nocapture: 0);
97 Type *ValTy = Val->getType();
98 if (!ValTy->getScalarType()->isHalfTy())
99 return nullptr;
100
101 IRBuilder<> Builder(Orig);
102 Type *IType = Type::getInt16Ty(C&: M->getContext());
103
104 Constant *ExpBitMask =
105 ValTy->isVectorTy()
106 ? ConstantVector::getSplat(
107 EC: ElementCount::getFixed(
108 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
109 Elt: ConstantInt::get(Ty: IType, V: 0x7c00))
110 : ConstantInt::get(Ty: IType, V: 0x7c00);
111 Constant *SigBitMask =
112 ValTy->isVectorTy()
113 ? ConstantVector::getSplat(
114 EC: ElementCount::getFixed(
115 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
116 Elt: ConstantInt::get(Ty: IType, V: 0x3ff))
117 : ConstantInt::get(Ty: IType, V: 0x3ff);
118
119 Constant *Zero =
120 ValTy->isVectorTy()
121 ? ConstantVector::getSplat(
122 EC: ElementCount::getFixed(
123 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
124 Elt: ConstantInt::get(Ty: IType, V: 0))
125 : ConstantInt::get(Ty: IType, V: 0);
126
127 Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType());
128 Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask);
129 Value *B1 = Builder.CreateICmpEQ(LHS: Exp, RHS: ExpBitMask);
130
131 Value *Sig = Builder.CreateAnd(LHS: IVal, RHS: SigBitMask);
132 Value *B2 = Builder.CreateICmpNE(LHS: Sig, RHS: Zero);
133 Value *B3 = Builder.CreateAnd(LHS: B1, RHS: B2);
134 return B3;
135}
136
137static Value *expand16BitIsFinite(CallInst *Orig) {
138 Module *M = Orig->getModule();
139 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
140 return nullptr;
141
142 Value *Val = Orig->getOperand(i_nocapture: 0);
143 Type *ValTy = Val->getType();
144 if (!ValTy->getScalarType()->isHalfTy())
145 return nullptr;
146
147 IRBuilder<> Builder(Orig);
148 Type *IType = Type::getInt16Ty(C&: M->getContext());
149
150 Constant *ExpBitMask =
151 ValTy->isVectorTy()
152 ? ConstantVector::getSplat(
153 EC: ElementCount::getFixed(
154 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
155 Elt: ConstantInt::get(Ty: IType, V: 0x7c00))
156 : ConstantInt::get(Ty: IType, V: 0x7c00);
157
158 Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType());
159 Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask);
160 Value *B1 = Builder.CreateICmpNE(LHS: Exp, RHS: ExpBitMask);
161 return B1;
162}
163
164static Value *expand16BitIsNormal(CallInst *Orig) {
165 Module *M = Orig->getModule();
166 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
167 return nullptr;
168
169 Value *Val = Orig->getOperand(i_nocapture: 0);
170 Type *ValTy = Val->getType();
171 if (!ValTy->getScalarType()->isHalfTy())
172 return nullptr;
173
174 IRBuilder<> Builder(Orig);
175 Type *IType = Type::getInt16Ty(C&: M->getContext());
176
177 Constant *ExpBitMask =
178 ValTy->isVectorTy()
179 ? ConstantVector::getSplat(
180 EC: ElementCount::getFixed(
181 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
182 Elt: ConstantInt::get(Ty: IType, V: 0x7c00))
183 : ConstantInt::get(Ty: IType, V: 0x7c00);
184 Constant *Zero =
185 ValTy->isVectorTy()
186 ? ConstantVector::getSplat(
187 EC: ElementCount::getFixed(
188 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
189 Elt: ConstantInt::get(Ty: IType, V: 0))
190 : ConstantInt::get(Ty: IType, V: 0);
191
192 Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType());
193 Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask);
194 Value *NotAllZeroes = Builder.CreateICmpNE(LHS: Exp, RHS: Zero);
195 Value *NotAllOnes = Builder.CreateICmpNE(LHS: Exp, RHS: ExpBitMask);
196 Value *B1 = Builder.CreateAnd(LHS: NotAllZeroes, RHS: NotAllOnes);
197 return B1;
198}
199
200static bool shouldExpandFloatDotIntrinsic(Function &F) {
201 assert(F.getIntrinsicID() == Intrinsic::dx_fdot &&
202 "Function is not a dx.fdot intrinsic");
203 auto *ParamTy = cast<FixedVectorType>(Val: F.getFunctionType()->getParamType(i: 0));
204 return ParamTy->getNumElements() <= 4 ||
205 F.getParent()->getTargetTriple().getOSVersion() < VersionTuple(6, 9);
206}
207
208static bool isIntrinsicExpansion(Function &F) {
209 switch (F.getIntrinsicID()) {
210 case Intrinsic::assume:
211 case Intrinsic::abs:
212 case Intrinsic::atan2:
213 case Intrinsic::copysign:
214 case Intrinsic::fshl:
215 case Intrinsic::fshr:
216 case Intrinsic::exp:
217 case Intrinsic::is_fpclass:
218 case Intrinsic::log:
219 case Intrinsic::log10:
220 case Intrinsic::pow:
221 case Intrinsic::powi:
222 case Intrinsic::dx_uclamp:
223 case Intrinsic::dx_sclamp:
224 case Intrinsic::dx_nclamp:
225 case Intrinsic::dx_isfinite:
226 case Intrinsic::dx_isinf:
227 case Intrinsic::dx_isnan:
228 case Intrinsic::dx_sdot:
229 case Intrinsic::dx_udot:
230 case Intrinsic::dx_sign:
231 case Intrinsic::usub_sat:
232 case Intrinsic::vector_reduce_add:
233 case Intrinsic::vector_reduce_fadd:
234 case Intrinsic::matrix_multiply:
235 case Intrinsic::matrix_transpose:
236 case Intrinsic::umul_with_overflow:
237 case Intrinsic::smul_with_overflow:
238 case Intrinsic::dx_load_input:
239 case Intrinsic::dx_store_output:
240 return true;
241 case Intrinsic::vector_reduce_and:
242 case Intrinsic::vector_reduce_or:
243 return F.getParent()->getTargetTriple().getDXILVersion() <
244 VersionTuple(1, 9);
245 case Intrinsic::dx_quad_all:
246 return F.getParent()->getTargetTriple().getDXILVersion() <
247 VersionTuple(1, 7);
248 case Intrinsic::dx_fdot:
249 return shouldExpandFloatDotIntrinsic(F);
250 case Intrinsic::dx_resource_load_rawbuffer:
251 return resourceAccessNeeds64BitExpansion(
252 M: F.getParent(), OverloadTy: F.getReturnType()->getStructElementType(N: 0),
253 /*IsRaw*/ true);
254 case Intrinsic::dx_resource_load_typedbuffer:
255 return resourceAccessNeeds64BitExpansion(
256 M: F.getParent(), OverloadTy: F.getReturnType()->getStructElementType(N: 0),
257 /*IsRaw*/ false);
258 case Intrinsic::dx_resource_store_rawbuffer:
259 return resourceAccessNeeds64BitExpansion(
260 M: F.getParent(), OverloadTy: F.getFunctionType()->getParamType(i: 3), /*IsRaw*/ true);
261 case Intrinsic::dx_resource_store_typedbuffer:
262 return resourceAccessNeeds64BitExpansion(
263 M: F.getParent(), OverloadTy: F.getFunctionType()->getParamType(i: 2), /*IsRaw*/ false);
264 }
265 return false;
266}
267
268static Value *expandUsubSat(CallInst *Orig) {
269 Value *A = Orig->getArgOperand(i: 0);
270 Value *B = Orig->getArgOperand(i: 1);
271 Type *Ty = A->getType();
272
273 IRBuilder<> Builder(Orig);
274
275 Value *Cmp = Builder.CreateICmpULT(LHS: A, RHS: B, Name: "usub.cmp");
276 Value *Sub = Builder.CreateSub(LHS: A, RHS: B, Name: "usub.sub");
277 Value *Zero = ConstantInt::get(Ty, V: 0);
278 return Builder.CreateSelect(C: Cmp, True: Zero, False: Sub, Name: "usub.sat");
279}
280
281// Compute the high N bits of the 2N-bit unsigned product of two N-bit values
282// using only N-bit arithmetic, so we don't introduce a wider integer type that
283// may be unsupported in DXIL.
284static Value *createMulHighUnsigned(IRBuilder<> &Builder, Value *A, Value *B,
285 Type *Ty, unsigned BW) {
286 assert(BW % 2 == 0 && "high-half split needs symmetric halves");
287 unsigned Half = BW / 2;
288 Value *HalfShift = ConstantInt::get(Ty, V: Half);
289 Value *LoMask = ConstantInt::get(Ty, V: APInt::getLowBitsSet(numBits: BW, loBitsSet: Half));
290
291 Value *U0 = Builder.CreateAnd(LHS: A, RHS: LoMask);
292 Value *U1 = Builder.CreateLShr(LHS: A, RHS: HalfShift);
293 Value *V0 = Builder.CreateAnd(LHS: B, RHS: LoMask);
294 Value *V1 = Builder.CreateLShr(LHS: B, RHS: HalfShift);
295
296 Value *W0 = Builder.CreateMul(LHS: U0, RHS: V0);
297 Value *T = Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U1, RHS: V0),
298 RHS: Builder.CreateLShr(LHS: W0, RHS: HalfShift));
299 Value *W1 = Builder.CreateAnd(LHS: T, RHS: LoMask);
300 Value *W2 = Builder.CreateLShr(LHS: T, RHS: HalfShift);
301 W1 = Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U0, RHS: V1), RHS: W1);
302 return Builder.CreateAdd(LHS: Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U1, RHS: V1), RHS: W2),
303 RHS: Builder.CreateLShr(LHS: W1, RHS: HalfShift));
304}
305
306// Expand a {u,s}mul.with.overflow intrinsic. The low half of the result is a
307// plain multiply; overflow is derived from the high half of the double-width
308// product.
309static Value *expandMulWithOverflow(CallInst *Orig, bool Signed) {
310 IRBuilder<> Builder(Orig);
311 Value *A = Orig->getArgOperand(i: 0);
312 Value *B = Orig->getArgOperand(i: 1);
313 Type *Ty = A->getType();
314 unsigned BW = Ty->getScalarSizeInBits();
315
316 Value *Lo;
317 Value *Ov;
318
319 // A plain double-width multiply is simplest, but we avoid it once it would
320 // introduce a 64-bit (or wider) integer, which DXIL does not always support.
321 // For i32 we use the native DXIL IMul/UMul ops, which return the full product
322 // as two i32s; wider types fall back to a same-width high-half computation.
323 if (2 * BW <= 32) {
324 Lo = Builder.CreateMul(LHS: A, RHS: B);
325 Type *WideTy = Ty->getWithNewBitWidth(NewBitWidth: 2 * BW);
326 Value *WideA =
327 Signed ? Builder.CreateSExt(V: A, DestTy: WideTy) : Builder.CreateZExt(V: A, DestTy: WideTy);
328 Value *WideB =
329 Signed ? Builder.CreateSExt(V: B, DestTy: WideTy) : Builder.CreateZExt(V: B, DestTy: WideTy);
330 Value *Wide = Builder.CreateMul(LHS: WideA, RHS: WideB);
331 if (Signed) {
332 // Overflow when the full product doesn't fit back into BW signed bits.
333 Ov = Builder.CreateICmpNE(LHS: Wide, RHS: Builder.CreateSExt(V: Lo, DestTy: WideTy));
334 } else {
335 Value *Hi = Builder.CreateLShr(LHS: Wide, RHS: ConstantInt::get(Ty: WideTy, V: BW));
336 Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty: WideTy, V: 0));
337 }
338 } else if (BW == 32) {
339 // IMul/UMul return {high, low}; index 0 is the high 32 bits.
340 Type *ResTy = StructType::get(elt1: Ty, elts: Ty);
341 Intrinsic::ID IntrinsicID =
342 Signed ? Intrinsic::dx_imul : Intrinsic::dx_umul;
343 Value *Mul = Builder.CreateIntrinsic(RetTy: ResTy, ID: IntrinsicID, Args: {A, B});
344 Value *Hi = Builder.CreateExtractValue(Agg: Mul, Idxs: 0);
345 Lo = Builder.CreateExtractValue(Agg: Mul, Idxs: 1);
346 if (Signed)
347 Ov = Builder.CreateICmpNE(
348 LHS: Hi, RHS: Builder.CreateAShr(LHS: Lo, RHS: ConstantInt::get(Ty, V: BW - 1)));
349 else
350 Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty, V: 0));
351 } else {
352 Lo = Builder.CreateMul(LHS: A, RHS: B);
353 Value *Hi = createMulHighUnsigned(Builder, A, B, Ty, BW);
354 if (Signed) {
355 // Turn the unsigned high half into the signed one, then overflow means it
356 // isn't the sign extension of the low half.
357 Value *SignShift = ConstantInt::get(Ty, V: BW - 1);
358 Value *ASign = Builder.CreateAShr(LHS: A, RHS: SignShift);
359 Value *BSign = Builder.CreateAShr(LHS: B, RHS: SignShift);
360 Hi = Builder.CreateSub(LHS: Hi, RHS: Builder.CreateAnd(LHS: ASign, RHS: B));
361 Hi = Builder.CreateSub(LHS: Hi, RHS: Builder.CreateAnd(LHS: BSign, RHS: A));
362 Ov = Builder.CreateICmpNE(LHS: Hi, RHS: Builder.CreateAShr(LHS: Lo, RHS: SignShift));
363 } else {
364 Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty, V: 0));
365 }
366 }
367
368 Value *Agg = PoisonValue::get(T: Orig->getType());
369 Agg = Builder.CreateInsertValue(Agg, Val: Lo, Idxs: 0);
370 return Builder.CreateInsertValue(Agg, Val: Ov, Idxs: 1);
371}
372
373static Value *expandVecReduceAdd(CallInst *Orig, Intrinsic::ID IntrinsicId) {
374 assert(IntrinsicId == Intrinsic::vector_reduce_add ||
375 IntrinsicId == Intrinsic::vector_reduce_fadd);
376
377 IRBuilder<> Builder(Orig);
378 bool IsFAdd = (IntrinsicId == Intrinsic::vector_reduce_fadd);
379
380 Value *X = Orig->getOperand(i_nocapture: IsFAdd ? 1 : 0);
381 Type *Ty = X->getType();
382 auto *XVec = dyn_cast<FixedVectorType>(Val: Ty);
383 unsigned XVecSize = XVec->getNumElements();
384 Value *Sum = Builder.CreateExtractElement(Vec: X, Idx: static_cast<uint64_t>(0));
385
386 // Handle the initial start value for floating-point addition.
387 if (IsFAdd) {
388 Constant *StartValue = dyn_cast<Constant>(Val: Orig->getOperand(i_nocapture: 0));
389 if (StartValue && !StartValue->isNullValue())
390 Sum = Builder.CreateFAdd(L: Sum, R: StartValue);
391 }
392
393 // Accumulate the remaining vector elements.
394 for (unsigned I = 1; I < XVecSize; I++) {
395 Value *Elt = Builder.CreateExtractElement(Vec: X, Idx: I);
396 if (IsFAdd)
397 Sum = Builder.CreateFAdd(L: Sum, R: Elt);
398 else
399 Sum = Builder.CreateAdd(LHS: Sum, RHS: Elt);
400 }
401
402 return Sum;
403}
404
405static Value *expandVecReduceAndOr(CallInst *Orig, Intrinsic::ID IntrinsicId) {
406 assert(IntrinsicId == Intrinsic::vector_reduce_and ||
407 IntrinsicId == Intrinsic::vector_reduce_or);
408
409 IRBuilder<> Builder(Orig);
410 Value *X = Orig->getArgOperand(i: 0);
411 auto *XVec = cast<FixedVectorType>(Val: X->getType());
412 Value *Result = Builder.CreateExtractElement(Vec: X, Idx: static_cast<uint64_t>(0));
413 for (unsigned I = 1; I < XVec->getNumElements(); ++I) {
414 Value *Elt = Builder.CreateExtractElement(Vec: X, Idx: I);
415 Result = IntrinsicId == Intrinsic::vector_reduce_or
416 ? Builder.CreateOr(LHS: Result, RHS: Elt)
417 : Builder.CreateAnd(LHS: Result, RHS: Elt);
418 }
419 return Result;
420}
421
422static Value *expandAbs(CallInst *Orig) {
423 Value *X = Orig->getOperand(i_nocapture: 0);
424 IRBuilder<> Builder(Orig);
425 Type *Ty = X->getType();
426 Type *EltTy = Ty->getScalarType();
427 Constant *Zero = Ty->isVectorTy()
428 ? ConstantVector::getSplat(
429 EC: ElementCount::getFixed(
430 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
431 Elt: ConstantInt::get(Ty: EltTy, V: 0))
432 : ConstantInt::get(Ty: EltTy, V: 0);
433 auto *V = Builder.CreateSub(LHS: Zero, RHS: X);
434 return Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::smax, Args: {X, V}, FMFSource: nullptr,
435 Name: "dx.max");
436}
437
438// Create a DXIL dot2, dot3, or dot4 for the given operands.
439static Value *expandFloatDotChunk(CallInst *Orig, Value *A, Value *B) {
440 Type *ATy = A->getType();
441 [[maybe_unused]] Type *BTy = B->getType();
442 assert(ATy->isVectorTy() && BTy->isVectorTy());
443
444 IRBuilder<> Builder(Orig);
445
446 auto *AVec = dyn_cast<FixedVectorType>(Val: ATy);
447
448 assert(ATy->getScalarType()->isFloatingPointTy());
449
450 unsigned NumElts = AVec->getNumElements();
451 Intrinsic::ID DotIntrinsic;
452 switch (NumElts) {
453 case 2:
454 DotIntrinsic = Intrinsic::dx_dot2;
455 break;
456 case 3:
457 DotIntrinsic = Intrinsic::dx_dot3;
458 break;
459 case 4:
460 DotIntrinsic = Intrinsic::dx_dot4;
461 break;
462 default:
463 reportFatalUsageError(
464 reason: "Invalid dot product input vector: length is outside 2-4");
465 }
466
467 SmallVector<Value *> Args;
468 for (unsigned I = 0; I < NumElts; ++I)
469 Args.push_back(Elt: Builder.CreateExtractElement(Vec: A, Idx: Builder.getInt32(C: I)));
470 for (unsigned I = 0; I < NumElts; ++I)
471 Args.push_back(Elt: Builder.CreateExtractElement(Vec: B, Idx: Builder.getInt32(C: I)));
472 return Builder.CreateIntrinsic(RetTy: ATy->getScalarType(), ID: DotIntrinsic, Args,
473 FMFSource: nullptr, Name: "dot");
474}
475
476// Expand an arbitrary-width float dot into the minimum number of legal DXIL
477// dot2, dot3, and dot4 operations.
478static Value *expandFloatDotIntrinsic(CallInst *Orig) {
479 Value *A = Orig->getOperand(i_nocapture: 0);
480 Value *B = Orig->getOperand(i_nocapture: 1);
481 unsigned NumElts = cast<FixedVectorType>(Val: A->getType())->getNumElements();
482
483 // We return early here to avoid constructing unnecessary identity shuffles.
484 if (NumElts <= 4)
485 return expandFloatDotChunk(Orig, A, B);
486
487 assert(Orig->getModule()->getTargetTriple().getOSVersion() <
488 VersionTuple(6, 9) &&
489 "long fdot must not be expanded for shader model 6.9 or later");
490
491 IRBuilder<> Builder(Orig);
492 Value *Result = nullptr;
493 for (unsigned Offset = 0; Offset < NumElts;) {
494 unsigned Remaining = NumElts - Offset;
495 // Taking four is optimal unless it would leave an illegal one-element
496 // tail. In that case, take three and finish with dot2.
497 unsigned ChunkSize = Remaining == 5 ? 3 : std::min(a: Remaining, b: 4u);
498 SmallVector<int, 4> Mask;
499 for (unsigned I = 0; I < ChunkSize; ++I)
500 Mask.push_back(Elt: Offset + I);
501 Value *AChunk = Builder.CreateShuffleVector(V: A, Mask);
502 Value *BChunk = Builder.CreateShuffleVector(V: B, Mask);
503 Value *Chunk = expandFloatDotChunk(Orig, A: AChunk, B: BChunk);
504 Result = Result ? Builder.CreateFAdd(L: Result, R: Chunk, Name: "dot.add") : Chunk;
505 Offset += ChunkSize;
506 }
507 return Result;
508}
509
510// Expand integer dot product to multiply and add ops
511static Value *expandIntegerDotIntrinsic(CallInst *Orig,
512 Intrinsic::ID DotIntrinsic) {
513 assert(DotIntrinsic == Intrinsic::dx_sdot ||
514 DotIntrinsic == Intrinsic::dx_udot);
515 Value *A = Orig->getOperand(i_nocapture: 0);
516 Value *B = Orig->getOperand(i_nocapture: 1);
517 Type *ATy = A->getType();
518 [[maybe_unused]] Type *BTy = B->getType();
519 assert(ATy->isVectorTy() && BTy->isVectorTy());
520
521 IRBuilder<> Builder(Orig);
522
523 auto *AVec = dyn_cast<FixedVectorType>(Val: ATy);
524
525 assert(ATy->getScalarType()->isIntegerTy());
526
527 Value *Result;
528 Intrinsic::ID MadIntrinsic = DotIntrinsic == Intrinsic::dx_sdot
529 ? Intrinsic::dx_imad
530 : Intrinsic::dx_umad;
531 Value *Elt0 = Builder.CreateExtractElement(Vec: A, Idx: (uint64_t)0);
532 Value *Elt1 = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
533 Result = Builder.CreateMul(LHS: Elt0, RHS: Elt1);
534 for (unsigned I = 1; I < AVec->getNumElements(); I++) {
535 Elt0 = Builder.CreateExtractElement(Vec: A, Idx: I);
536 Elt1 = Builder.CreateExtractElement(Vec: B, Idx: I);
537 Result = Builder.CreateIntrinsic(RetTy: Result->getType(), ID: MadIntrinsic,
538 Args: ArrayRef<Value *>{Elt0, Elt1, Result},
539 FMFSource: nullptr, Name: "dx.mad");
540 }
541 return Result;
542}
543
544static Value *expandExpIntrinsic(CallInst *Orig) {
545 Value *X = Orig->getOperand(i_nocapture: 0);
546 IRBuilder<> Builder(Orig);
547 Type *Ty = X->getType();
548 Type *EltTy = Ty->getScalarType();
549 Constant *Log2eConst =
550 Ty->isVectorTy() ? ConstantVector::getSplat(
551 EC: ElementCount::getFixed(
552 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
553 Elt: ConstantFP::get(Ty: EltTy, V: numbers::log2ef))
554 : ConstantFP::get(Ty: EltTy, V: numbers::log2ef);
555 Value *NewX = Builder.CreateFMul(L: Log2eConst, R: X);
556 CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding(
557 RetTy: Ty, ID: Intrinsic::exp2, Args: {NewX}, FMFSource: nullptr, Name: "dx.exp2");
558 Exp2Call->setTailCall(Orig->isTailCall());
559 Exp2Call->setAttributes(Orig->getAttributes());
560 return Exp2Call;
561}
562
563static Value *expandIsFPClass(CallInst *Orig) {
564 Value *T = Orig->getArgOperand(i: 1);
565 auto *TCI = dyn_cast<ConstantInt>(Val: T);
566
567 // These FPClassTest cases have DXIL opcodes, so they will be handled in
568 // DXIL Op Lowering instead for all non f16 cases.
569 switch (TCI->getZExtValue()) {
570 case FPClassTest::fcInf:
571 return expand16BitIsInf(Orig);
572 case FPClassTest::fcNan:
573 return expand16BitIsNaN(Orig);
574 case FPClassTest::fcNormal:
575 return expand16BitIsNormal(Orig);
576 case FPClassTest::fcFinite:
577 return expand16BitIsFinite(Orig);
578 }
579
580 IRBuilder<> Builder(Orig);
581
582 Value *F = Orig->getArgOperand(i: 0);
583 Type *FTy = F->getType();
584 unsigned FNumElem = 0; // 0 => F is not a vector
585
586 unsigned BitWidth; // Bit width of F or the ElemTy of F
587 Type *BitCastTy; // An IntNTy of the same bitwidth as F or ElemTy of F
588
589 if (auto *FVecTy = dyn_cast<FixedVectorType>(Val: FTy)) {
590 Type *ElemTy = FVecTy->getElementType();
591 FNumElem = FVecTy->getNumElements();
592 BitWidth = ElemTy->getPrimitiveSizeInBits();
593 BitCastTy = FixedVectorType::get(ElementType: Builder.getIntNTy(N: BitWidth), NumElts: FNumElem);
594 } else {
595 BitWidth = FTy->getPrimitiveSizeInBits();
596 BitCastTy = Builder.getIntNTy(N: BitWidth);
597 }
598
599 Value *FBitCast = Builder.CreateBitCast(V: F, DestTy: BitCastTy);
600 switch (TCI->getZExtValue()) {
601 case FPClassTest::fcNegZero: {
602 Value *NegZero = ConstantInt::get(Ty: Builder.getIntNTy(N: BitWidth),
603 V: APInt::getSignMask(BitWidth));
604 Value *RetVal;
605 if (FNumElem) {
606 Value *NegZeroSplat = Builder.CreateVectorSplat(NumElts: FNumElem, V: NegZero);
607 RetVal =
608 Builder.CreateICmpEQ(LHS: FBitCast, RHS: NegZeroSplat, Name: "is.fpclass.negzero");
609 } else
610 RetVal = Builder.CreateICmpEQ(LHS: FBitCast, RHS: NegZero, Name: "is.fpclass.negzero");
611 return RetVal;
612 }
613 default:
614 reportFatalUsageError(reason: "Unsupported FPClassTest");
615 }
616}
617
618static Value *expandLogIntrinsic(CallInst *Orig,
619 float LogConstVal = numbers::ln2f) {
620 Value *X = Orig->getOperand(i_nocapture: 0);
621 IRBuilder<> Builder(Orig);
622 Type *Ty = X->getType();
623 Type *EltTy = Ty->getScalarType();
624 Constant *Ln2Const =
625 Ty->isVectorTy() ? ConstantVector::getSplat(
626 EC: ElementCount::getFixed(
627 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
628 Elt: ConstantFP::get(Ty: EltTy, V: LogConstVal))
629 : ConstantFP::get(Ty: EltTy, V: LogConstVal);
630 CallInst *Log2Call = Builder.CreateIntrinsicWithoutFolding(
631 RetTy: Ty, ID: Intrinsic::log2, Args: {X}, FMFSource: nullptr, Name: "elt.log2");
632 Log2Call->setTailCall(Orig->isTailCall());
633 Log2Call->setAttributes(Orig->getAttributes());
634 return Builder.CreateFMul(L: Ln2Const, R: Log2Call);
635}
636static Value *expandLog10Intrinsic(CallInst *Orig) {
637 return expandLogIntrinsic(Orig, LogConstVal: numbers::ln2f / numbers::ln10f);
638}
639
640static Value *expandAtan2Intrinsic(CallInst *Orig) {
641 Value *Y = Orig->getOperand(i_nocapture: 0);
642 Value *X = Orig->getOperand(i_nocapture: 1);
643 Type *Ty = X->getType();
644 IRBuilder<> Builder(Orig);
645 Builder.setFastMathFlags(Orig->getFastMathFlags());
646
647 Value *Tan = Builder.CreateFDiv(L: Y, R: X);
648
649 CallInst *Atan = Builder.CreateIntrinsicWithoutFolding(
650 RetTy: Ty, ID: Intrinsic::atan, Args: {Tan}, FMFSource: nullptr, Name: "Elt.Atan");
651 Atan->setTailCall(Orig->isTailCall());
652 Atan->setAttributes(Orig->getAttributes());
653
654 // Modify atan result based on https://en.wikipedia.org/wiki/Atan2.
655 Constant *Pi = ConstantFP::get(Ty, V: llvm::numbers::pi);
656 Constant *HalfPi = ConstantFP::get(Ty, V: llvm::numbers::pi / 2);
657 Constant *NegHalfPi = ConstantFP::get(Ty, V: -llvm::numbers::pi / 2);
658 Constant *Zero = ConstantFP::get(Ty, V: 0);
659 Value *AtanAddPi = Builder.CreateFAdd(L: Atan, R: Pi);
660 Value *AtanSubPi = Builder.CreateFSub(L: Atan, R: Pi);
661
662 // x > 0 -> atan.
663 Value *Result = Atan;
664 Value *XLt0 = Builder.CreateFCmpOLT(LHS: X, RHS: Zero);
665 Value *XEq0 = Builder.CreateFCmpOEQ(LHS: X, RHS: Zero);
666 Value *YGe0 = Builder.CreateFCmpOGE(LHS: Y, RHS: Zero);
667 Value *YLt0 = Builder.CreateFCmpOLT(LHS: Y, RHS: Zero);
668
669 // x < 0, y >= 0 -> atan + pi.
670 Value *XLt0AndYGe0 = Builder.CreateAnd(LHS: XLt0, RHS: YGe0);
671 Result = Builder.CreateSelect(C: XLt0AndYGe0, True: AtanAddPi, False: Result);
672
673 // x < 0, y < 0 -> atan - pi.
674 Value *XLt0AndYLt0 = Builder.CreateAnd(LHS: XLt0, RHS: YLt0);
675 Result = Builder.CreateSelect(C: XLt0AndYLt0, True: AtanSubPi, False: Result);
676
677 // x == 0, y < 0 -> -pi/2
678 Value *XEq0AndYLt0 = Builder.CreateAnd(LHS: XEq0, RHS: YLt0);
679 Result = Builder.CreateSelect(C: XEq0AndYLt0, True: NegHalfPi, False: Result);
680
681 // x == 0, y > 0 -> pi/2
682 Value *XEq0AndYGe0 = Builder.CreateAnd(LHS: XEq0, RHS: YGe0);
683 Result = Builder.CreateSelect(C: XEq0AndYGe0, True: HalfPi, False: Result);
684
685 return Result;
686}
687
688template <bool LeftFunnel>
689static Value *expandFunnelShiftIntrinsic(CallInst *Orig) {
690 Type *Ty = Orig->getType();
691 Value *A = Orig->getOperand(i_nocapture: 0);
692 Value *B = Orig->getOperand(i_nocapture: 1);
693 Value *Shift = Orig->getOperand(i_nocapture: 2);
694
695 IRBuilder<> Builder(Orig);
696
697 assert(llvm::isPowerOf2_32(Ty->getScalarSizeInBits()) &&
698 "Can't use Mask to compute modulo and inverse");
699
700 // Note: if (Shift % BitWidth) == 0 then (BitWidth - Shift) == BitWidth,
701 // shifting by the bitwidth for shl/lshr returns a poisoned result. As such,
702 // we implement the same formula as LegalizerHelper::lowerFunnelShiftAsShifts.
703 //
704 // The funnel shift is expanded like so:
705 // fshl
706 // -> msb_extract((concat(A, B) << (Shift % BitWidth)), BitWidth)
707 // -> A << (Shift % BitWidth) | B >> 1 >> (BitWidth - 1 - (Shift % BitWidth))
708 // fshr
709 // -> lsb_extract((concat(A, B) >> (Shift % BitWidth), BitWidth))
710 // -> A << 1 << (BitWidth - 1 - (Shift % BitWidth)) | B >> (Shift % BitWidth)
711
712 // (BitWidth - 1) -> Mask
713 Constant *Mask = ConstantInt::get(Ty, V: Ty->getScalarSizeInBits() - 1);
714
715 // Shift % BitWidth
716 // -> Shift & (BitWidth - 1)
717 // -> Shift & Mask
718 Value *MaskedShift = Builder.CreateAnd(LHS: Shift, RHS: Mask);
719
720 // (BitWidth - 1) - (Shift % BitWidth)
721 // -> ~Shift & (BitWidth - 1)
722 // -> ~Shift & Mask
723 Value *NotShift = Builder.CreateNot(V: Shift);
724 Value *InverseShift = Builder.CreateAnd(LHS: NotShift, RHS: Mask);
725
726 Constant *One = ConstantInt::get(Ty, V: 1);
727 Value *ShiftedA;
728 Value *ShiftedB;
729
730 if (LeftFunnel) {
731 ShiftedA = Builder.CreateShl(LHS: A, RHS: MaskedShift);
732 Value *ShiftB1 = Builder.CreateLShr(LHS: B, RHS: One);
733 ShiftedB = Builder.CreateLShr(LHS: ShiftB1, RHS: InverseShift);
734 } else {
735 Value *ShiftA1 = Builder.CreateShl(LHS: A, RHS: One);
736 ShiftedA = Builder.CreateShl(LHS: ShiftA1, RHS: InverseShift);
737 ShiftedB = Builder.CreateLShr(LHS: B, RHS: MaskedShift);
738 }
739
740 Value *Result = Builder.CreateOr(LHS: ShiftedA, RHS: ShiftedB);
741 return Result;
742}
743
744static Value *expandPowIntrinsic(CallInst *Orig, Intrinsic::ID IntrinsicId) {
745
746 Value *X = Orig->getOperand(i_nocapture: 0);
747 Value *Y = Orig->getOperand(i_nocapture: 1);
748 Type *Ty = X->getType();
749 IRBuilder<> Builder(Orig);
750
751 if (IntrinsicId == Intrinsic::powi)
752 Y = Builder.CreateSIToFP(V: Y, DestTy: Ty);
753
754 Value *Log2Call =
755 Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::log2, Args: {X}, FMFSource: nullptr, Name: "elt.log2");
756 auto *Mul = Builder.CreateFMul(L: Log2Call, R: Y);
757 CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding(
758 RetTy: Ty, ID: Intrinsic::exp2, Args: {Mul}, FMFSource: nullptr, Name: "elt.exp2");
759 Exp2Call->setTailCall(Orig->isTailCall());
760 Exp2Call->setAttributes(Orig->getAttributes());
761 return Exp2Call;
762}
763
764static bool expandBufferLoadIntrinsic(CallInst *Orig, bool IsRaw) {
765 IRBuilder<> Builder(Orig);
766
767 Type *BufferTy = Orig->getType()->getStructElementType(N: 0);
768 Type *ScalarTy = BufferTy->getScalarType();
769 bool IsDouble = ScalarTy->isDoubleTy();
770 assert(IsDouble || ScalarTy->isIntegerTy(64) &&
771 "Only expand double or int64 scalars or vectors");
772 bool IsVector = false;
773 unsigned ExtractNum = 2;
774 if (auto *VT = dyn_cast<FixedVectorType>(Val: BufferTy)) {
775 ExtractNum = 2 * VT->getNumElements();
776 IsVector = true;
777 assert(IsRaw || ExtractNum == 4 && "TypedBufferLoad vector must be size 2");
778 }
779
780 SmallVector<Value *, 2> Loads;
781 Value *Result = PoisonValue::get(T: BufferTy);
782 unsigned Base = 0;
783 // If we need to extract more than 4 i32; we need to break it up into
784 // more than one load. LoadNum tells us how many i32s we are loading in
785 // each load
786 while (ExtractNum > 0) {
787 unsigned LoadNum = std::min(a: ExtractNum, b: 4u);
788 Type *Ty = VectorType::get(ElementType: Builder.getInt32Ty(), NumElements: LoadNum, Scalable: false);
789
790 Type *LoadType = StructType::get(elt1: Ty, elts: Builder.getInt1Ty());
791 Intrinsic::ID LoadIntrinsic = Intrinsic::dx_resource_load_typedbuffer;
792 SmallVector<Value *, 3> Args = {Orig->getOperand(i_nocapture: 0), Orig->getOperand(i_nocapture: 1)};
793 if (IsRaw) {
794 LoadIntrinsic = Intrinsic::dx_resource_load_rawbuffer;
795 Value *Tmp = Builder.getInt32(C: 4 * Base * 2);
796 Value *Offset = Orig->getOperand(i_nocapture: 2);
797 Args.push_back(Elt: Offset);
798 unsigned AddressArg = isa<PoisonValue>(Val: Offset) ? 1 : 2;
799 if (Base != 0)
800 Args[AddressArg] = Builder.CreateAdd(LHS: Args[AddressArg], RHS: Tmp);
801 }
802
803 Value *Load = Builder.CreateIntrinsic(RetTy: LoadType, ID: LoadIntrinsic, Args);
804 Loads.push_back(Elt: Load);
805
806 // extract the buffer load's result
807 Value *Extract = Builder.CreateExtractValue(Agg: Load, Idxs: {0});
808
809 SmallVector<Value *> ExtractElements;
810 for (unsigned I = 0; I < LoadNum; ++I)
811 ExtractElements.push_back(
812 Elt: Builder.CreateExtractElement(Vec: Extract, Idx: Builder.getInt32(C: I)));
813
814 // combine into double(s) or int64(s)
815 for (unsigned I = 0; I < LoadNum; I += 2) {
816 Value *Combined = nullptr;
817 if (IsDouble)
818 // For doubles, use dx_asdouble intrinsic
819 Combined = Builder.CreateIntrinsic(
820 RetTy: Builder.getDoubleTy(), ID: Intrinsic::dx_asdouble,
821 Args: {ExtractElements[I], ExtractElements[I + 1]});
822 else {
823 // For int64, manually combine two int32s
824 // First, zero-extend both values to i64
825 Value *Lo =
826 Builder.CreateZExt(V: ExtractElements[I], DestTy: Builder.getInt64Ty());
827 Value *Hi =
828 Builder.CreateZExt(V: ExtractElements[I + 1], DestTy: Builder.getInt64Ty());
829 // Shift the high bits left by 32 bits
830 Value *ShiftedHi = Builder.CreateShl(LHS: Hi, RHS: Builder.getInt64(C: 32));
831 // OR the high and low bits together
832 Combined = Builder.CreateOr(LHS: Lo, RHS: ShiftedHi);
833 }
834
835 if (IsVector)
836 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Combined,
837 Idx: Builder.getInt32(C: (I / 2) + Base));
838 else
839 Result = Combined;
840 }
841
842 ExtractNum -= LoadNum;
843 Base += LoadNum / 2;
844 }
845
846 Value *CheckBit = nullptr;
847 for (User *U : make_early_inc_range(Range: Orig->users())) {
848 // If it's not a ExtractValueInst, we don't know how to
849 // handle it
850 auto *EVI = dyn_cast<ExtractValueInst>(Val: U);
851 if (!EVI)
852 llvm_unreachable("Unexpected user of typedbufferload");
853
854 ArrayRef<unsigned> Indices = EVI->getIndices();
855 assert(Indices.size() == 1);
856
857 if (Indices[0] == 0) {
858 // Use of the value(s)
859 EVI->replaceAllUsesWith(V: Result);
860 } else {
861 // Use of the check bit
862 assert(Indices[0] == 1 && "Unexpected type for typedbufferload");
863 // Note: This does not always match the historical behaviour of DXC.
864 // See https://github.com/microsoft/DirectXShaderCompiler/issues/7622
865 if (!CheckBit) {
866 SmallVector<Value *, 2> CheckBits;
867 for (Value *L : Loads)
868 CheckBits.push_back(Elt: Builder.CreateExtractValue(Agg: L, Idxs: {1}));
869 CheckBit = Builder.CreateAnd(Ops: CheckBits);
870 }
871 EVI->replaceAllUsesWith(V: CheckBit);
872 }
873 EVI->eraseFromParent();
874 }
875 Orig->eraseFromParent();
876 return true;
877}
878
879static bool expandBufferStoreIntrinsic(CallInst *Orig, bool IsRaw) {
880 IRBuilder<> Builder(Orig);
881
882 unsigned ValIndex = IsRaw ? 3 : 2;
883 Type *BufferTy = Orig->getFunctionType()->getParamType(i: ValIndex);
884 Type *ScalarTy = BufferTy->getScalarType();
885 bool IsDouble = ScalarTy->isDoubleTy();
886 assert((IsDouble || ScalarTy->isIntegerTy(64)) &&
887 "Only expand double or int64 scalars or vectors");
888
889 // Determine if we're dealing with a vector or scalar
890 bool IsVector = false;
891 unsigned ExtractNum = 2;
892 unsigned VecLen = 0;
893 if (auto *VT = dyn_cast<FixedVectorType>(Val: BufferTy)) {
894 VecLen = VT->getNumElements();
895 assert(IsRaw || VecLen == 2 && "TypedBufferStore vector must be size 2");
896 ExtractNum = VecLen * 2;
897 IsVector = true;
898 }
899
900 // Create the appropriate vector type for the result
901 Type *Int32Ty = Builder.getInt32Ty();
902 Type *ResultTy = VectorType::get(ElementType: Int32Ty, NumElements: ExtractNum, Scalable: false);
903 Value *Val = PoisonValue::get(T: ResultTy);
904
905 Type *SplitElementTy = Int32Ty;
906 if (IsVector)
907 SplitElementTy = VectorType::get(ElementType: SplitElementTy, NumElements: VecLen, Scalable: false);
908
909 Value *LowBits = nullptr;
910 Value *HighBits = nullptr;
911 // Split the 64-bit values into 32-bit components
912 if (IsDouble) {
913 auto *SplitTy = llvm::StructType::get(elt1: SplitElementTy, elts: SplitElementTy);
914 Value *Split = Builder.CreateIntrinsic(RetTy: SplitTy, ID: Intrinsic::dx_splitdouble,
915 Args: {Orig->getOperand(i_nocapture: ValIndex)});
916 LowBits = Builder.CreateExtractValue(Agg: Split, Idxs: 0);
917 HighBits = Builder.CreateExtractValue(Agg: Split, Idxs: 1);
918 } else {
919 // Handle int64 type(s)
920 Value *InputVal = Orig->getOperand(i_nocapture: ValIndex);
921 Constant *ShiftAmt = Builder.getInt64(C: 32);
922 if (IsVector)
923 ShiftAmt =
924 ConstantVector::getSplat(EC: ElementCount::getFixed(MinVal: VecLen), Elt: ShiftAmt);
925
926 // Split into low and high 32-bit parts
927 LowBits = Builder.CreateTrunc(V: InputVal, DestTy: SplitElementTy);
928 Value *ShiftedVal = Builder.CreateLShr(LHS: InputVal, RHS: ShiftAmt);
929 HighBits = Builder.CreateTrunc(V: ShiftedVal, DestTy: SplitElementTy);
930 }
931
932 if (IsVector) {
933 SmallVector<int, 8> Mask;
934 for (unsigned I = 0; I < VecLen; ++I) {
935 Mask.push_back(Elt: I);
936 Mask.push_back(Elt: I + VecLen);
937 }
938 Val = Builder.CreateShuffleVector(V1: LowBits, V2: HighBits, Mask);
939 } else {
940 Val = Builder.CreateInsertElement(Vec: Val, NewElt: LowBits, Idx: Builder.getInt32(C: 0));
941 Val = Builder.CreateInsertElement(Vec: Val, NewElt: HighBits, Idx: Builder.getInt32(C: 1));
942 }
943
944 // If we need to extract more than 4 i32; we need to break it up into
945 // more than one store. StoreNum tells us how many i32s we are storing in
946 // each store
947 unsigned Base = 0;
948 while (ExtractNum > 0) {
949 unsigned StoreNum = std::min(a: ExtractNum, b: 4u);
950
951 Intrinsic::ID StoreIntrinsic = Intrinsic::dx_resource_store_typedbuffer;
952 SmallVector<Value *, 4> Args = {Orig->getOperand(i_nocapture: 0), Orig->getOperand(i_nocapture: 1)};
953 if (IsRaw) {
954 StoreIntrinsic = Intrinsic::dx_resource_store_rawbuffer;
955 Value *Tmp = Builder.getInt32(C: 4 * Base);
956 Value *Offset = Orig->getOperand(i_nocapture: 2);
957 Args.push_back(Elt: Offset);
958 unsigned AddressArg = isa<PoisonValue>(Val: Offset) ? 1 : 2;
959 if (Base != 0)
960 Args[AddressArg] = Builder.CreateAdd(LHS: Args[AddressArg], RHS: Tmp);
961 }
962
963 SmallVector<int, 4> Mask;
964 for (unsigned I = 0; I < StoreNum; ++I) {
965 Mask.push_back(Elt: Base + I);
966 }
967
968 Value *SubVal = Val;
969 if (VecLen > 2)
970 SubVal = Builder.CreateShuffleVector(V: Val, Mask);
971
972 Args.push_back(Elt: SubVal);
973 // Create the final intrinsic call
974 Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(), ID: StoreIntrinsic, Args);
975
976 ExtractNum -= StoreNum;
977 Base += StoreNum;
978 }
979 Orig->eraseFromParent();
980 return true;
981}
982
983static Intrinsic::ID getMaxForClamp(Intrinsic::ID ClampIntrinsic) {
984 if (ClampIntrinsic == Intrinsic::dx_uclamp)
985 return Intrinsic::umax;
986 if (ClampIntrinsic == Intrinsic::dx_sclamp)
987 return Intrinsic::smax;
988 assert(ClampIntrinsic == Intrinsic::dx_nclamp);
989 return Intrinsic::maxnum;
990}
991
992static Intrinsic::ID getMinForClamp(Intrinsic::ID ClampIntrinsic) {
993 if (ClampIntrinsic == Intrinsic::dx_uclamp)
994 return Intrinsic::umin;
995 if (ClampIntrinsic == Intrinsic::dx_sclamp)
996 return Intrinsic::smin;
997 assert(ClampIntrinsic == Intrinsic::dx_nclamp);
998 return Intrinsic::minnum;
999}
1000
1001static Value *expandClampIntrinsic(CallInst *Orig,
1002 Intrinsic::ID ClampIntrinsic) {
1003 Value *X = Orig->getOperand(i_nocapture: 0);
1004 Value *Min = Orig->getOperand(i_nocapture: 1);
1005 Value *Max = Orig->getOperand(i_nocapture: 2);
1006 Type *Ty = X->getType();
1007 IRBuilder<> Builder(Orig);
1008 auto *MaxCall = Builder.CreateIntrinsic(RetTy: Ty, ID: getMaxForClamp(ClampIntrinsic),
1009 Args: {X, Min}, FMFSource: nullptr, Name: "dx.max");
1010 return Builder.CreateIntrinsic(RetTy: Ty, ID: getMinForClamp(ClampIntrinsic),
1011 Args: {MaxCall, Max}, FMFSource: nullptr, Name: "dx.min");
1012}
1013
1014static Value *expandSignIntrinsic(CallInst *Orig) {
1015 Value *X = Orig->getOperand(i_nocapture: 0);
1016 Type *Ty = X->getType();
1017 Type *ScalarTy = Ty->getScalarType();
1018 Type *RetTy = Orig->getType();
1019 Constant *Zero = Constant::getNullValue(Ty);
1020
1021 IRBuilder<> Builder(Orig);
1022
1023 Value *GT;
1024 Value *LT;
1025 if (ScalarTy->isFloatingPointTy()) {
1026 GT = Builder.CreateFCmpOLT(LHS: Zero, RHS: X);
1027 LT = Builder.CreateFCmpOLT(LHS: X, RHS: Zero);
1028 } else {
1029 assert(ScalarTy->isIntegerTy());
1030 GT = Builder.CreateICmpSLT(LHS: Zero, RHS: X);
1031 LT = Builder.CreateICmpSLT(LHS: X, RHS: Zero);
1032 }
1033
1034 Value *ZextGT = Builder.CreateZExt(V: GT, DestTy: RetTy);
1035 Value *ZextLT = Builder.CreateZExt(V: LT, DestTy: RetTy);
1036
1037 return Builder.CreateSub(LHS: ZextGT, RHS: ZextLT);
1038}
1039
1040// Expand llvm.copysign by combining the sign bit with the magnitude bits using
1041// bitwise operations.
1042static Value *expandCopySignIntrinsic(CallInst *Orig) {
1043 Value *Magnitude = Orig->getOperand(i_nocapture: 0);
1044 Value *Sign = Orig->getOperand(i_nocapture: 1);
1045 Type *Ty = Orig->getType();
1046
1047 IRBuilder<> Builder(Orig);
1048
1049 bool IsDouble = Ty->getScalarType()->isDoubleTy();
1050 unsigned BitWidth = IsDouble ? 32 : Ty->getScalarSizeInBits();
1051 Type *IntTy = Ty->getWithNewType(EltTy: Builder.getIntNTy(N: BitWidth));
1052
1053 auto CopySignBit = [&](Value *MagnitudeInt, Value *SignInt) {
1054 APInt SignMaskVal = APInt::getSignMask(BitWidth);
1055 // `ConstantInt::get` broadcasts to a splat when `IntTy` is a vector.
1056 Constant *SignMask = ConstantInt::get(Ty: IntTy, V: SignMaskVal);
1057 Constant *NotSignMask = ConstantInt::get(Ty: IntTy, V: ~SignMaskVal);
1058
1059 Value *MagnitudeBits = Builder.CreateAnd(LHS: MagnitudeInt, RHS: NotSignMask);
1060 Value *SignBits = Builder.CreateAnd(LHS: SignInt, RHS: SignMask);
1061 return Builder.CreateOr(LHS: MagnitudeBits, RHS: SignBits);
1062 };
1063
1064 // Avoid i64 bitwise ops, which require the Int64Ops shader feature.
1065 if (IsDouble) {
1066 auto *SplitTy = StructType::get(elt1: IntTy, elts: IntTy);
1067 Value *MagnitudeHalves = Builder.CreateIntrinsic(
1068 RetTy: SplitTy, ID: Intrinsic::dx_splitdouble, Args: {Magnitude});
1069 Value *SignHalves =
1070 Builder.CreateIntrinsic(RetTy: SplitTy, ID: Intrinsic::dx_splitdouble, Args: {Sign});
1071 Value *MagnitudeLow = Builder.CreateExtractValue(Agg: MagnitudeHalves, Idxs: 0);
1072 Value *MagnitudeHigh = Builder.CreateExtractValue(Agg: MagnitudeHalves, Idxs: 1);
1073 Value *SignHigh = Builder.CreateExtractValue(Agg: SignHalves, Idxs: 1);
1074
1075 Value *CombinedHigh = CopySignBit(MagnitudeHigh, SignHigh);
1076 return Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::dx_asdouble,
1077 Args: {MagnitudeLow, CombinedHigh});
1078 }
1079
1080 Value *MagnitudeInt = Builder.CreateBitCast(V: Magnitude, DestTy: IntTy);
1081 Value *SignInt = Builder.CreateBitCast(V: Sign, DestTy: IntTy);
1082 Value *CombinedInt = CopySignBit(MagnitudeInt, SignInt);
1083 return Builder.CreateBitCast(V: CombinedInt, DestTy: Ty);
1084}
1085
1086// Expand llvm.matrix.multiply by extracting row/column vectors and computing
1087// dot products.
1088// Result[r,c] = dot(row_r(LHS), col_c(RHS))
1089// Element (r,c) is at index c*NumRows + r (column-major).
1090static Value *expandMatrixMultiply(CallInst *Orig) {
1091 Value *LHS = Orig->getArgOperand(i: 0);
1092 Value *RHS = Orig->getArgOperand(i: 1);
1093 unsigned LHSRows = cast<ConstantInt>(Val: Orig->getArgOperand(i: 2))->getZExtValue();
1094 unsigned LHSCols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 3))->getZExtValue();
1095 unsigned RHSCols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 4))->getZExtValue();
1096
1097 auto *RetTy = cast<FixedVectorType>(Val: Orig->getType());
1098 Type *EltTy = RetTy->getElementType();
1099 bool IsFP = EltTy->isFloatingPointTy();
1100
1101 IRBuilder<> Builder(Orig);
1102
1103 // Column-major indexing:
1104 // LHS row R, element K: index = K * LHSRows + R
1105 // RHS col C, element K: index = C * LHSCols + K
1106 Value *Result = PoisonValue::get(T: RetTy);
1107
1108 // Extract all scalar elements from LHS and RHS once, then reuse them.
1109 unsigned LHSSize = LHSRows * LHSCols;
1110 unsigned RHSSize = LHSCols * RHSCols;
1111 SmallVector<Value *, 16> LHSElts(LHSSize);
1112 SmallVector<Value *, 16> RHSElts(RHSSize);
1113 for (unsigned I = 0; I < LHSSize; ++I)
1114 LHSElts[I] = Builder.CreateExtractElement(Vec: LHS, Idx: I);
1115 for (unsigned I = 0; I < RHSSize; ++I)
1116 RHSElts[I] = Builder.CreateExtractElement(Vec: RHS, Idx: I);
1117
1118 // Choose the appropriate scalar-arg dot intrinsic for floats.
1119 // K=1 and double types use scalar expansion instead.
1120 Intrinsic::ID FloatDotID = Intrinsic::not_intrinsic;
1121 bool UseScalarFP = IsFP && (EltTy->isDoubleTy() || LHSCols == 1);
1122 if (IsFP && !UseScalarFP) {
1123 switch (LHSCols) {
1124 case 2:
1125 FloatDotID = Intrinsic::dx_dot2;
1126 break;
1127 case 3:
1128 FloatDotID = Intrinsic::dx_dot3;
1129 break;
1130 case 4:
1131 FloatDotID = Intrinsic::dx_dot4;
1132 break;
1133 default:
1134 reportFatalUsageError(
1135 reason: "Invalid matrix inner dimension for dot product: must be 2-4");
1136 return nullptr;
1137 }
1138 }
1139
1140 for (unsigned C = 0; C < RHSCols; ++C) {
1141 for (unsigned R = 0; R < LHSRows; ++R) {
1142 // Gather row R from LHS and column C from RHS.
1143 SmallVector<Value *, 4> RowElts, ColElts;
1144 for (unsigned K = 0; K < LHSCols; ++K) {
1145 RowElts.push_back(Elt: LHSElts[K * LHSRows + R]);
1146 ColElts.push_back(Elt: RHSElts[C * LHSCols + K]);
1147 }
1148
1149 Value *Dot;
1150 if (UseScalarFP) {
1151 // Scalar fmul+fmuladd expansion for double types and K=1.
1152 Dot = Builder.CreateFMul(L: RowElts[0], R: ColElts[0]);
1153 for (unsigned K = 1; K < LHSCols; ++K)
1154 Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: Intrinsic::fmuladd,
1155 Args: {RowElts[K], ColElts[K], Dot});
1156 } else if (IsFP) {
1157 // Emit scalar-arg DXIL dot directly (dx.dot2/dx.dot3/dx.dot4).
1158 SmallVector<Value *, 8> Args;
1159 Args.append(in_start: RowElts.begin(), in_end: RowElts.end());
1160 Args.append(in_start: ColElts.begin(), in_end: ColElts.end());
1161 Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: FloatDotID, Args);
1162 } else {
1163 // Integer: emit multiply + imad chain.
1164 Dot = Builder.CreateMul(LHS: RowElts[0], RHS: ColElts[0]);
1165 for (unsigned K = 1; K < LHSCols; ++K)
1166 Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: Intrinsic::dx_imad,
1167 Args: {RowElts[K], ColElts[K], Dot});
1168 }
1169 unsigned ResIdx = C * LHSRows + R;
1170 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Dot, Idx: ResIdx);
1171 }
1172 }
1173 return Result;
1174}
1175
1176// Expand llvm.matrix.transpose as a shufflevector that permutes elements
1177// from column-major source to column-major transposed layout.
1178// Element (r,c) at index c*Rows + r moves to index r*Cols + c.
1179static Value *expandMatrixTranspose(CallInst *Orig) {
1180 Value *Mat = Orig->getArgOperand(i: 0);
1181 unsigned Rows = cast<ConstantInt>(Val: Orig->getArgOperand(i: 1))->getZExtValue();
1182 unsigned Cols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 2))->getZExtValue();
1183
1184 unsigned NumElts = Rows * Cols;
1185 SmallVector<int, 16> Mask(NumElts);
1186 for (unsigned I = 0; I < NumElts; ++I)
1187 Mask[I] = (I % Cols) * Rows + (I / Cols);
1188
1189 IRBuilder<> Builder(Orig);
1190 return Builder.CreateShuffleVector(V: Mat, Mask);
1191}
1192
1193// Scalarize a vector int_dx_store_output call into per-component scalar calls.
1194// The DXIL StoreOutput op is per-component; vector intrinsics are split here
1195// so that DXILOpLowering sees only scalar variants.
1196static bool expandStoreOutput(CallInst *Orig) {
1197 auto *VT = dyn_cast<FixedVectorType>(Val: Orig->getArgOperand(i: 3)->getType());
1198 if (!VT)
1199 return false; // already scalar, nothing to expand
1200
1201 IRBuilder<> Builder(Orig);
1202 Module *M = Orig->getModule();
1203 Type *Int8Ty = Builder.getInt8Ty();
1204 Type *Int32Ty = Builder.getInt32Ty();
1205 Type *ScalarTy = VT->getElementType();
1206 unsigned NumElems = VT->getNumElements();
1207
1208 Value *SigElementId = Orig->getArgOperand(i: 0);
1209 Value *RowIndex = Orig->getArgOperand(i: 1);
1210 Value *StartCol = Orig->getArgOperand(i: 2); // i8
1211 Value *Data = Orig->getArgOperand(i: 3);
1212 Value *StartColI32 = Builder.CreateZExt(V: StartCol, DestTy: Int32Ty);
1213
1214 Function *ScalarFn = Intrinsic::getOrInsertDeclaration(
1215 M, id: Intrinsic::dx_store_output, OverloadTys: {ScalarTy});
1216
1217 for (unsigned I = 0; I < NumElems; ++I) {
1218 Value *Scalar =
1219 Builder.CreateExtractElement(Vec: Data, Idx: ConstantInt::get(Ty: Int32Ty, V: I));
1220 Value *ColIdx =
1221 Builder.CreateAdd(LHS: StartColI32, RHS: ConstantInt::get(Ty: Int32Ty, V: I));
1222 Value *ColI8 = Builder.CreateTrunc(V: ColIdx, DestTy: Int8Ty);
1223 Builder.CreateCall(Callee: ScalarFn, Args: {SigElementId, RowIndex, ColI8, Scalar});
1224 }
1225
1226 Orig->eraseFromParent();
1227 return true;
1228}
1229
1230// Scalarize a vector int_dx_load_input call into per-component scalar calls
1231// and reassemble the vector. The DXIL LoadInput op is per-component.
1232static Value *expandLoadInput(CallInst *Orig) {
1233 auto *VT = dyn_cast<FixedVectorType>(Val: Orig->getType());
1234 if (!VT)
1235 return nullptr; // already scalar, nothing to expand
1236
1237 IRBuilder<> Builder(Orig);
1238 Module *M = Orig->getModule();
1239 Type *Int8Ty = Builder.getInt8Ty();
1240 Type *Int32Ty = Builder.getInt32Ty();
1241 Type *ScalarTy = VT->getElementType();
1242 unsigned NumElems = VT->getNumElements();
1243
1244 Value *SigElementId = Orig->getArgOperand(i: 0);
1245 Value *RowIndex = Orig->getArgOperand(i: 1);
1246 Value *StartCol = Orig->getArgOperand(i: 2); // i8
1247 Value *GsVertexOrPrimIndex = Orig->getArgOperand(i: 3);
1248 Value *StartColI32 = Builder.CreateZExt(V: StartCol, DestTy: Int32Ty);
1249
1250 Function *ScalarFn = Intrinsic::getOrInsertDeclaration(
1251 M, id: Intrinsic::dx_load_input, OverloadTys: {ScalarTy});
1252
1253 Value *Vec = PoisonValue::get(T: VT);
1254 for (unsigned I = 0; I < NumElems; ++I) {
1255 Value *ColIdx =
1256 Builder.CreateAdd(LHS: StartColI32, RHS: ConstantInt::get(Ty: Int32Ty, V: I));
1257 Value *ColI8 = Builder.CreateTrunc(V: ColIdx, DestTy: Int8Ty);
1258 Value *Scalar = Builder.CreateCall(
1259 Callee: ScalarFn, Args: {SigElementId, RowIndex, ColI8, GsVertexOrPrimIndex});
1260 Vec =
1261 Builder.CreateInsertElement(Vec, NewElt: Scalar, Idx: ConstantInt::get(Ty: Int32Ty, V: I));
1262 }
1263
1264 return Vec;
1265}
1266
1267static Value *expandQuadAll(CallInst *Orig) {
1268 IRBuilder<> Builder(Orig);
1269 SmallVector<OperandBundleDef, 1> Bundles;
1270 Orig->getOperandBundlesAsDefs(Defs&: Bundles);
1271
1272 Value *Cond =
1273 Builder.CreateSExt(V: Orig->getArgOperand(i: 0), DestTy: Builder.getInt32Ty());
1274 Value *Result = Cond;
1275 for (Intrinsic::ID ID :
1276 {Intrinsic::dx_quad_read_across_x, Intrinsic::dx_quad_read_across_y,
1277 Intrinsic::dx_quad_read_across_diagonal}) {
1278 Value *QuadValue = Builder.CreateIntrinsic(ID, OverloadTypes: {Builder.getInt32Ty()}, Args: Cond,
1279 FMFSource: nullptr, Name: "", OpBundles: Bundles);
1280 Result = Builder.CreateAnd(LHS: Result, RHS: QuadValue);
1281 }
1282 return Builder.CreateTrunc(V: Result, DestTy: Builder.getInt1Ty());
1283}
1284
1285static bool expandIntrinsic(Function &F, CallInst *Orig) {
1286 Value *Result = nullptr;
1287 Intrinsic::ID IntrinsicId = F.getIntrinsicID();
1288 switch (IntrinsicId) {
1289 case Intrinsic::abs:
1290 Result = expandAbs(Orig);
1291 break;
1292 case Intrinsic::assume:
1293 Orig->eraseFromParent();
1294 return true;
1295 case Intrinsic::atan2:
1296 Result = expandAtan2Intrinsic(Orig);
1297 break;
1298 case Intrinsic::copysign:
1299 Result = expandCopySignIntrinsic(Orig);
1300 break;
1301 case Intrinsic::fshl:
1302 Result = expandFunnelShiftIntrinsic<true>(Orig);
1303 break;
1304 case Intrinsic::fshr:
1305 Result = expandFunnelShiftIntrinsic<false>(Orig);
1306 break;
1307 case Intrinsic::exp:
1308 Result = expandExpIntrinsic(Orig);
1309 break;
1310 case Intrinsic::is_fpclass:
1311 Result = expandIsFPClass(Orig);
1312 break;
1313 case Intrinsic::log:
1314 Result = expandLogIntrinsic(Orig);
1315 break;
1316 case Intrinsic::log10:
1317 Result = expandLog10Intrinsic(Orig);
1318 break;
1319 case Intrinsic::pow:
1320 case Intrinsic::powi:
1321 Result = expandPowIntrinsic(Orig, IntrinsicId);
1322 break;
1323 case Intrinsic::vector_reduce_and:
1324 case Intrinsic::vector_reduce_or:
1325 Result = expandVecReduceAndOr(Orig, IntrinsicId);
1326 break;
1327 case Intrinsic::dx_uclamp:
1328 case Intrinsic::dx_sclamp:
1329 case Intrinsic::dx_nclamp:
1330 Result = expandClampIntrinsic(Orig, ClampIntrinsic: IntrinsicId);
1331 break;
1332 case Intrinsic::dx_isfinite:
1333 Result = expand16BitIsFinite(Orig);
1334 break;
1335 case Intrinsic::dx_isinf:
1336 Result = expand16BitIsInf(Orig);
1337 break;
1338 case Intrinsic::dx_isnan:
1339 Result = expand16BitIsNaN(Orig);
1340 break;
1341 case Intrinsic::dx_fdot:
1342 Result = expandFloatDotIntrinsic(Orig);
1343 break;
1344 case Intrinsic::dx_sdot:
1345 case Intrinsic::dx_udot:
1346 Result = expandIntegerDotIntrinsic(Orig, DotIntrinsic: IntrinsicId);
1347 break;
1348 case Intrinsic::dx_sign:
1349 Result = expandSignIntrinsic(Orig);
1350 break;
1351 case Intrinsic::dx_quad_all:
1352 Result = expandQuadAll(Orig);
1353 break;
1354 case Intrinsic::dx_load_input:
1355 Result = expandLoadInput(Orig);
1356 break;
1357 case Intrinsic::dx_store_output:
1358 if (expandStoreOutput(Orig))
1359 return true;
1360 break;
1361 case Intrinsic::dx_resource_load_rawbuffer:
1362 if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ true))
1363 return true;
1364 break;
1365 case Intrinsic::dx_resource_store_rawbuffer:
1366 if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ true))
1367 return true;
1368 break;
1369 case Intrinsic::dx_resource_load_typedbuffer:
1370 if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ false))
1371 return true;
1372 break;
1373 case Intrinsic::dx_resource_store_typedbuffer:
1374 if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ false))
1375 return true;
1376 break;
1377 case Intrinsic::usub_sat:
1378 Result = expandUsubSat(Orig);
1379 break;
1380 case Intrinsic::umul_with_overflow:
1381 case Intrinsic::smul_with_overflow:
1382 Result = expandMulWithOverflow(Orig, /*Signed=*/IntrinsicId ==
1383 Intrinsic::smul_with_overflow);
1384 break;
1385 case Intrinsic::vector_reduce_add:
1386 case Intrinsic::vector_reduce_fadd:
1387 Result = expandVecReduceAdd(Orig, IntrinsicId);
1388 break;
1389 case Intrinsic::matrix_multiply:
1390 Result = expandMatrixMultiply(Orig);
1391 break;
1392 case Intrinsic::matrix_transpose:
1393 Result = expandMatrixTranspose(Orig);
1394 break;
1395 }
1396 if (Result) {
1397 Orig->replaceAllUsesWith(V: Result);
1398 Orig->eraseFromParent();
1399 return true;
1400 }
1401 return false;
1402}
1403
1404static bool expansionIntrinsics(Module &M) {
1405 for (auto &F : make_early_inc_range(Range: M.functions())) {
1406 if (!isIntrinsicExpansion(F))
1407 continue;
1408 bool IntrinsicExpanded = false;
1409 for (User *U : make_early_inc_range(Range: F.users())) {
1410 auto *IntrinsicCall = dyn_cast<CallInst>(Val: U);
1411 if (!IntrinsicCall)
1412 continue;
1413 IntrinsicExpanded = expandIntrinsic(F, Orig: IntrinsicCall);
1414 }
1415 if (F.user_empty() && IntrinsicExpanded)
1416 F.eraseFromParent();
1417 }
1418 return true;
1419}
1420
1421PreservedAnalyses DXILIntrinsicExpansionPass::run(Module &M,
1422 ModuleAnalysisManager &) {
1423 if (expansionIntrinsics(M))
1424 return PreservedAnalyses::none();
1425 return PreservedAnalyses::all();
1426}
1427
1428bool DXILIntrinsicExpansionLegacy::runOnModule(Module &M) {
1429 return expansionIntrinsics(M);
1430}
1431
1432char DXILIntrinsicExpansionLegacy::ID = 0;
1433
1434INITIALIZE_PASS_BEGIN(DXILIntrinsicExpansionLegacy, DEBUG_TYPE,
1435 "DXIL Intrinsic Expansion", false, false)
1436INITIALIZE_PASS_END(DXILIntrinsicExpansionLegacy, DEBUG_TYPE,
1437 "DXIL Intrinsic Expansion", false, false)
1438
1439ModulePass *llvm::createDXILIntrinsicExpansionLegacyPass() {
1440 return new DXILIntrinsicExpansionLegacy();
1441}
1442