1//===- DXILIntrinsicExpansion.cpp - Prepare LLVM Module for DXIL encoding--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file This file contains DXIL intrinsic expansions for those that don't have
10// opcodes in DirectX Intermediate Language (DXIL).
11//===----------------------------------------------------------------------===//
12
13#include "DXILIntrinsicExpansion.h"
14#include "DirectX.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/CodeGen/Passes.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/InstrTypes.h"
22#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsDirectX.h"
26#include "llvm/IR/MatrixBuilder.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/PassManager.h"
29#include "llvm/IR/Type.h"
30#include "llvm/Pass.h"
31#include "llvm/Support/Casting.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/MathExtras.h"
34
35#define DEBUG_TYPE "dxil-intrinsic-expansion"
36
37using namespace llvm;
38
39class DXILIntrinsicExpansionLegacy : public ModulePass {
40
41public:
42 bool runOnModule(Module &M) override;
43 DXILIntrinsicExpansionLegacy() : ModulePass(ID) {}
44
45 static char ID; // Pass identification.
46};
47
48static bool resourceAccessNeeds64BitExpansion(Module *M, Type *OverloadTy,
49 bool IsRaw) {
50 if (IsRaw && M->getTargetTriple().getDXILVersion() > VersionTuple(1, 2))
51 return false;
52
53 Type *ScalarTy = OverloadTy->getScalarType();
54 return ScalarTy->isDoubleTy() || ScalarTy->isIntegerTy(BitWidth: 64);
55}
56
57static Value *expand16BitIsInf(CallInst *Orig) {
58 Module *M = Orig->getModule();
59 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
60 return nullptr;
61
62 Value *Val = Orig->getOperand(i_nocapture: 0);
63 Type *ValTy = Val->getType();
64 if (!ValTy->getScalarType()->isHalfTy())
65 return nullptr;
66
67 IRBuilder<> Builder(Orig);
68 Type *IType = Type::getInt16Ty(C&: M->getContext());
69 Constant *PosInf =
70 ValTy->isVectorTy()
71 ? ConstantVector::getSplat(
72 EC: ElementCount::getFixed(
73 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
74 Elt: ConstantInt::get(Ty: IType, V: 0x7c00))
75 : ConstantInt::get(Ty: IType, V: 0x7c00);
76
77 Constant *NegInf =
78 ValTy->isVectorTy()
79 ? ConstantVector::getSplat(
80 EC: ElementCount::getFixed(
81 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
82 Elt: ConstantInt::get(Ty: IType, V: 0xfc00))
83 : ConstantInt::get(Ty: IType, V: 0xfc00);
84
85 Value *IVal = Builder.CreateBitCast(V: Val, DestTy: PosInf->getType());
86 Value *B1 = Builder.CreateICmpEQ(LHS: IVal, RHS: PosInf);
87 Value *B2 = Builder.CreateICmpEQ(LHS: IVal, RHS: NegInf);
88 Value *B3 = Builder.CreateOr(LHS: B1, RHS: B2);
89 return B3;
90}
91
92static Value *expand16BitIsNaN(CallInst *Orig) {
93 Module *M = Orig->getModule();
94 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
95 return nullptr;
96
97 Value *Val = Orig->getOperand(i_nocapture: 0);
98 Type *ValTy = Val->getType();
99 if (!ValTy->getScalarType()->isHalfTy())
100 return nullptr;
101
102 IRBuilder<> Builder(Orig);
103 Type *IType = Type::getInt16Ty(C&: M->getContext());
104
105 Constant *ExpBitMask =
106 ValTy->isVectorTy()
107 ? ConstantVector::getSplat(
108 EC: ElementCount::getFixed(
109 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
110 Elt: ConstantInt::get(Ty: IType, V: 0x7c00))
111 : ConstantInt::get(Ty: IType, V: 0x7c00);
112 Constant *SigBitMask =
113 ValTy->isVectorTy()
114 ? ConstantVector::getSplat(
115 EC: ElementCount::getFixed(
116 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
117 Elt: ConstantInt::get(Ty: IType, V: 0x3ff))
118 : ConstantInt::get(Ty: IType, V: 0x3ff);
119
120 Constant *Zero =
121 ValTy->isVectorTy()
122 ? ConstantVector::getSplat(
123 EC: ElementCount::getFixed(
124 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
125 Elt: ConstantInt::get(Ty: IType, V: 0))
126 : ConstantInt::get(Ty: IType, V: 0);
127
128 Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType());
129 Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask);
130 Value *B1 = Builder.CreateICmpEQ(LHS: Exp, RHS: ExpBitMask);
131
132 Value *Sig = Builder.CreateAnd(LHS: IVal, RHS: SigBitMask);
133 Value *B2 = Builder.CreateICmpNE(LHS: Sig, RHS: Zero);
134 Value *B3 = Builder.CreateAnd(LHS: B1, RHS: B2);
135 return B3;
136}
137
138static Value *expand16BitIsFinite(CallInst *Orig) {
139 Module *M = Orig->getModule();
140 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
141 return nullptr;
142
143 Value *Val = Orig->getOperand(i_nocapture: 0);
144 Type *ValTy = Val->getType();
145 if (!ValTy->getScalarType()->isHalfTy())
146 return nullptr;
147
148 IRBuilder<> Builder(Orig);
149 Type *IType = Type::getInt16Ty(C&: M->getContext());
150
151 Constant *ExpBitMask =
152 ValTy->isVectorTy()
153 ? ConstantVector::getSplat(
154 EC: ElementCount::getFixed(
155 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
156 Elt: ConstantInt::get(Ty: IType, V: 0x7c00))
157 : ConstantInt::get(Ty: IType, V: 0x7c00);
158
159 Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType());
160 Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask);
161 Value *B1 = Builder.CreateICmpNE(LHS: Exp, RHS: ExpBitMask);
162 return B1;
163}
164
165static Value *expand16BitIsNormal(CallInst *Orig) {
166 Module *M = Orig->getModule();
167 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
168 return nullptr;
169
170 Value *Val = Orig->getOperand(i_nocapture: 0);
171 Type *ValTy = Val->getType();
172 if (!ValTy->getScalarType()->isHalfTy())
173 return nullptr;
174
175 IRBuilder<> Builder(Orig);
176 Type *IType = Type::getInt16Ty(C&: M->getContext());
177
178 Constant *ExpBitMask =
179 ValTy->isVectorTy()
180 ? ConstantVector::getSplat(
181 EC: ElementCount::getFixed(
182 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
183 Elt: ConstantInt::get(Ty: IType, V: 0x7c00))
184 : ConstantInt::get(Ty: IType, V: 0x7c00);
185 Constant *Zero =
186 ValTy->isVectorTy()
187 ? ConstantVector::getSplat(
188 EC: ElementCount::getFixed(
189 MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()),
190 Elt: ConstantInt::get(Ty: IType, V: 0))
191 : ConstantInt::get(Ty: IType, V: 0);
192
193 Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType());
194 Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask);
195 Value *NotAllZeroes = Builder.CreateICmpNE(LHS: Exp, RHS: Zero);
196 Value *NotAllOnes = Builder.CreateICmpNE(LHS: Exp, RHS: ExpBitMask);
197 Value *B1 = Builder.CreateAnd(LHS: NotAllZeroes, RHS: NotAllOnes);
198 return B1;
199}
200
201static bool shouldExpandFloatDotIntrinsic(Function &F) {
202 assert(F.getIntrinsicID() == Intrinsic::dx_fdot &&
203 "Function is not a dx.fdot intrinsic");
204 auto *ParamTy = cast<FixedVectorType>(Val: F.getFunctionType()->getParamType(i: 0));
205 return ParamTy->getNumElements() <= 4 ||
206 F.getParent()->getTargetTriple().getOSVersion() < VersionTuple(6, 9);
207}
208
209static bool isIntrinsicExpansion(Function &F) {
210 switch (F.getIntrinsicID()) {
211 case Intrinsic::assume:
212 case Intrinsic::abs:
213 case Intrinsic::atan2:
214 case Intrinsic::copysign:
215 case Intrinsic::fshl:
216 case Intrinsic::fshr:
217 case Intrinsic::exp:
218 case Intrinsic::is_fpclass:
219 case Intrinsic::log:
220 case Intrinsic::log10:
221 case Intrinsic::pow:
222 case Intrinsic::powi:
223 case Intrinsic::dx_all:
224 case Intrinsic::dx_any:
225 case Intrinsic::dx_uclamp:
226 case Intrinsic::dx_sclamp:
227 case Intrinsic::dx_nclamp:
228 case Intrinsic::dx_isinf:
229 case Intrinsic::dx_isnan:
230 case Intrinsic::dx_sdot:
231 case Intrinsic::dx_udot:
232 case Intrinsic::dx_sign:
233 case Intrinsic::usub_sat:
234 case Intrinsic::vector_reduce_add:
235 case Intrinsic::vector_reduce_fadd:
236 case Intrinsic::matrix_multiply:
237 case Intrinsic::matrix_transpose:
238 case Intrinsic::umul_with_overflow:
239 case Intrinsic::smul_with_overflow:
240 case Intrinsic::dx_load_input:
241 case Intrinsic::dx_store_output:
242 return true;
243 case Intrinsic::dx_fdot:
244 return shouldExpandFloatDotIntrinsic(F);
245 case Intrinsic::dx_resource_load_rawbuffer:
246 return resourceAccessNeeds64BitExpansion(
247 M: F.getParent(), OverloadTy: F.getReturnType()->getStructElementType(N: 0),
248 /*IsRaw*/ true);
249 case Intrinsic::dx_resource_load_typedbuffer:
250 return resourceAccessNeeds64BitExpansion(
251 M: F.getParent(), OverloadTy: F.getReturnType()->getStructElementType(N: 0),
252 /*IsRaw*/ false);
253 case Intrinsic::dx_resource_store_rawbuffer:
254 return resourceAccessNeeds64BitExpansion(
255 M: F.getParent(), OverloadTy: F.getFunctionType()->getParamType(i: 3), /*IsRaw*/ true);
256 case Intrinsic::dx_resource_store_typedbuffer:
257 return resourceAccessNeeds64BitExpansion(
258 M: F.getParent(), OverloadTy: F.getFunctionType()->getParamType(i: 2), /*IsRaw*/ false);
259 }
260 return false;
261}
262
263static Value *expandUsubSat(CallInst *Orig) {
264 Value *A = Orig->getArgOperand(i: 0);
265 Value *B = Orig->getArgOperand(i: 1);
266 Type *Ty = A->getType();
267
268 IRBuilder<> Builder(Orig);
269
270 Value *Cmp = Builder.CreateICmpULT(LHS: A, RHS: B, Name: "usub.cmp");
271 Value *Sub = Builder.CreateSub(LHS: A, RHS: B, Name: "usub.sub");
272 Value *Zero = ConstantInt::get(Ty, V: 0);
273 return Builder.CreateSelect(C: Cmp, True: Zero, False: Sub, Name: "usub.sat");
274}
275
276// Compute the high N bits of the 2N-bit unsigned product of two N-bit values
277// using only N-bit arithmetic, so we don't introduce a wider integer type that
278// may be unsupported in DXIL.
279static Value *createMulHighUnsigned(IRBuilder<> &Builder, Value *A, Value *B,
280 Type *Ty, unsigned BW) {
281 assert(BW % 2 == 0 && "high-half split needs symmetric halves");
282 unsigned Half = BW / 2;
283 Value *HalfShift = ConstantInt::get(Ty, V: Half);
284 Value *LoMask = ConstantInt::get(Ty, V: APInt::getLowBitsSet(numBits: BW, loBitsSet: Half));
285
286 Value *U0 = Builder.CreateAnd(LHS: A, RHS: LoMask);
287 Value *U1 = Builder.CreateLShr(LHS: A, RHS: HalfShift);
288 Value *V0 = Builder.CreateAnd(LHS: B, RHS: LoMask);
289 Value *V1 = Builder.CreateLShr(LHS: B, RHS: HalfShift);
290
291 Value *W0 = Builder.CreateMul(LHS: U0, RHS: V0);
292 Value *T = Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U1, RHS: V0),
293 RHS: Builder.CreateLShr(LHS: W0, RHS: HalfShift));
294 Value *W1 = Builder.CreateAnd(LHS: T, RHS: LoMask);
295 Value *W2 = Builder.CreateLShr(LHS: T, RHS: HalfShift);
296 W1 = Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U0, RHS: V1), RHS: W1);
297 return Builder.CreateAdd(LHS: Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U1, RHS: V1), RHS: W2),
298 RHS: Builder.CreateLShr(LHS: W1, RHS: HalfShift));
299}
300
301// Expand a {u,s}mul.with.overflow intrinsic. The low half of the result is a
302// plain multiply; overflow is derived from the high half of the double-width
303// product.
304static Value *expandMulWithOverflow(CallInst *Orig, bool Signed) {
305 IRBuilder<> Builder(Orig);
306 Value *A = Orig->getArgOperand(i: 0);
307 Value *B = Orig->getArgOperand(i: 1);
308 Type *Ty = A->getType();
309 unsigned BW = Ty->getScalarSizeInBits();
310
311 Value *Lo;
312 Value *Ov;
313
314 // A plain double-width multiply is simplest, but we avoid it once it would
315 // introduce a 64-bit (or wider) integer, which DXIL does not always support.
316 // For i32 we use the native DXIL IMul/UMul ops, which return the full product
317 // as two i32s; wider types fall back to a same-width high-half computation.
318 if (2 * BW <= 32) {
319 Lo = Builder.CreateMul(LHS: A, RHS: B);
320 Type *WideTy = Ty->getWithNewBitWidth(NewBitWidth: 2 * BW);
321 Value *WideA =
322 Signed ? Builder.CreateSExt(V: A, DestTy: WideTy) : Builder.CreateZExt(V: A, DestTy: WideTy);
323 Value *WideB =
324 Signed ? Builder.CreateSExt(V: B, DestTy: WideTy) : Builder.CreateZExt(V: B, DestTy: WideTy);
325 Value *Wide = Builder.CreateMul(LHS: WideA, RHS: WideB);
326 if (Signed) {
327 // Overflow when the full product doesn't fit back into BW signed bits.
328 Ov = Builder.CreateICmpNE(LHS: Wide, RHS: Builder.CreateSExt(V: Lo, DestTy: WideTy));
329 } else {
330 Value *Hi = Builder.CreateLShr(LHS: Wide, RHS: ConstantInt::get(Ty: WideTy, V: BW));
331 Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty: WideTy, V: 0));
332 }
333 } else if (BW == 32) {
334 // IMul/UMul return {high, low}; index 0 is the high 32 bits.
335 Type *ResTy = StructType::get(elt1: Ty, elts: Ty);
336 Intrinsic::ID IntrinsicID =
337 Signed ? Intrinsic::dx_imul : Intrinsic::dx_umul;
338 Value *Mul = Builder.CreateIntrinsic(RetTy: ResTy, ID: IntrinsicID, Args: {A, B});
339 Value *Hi = Builder.CreateExtractValue(Agg: Mul, Idxs: 0);
340 Lo = Builder.CreateExtractValue(Agg: Mul, Idxs: 1);
341 if (Signed)
342 Ov = Builder.CreateICmpNE(
343 LHS: Hi, RHS: Builder.CreateAShr(LHS: Lo, RHS: ConstantInt::get(Ty, V: BW - 1)));
344 else
345 Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty, V: 0));
346 } else {
347 Lo = Builder.CreateMul(LHS: A, RHS: B);
348 Value *Hi = createMulHighUnsigned(Builder, A, B, Ty, BW);
349 if (Signed) {
350 // Turn the unsigned high half into the signed one, then overflow means it
351 // isn't the sign extension of the low half.
352 Value *SignShift = ConstantInt::get(Ty, V: BW - 1);
353 Value *ASign = Builder.CreateAShr(LHS: A, RHS: SignShift);
354 Value *BSign = Builder.CreateAShr(LHS: B, RHS: SignShift);
355 Hi = Builder.CreateSub(LHS: Hi, RHS: Builder.CreateAnd(LHS: ASign, RHS: B));
356 Hi = Builder.CreateSub(LHS: Hi, RHS: Builder.CreateAnd(LHS: BSign, RHS: A));
357 Ov = Builder.CreateICmpNE(LHS: Hi, RHS: Builder.CreateAShr(LHS: Lo, RHS: SignShift));
358 } else {
359 Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty, V: 0));
360 }
361 }
362
363 Value *Agg = PoisonValue::get(T: Orig->getType());
364 Agg = Builder.CreateInsertValue(Agg, Val: Lo, Idxs: 0);
365 return Builder.CreateInsertValue(Agg, Val: Ov, Idxs: 1);
366}
367
368static Value *expandVecReduceAdd(CallInst *Orig, Intrinsic::ID IntrinsicId) {
369 assert(IntrinsicId == Intrinsic::vector_reduce_add ||
370 IntrinsicId == Intrinsic::vector_reduce_fadd);
371
372 IRBuilder<> Builder(Orig);
373 bool IsFAdd = (IntrinsicId == Intrinsic::vector_reduce_fadd);
374
375 Value *X = Orig->getOperand(i_nocapture: IsFAdd ? 1 : 0);
376 Type *Ty = X->getType();
377 auto *XVec = dyn_cast<FixedVectorType>(Val: Ty);
378 unsigned XVecSize = XVec->getNumElements();
379 Value *Sum = Builder.CreateExtractElement(Vec: X, Idx: static_cast<uint64_t>(0));
380
381 // Handle the initial start value for floating-point addition.
382 if (IsFAdd) {
383 Constant *StartValue = dyn_cast<Constant>(Val: Orig->getOperand(i_nocapture: 0));
384 if (StartValue && !StartValue->isNullValue())
385 Sum = Builder.CreateFAdd(L: Sum, R: StartValue);
386 }
387
388 // Accumulate the remaining vector elements.
389 for (unsigned I = 1; I < XVecSize; I++) {
390 Value *Elt = Builder.CreateExtractElement(Vec: X, Idx: I);
391 if (IsFAdd)
392 Sum = Builder.CreateFAdd(L: Sum, R: Elt);
393 else
394 Sum = Builder.CreateAdd(LHS: Sum, RHS: Elt);
395 }
396
397 return Sum;
398}
399
400static Value *expandAbs(CallInst *Orig) {
401 Value *X = Orig->getOperand(i_nocapture: 0);
402 IRBuilder<> Builder(Orig);
403 Type *Ty = X->getType();
404 Type *EltTy = Ty->getScalarType();
405 Constant *Zero = Ty->isVectorTy()
406 ? ConstantVector::getSplat(
407 EC: ElementCount::getFixed(
408 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
409 Elt: ConstantInt::get(Ty: EltTy, V: 0))
410 : ConstantInt::get(Ty: EltTy, V: 0);
411 auto *V = Builder.CreateSub(LHS: Zero, RHS: X);
412 return Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::smax, Args: {X, V}, FMFSource: nullptr,
413 Name: "dx.max");
414}
415
416// Create a DXIL dot2, dot3, or dot4 for the given operands.
417static Value *expandFloatDotChunk(CallInst *Orig, Value *A, Value *B) {
418 Type *ATy = A->getType();
419 [[maybe_unused]] Type *BTy = B->getType();
420 assert(ATy->isVectorTy() && BTy->isVectorTy());
421
422 IRBuilder<> Builder(Orig);
423
424 auto *AVec = dyn_cast<FixedVectorType>(Val: ATy);
425
426 assert(ATy->getScalarType()->isFloatingPointTy());
427
428 unsigned NumElts = AVec->getNumElements();
429 Intrinsic::ID DotIntrinsic;
430 switch (NumElts) {
431 case 2:
432 DotIntrinsic = Intrinsic::dx_dot2;
433 break;
434 case 3:
435 DotIntrinsic = Intrinsic::dx_dot3;
436 break;
437 case 4:
438 DotIntrinsic = Intrinsic::dx_dot4;
439 break;
440 default:
441 reportFatalUsageError(
442 reason: "Invalid dot product input vector: length is outside 2-4");
443 }
444
445 SmallVector<Value *> Args;
446 for (unsigned I = 0; I < NumElts; ++I)
447 Args.push_back(Elt: Builder.CreateExtractElement(Vec: A, Idx: Builder.getInt32(C: I)));
448 for (unsigned I = 0; I < NumElts; ++I)
449 Args.push_back(Elt: Builder.CreateExtractElement(Vec: B, Idx: Builder.getInt32(C: I)));
450 return Builder.CreateIntrinsic(RetTy: ATy->getScalarType(), ID: DotIntrinsic, Args,
451 FMFSource: nullptr, Name: "dot");
452}
453
454// Expand an arbitrary-width float dot into the minimum number of legal DXIL
455// dot2, dot3, and dot4 operations.
456static Value *expandFloatDotIntrinsic(CallInst *Orig) {
457 Value *A = Orig->getOperand(i_nocapture: 0);
458 Value *B = Orig->getOperand(i_nocapture: 1);
459 unsigned NumElts = cast<FixedVectorType>(Val: A->getType())->getNumElements();
460
461 // We return early here to avoid constructing unnecessary identity shuffles.
462 if (NumElts <= 4)
463 return expandFloatDotChunk(Orig, A, B);
464
465 assert(Orig->getModule()->getTargetTriple().getOSVersion() <
466 VersionTuple(6, 9) &&
467 "long fdot must not be expanded for shader model 6.9 or later");
468
469 IRBuilder<> Builder(Orig);
470 Value *Result = nullptr;
471 for (unsigned Offset = 0; Offset < NumElts;) {
472 unsigned Remaining = NumElts - Offset;
473 // Taking four is optimal unless it would leave an illegal one-element
474 // tail. In that case, take three and finish with dot2.
475 unsigned ChunkSize = Remaining == 5 ? 3 : std::min(a: Remaining, b: 4u);
476 SmallVector<int, 4> Mask;
477 for (unsigned I = 0; I < ChunkSize; ++I)
478 Mask.push_back(Elt: Offset + I);
479 Value *AChunk = Builder.CreateShuffleVector(V: A, Mask);
480 Value *BChunk = Builder.CreateShuffleVector(V: B, Mask);
481 Value *Chunk = expandFloatDotChunk(Orig, A: AChunk, B: BChunk);
482 Result = Result ? Builder.CreateFAdd(L: Result, R: Chunk, Name: "dot.add") : Chunk;
483 Offset += ChunkSize;
484 }
485 return Result;
486}
487
488// Expand integer dot product to multiply and add ops
489static Value *expandIntegerDotIntrinsic(CallInst *Orig,
490 Intrinsic::ID DotIntrinsic) {
491 assert(DotIntrinsic == Intrinsic::dx_sdot ||
492 DotIntrinsic == Intrinsic::dx_udot);
493 Value *A = Orig->getOperand(i_nocapture: 0);
494 Value *B = Orig->getOperand(i_nocapture: 1);
495 Type *ATy = A->getType();
496 [[maybe_unused]] Type *BTy = B->getType();
497 assert(ATy->isVectorTy() && BTy->isVectorTy());
498
499 IRBuilder<> Builder(Orig);
500
501 auto *AVec = dyn_cast<FixedVectorType>(Val: ATy);
502
503 assert(ATy->getScalarType()->isIntegerTy());
504
505 Value *Result;
506 Intrinsic::ID MadIntrinsic = DotIntrinsic == Intrinsic::dx_sdot
507 ? Intrinsic::dx_imad
508 : Intrinsic::dx_umad;
509 Value *Elt0 = Builder.CreateExtractElement(Vec: A, Idx: (uint64_t)0);
510 Value *Elt1 = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
511 Result = Builder.CreateMul(LHS: Elt0, RHS: Elt1);
512 for (unsigned I = 1; I < AVec->getNumElements(); I++) {
513 Elt0 = Builder.CreateExtractElement(Vec: A, Idx: I);
514 Elt1 = Builder.CreateExtractElement(Vec: B, Idx: I);
515 Result = Builder.CreateIntrinsic(RetTy: Result->getType(), ID: MadIntrinsic,
516 Args: ArrayRef<Value *>{Elt0, Elt1, Result},
517 FMFSource: nullptr, Name: "dx.mad");
518 }
519 return Result;
520}
521
522static Value *expandExpIntrinsic(CallInst *Orig) {
523 Value *X = Orig->getOperand(i_nocapture: 0);
524 IRBuilder<> Builder(Orig);
525 Type *Ty = X->getType();
526 Type *EltTy = Ty->getScalarType();
527 Constant *Log2eConst =
528 Ty->isVectorTy() ? ConstantVector::getSplat(
529 EC: ElementCount::getFixed(
530 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
531 Elt: ConstantFP::get(Ty: EltTy, V: numbers::log2ef))
532 : ConstantFP::get(Ty: EltTy, V: numbers::log2ef);
533 Value *NewX = Builder.CreateFMul(L: Log2eConst, R: X);
534 CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding(
535 RetTy: Ty, ID: Intrinsic::exp2, Args: {NewX}, FMFSource: nullptr, Name: "dx.exp2");
536 Exp2Call->setTailCall(Orig->isTailCall());
537 Exp2Call->setAttributes(Orig->getAttributes());
538 return Exp2Call;
539}
540
541static Value *expandIsFPClass(CallInst *Orig) {
542 Value *T = Orig->getArgOperand(i: 1);
543 auto *TCI = dyn_cast<ConstantInt>(Val: T);
544
545 // These FPClassTest cases have DXIL opcodes, so they will be handled in
546 // DXIL Op Lowering instead for all non f16 cases.
547 switch (TCI->getZExtValue()) {
548 case FPClassTest::fcInf:
549 return expand16BitIsInf(Orig);
550 case FPClassTest::fcNan:
551 return expand16BitIsNaN(Orig);
552 case FPClassTest::fcNormal:
553 return expand16BitIsNormal(Orig);
554 case FPClassTest::fcFinite:
555 return expand16BitIsFinite(Orig);
556 }
557
558 IRBuilder<> Builder(Orig);
559
560 Value *F = Orig->getArgOperand(i: 0);
561 Type *FTy = F->getType();
562 unsigned FNumElem = 0; // 0 => F is not a vector
563
564 unsigned BitWidth; // Bit width of F or the ElemTy of F
565 Type *BitCastTy; // An IntNTy of the same bitwidth as F or ElemTy of F
566
567 if (auto *FVecTy = dyn_cast<FixedVectorType>(Val: FTy)) {
568 Type *ElemTy = FVecTy->getElementType();
569 FNumElem = FVecTy->getNumElements();
570 BitWidth = ElemTy->getPrimitiveSizeInBits();
571 BitCastTy = FixedVectorType::get(ElementType: Builder.getIntNTy(N: BitWidth), NumElts: FNumElem);
572 } else {
573 BitWidth = FTy->getPrimitiveSizeInBits();
574 BitCastTy = Builder.getIntNTy(N: BitWidth);
575 }
576
577 Value *FBitCast = Builder.CreateBitCast(V: F, DestTy: BitCastTy);
578 switch (TCI->getZExtValue()) {
579 case FPClassTest::fcNegZero: {
580 Value *NegZero =
581 ConstantInt::get(Ty: Builder.getIntNTy(N: BitWidth), V: 1 << (BitWidth - 1),
582 /*IsSigned=*/true);
583 Value *RetVal;
584 if (FNumElem) {
585 Value *NegZeroSplat = Builder.CreateVectorSplat(NumElts: FNumElem, V: NegZero);
586 RetVal =
587 Builder.CreateICmpEQ(LHS: FBitCast, RHS: NegZeroSplat, Name: "is.fpclass.negzero");
588 } else
589 RetVal = Builder.CreateICmpEQ(LHS: FBitCast, RHS: NegZero, Name: "is.fpclass.negzero");
590 return RetVal;
591 }
592 default:
593 reportFatalUsageError(reason: "Unsupported FPClassTest");
594 }
595}
596
597static Value *expandAnyOrAllIntrinsic(CallInst *Orig,
598 Intrinsic::ID IntrinsicId) {
599 Value *X = Orig->getOperand(i_nocapture: 0);
600 IRBuilder<> Builder(Orig);
601 Type *Ty = X->getType();
602 Type *EltTy = Ty->getScalarType();
603
604 auto ApplyOp = [&Builder](Intrinsic::ID IntrinsicId, Value *Result,
605 Value *Elt) {
606 if (IntrinsicId == Intrinsic::dx_any)
607 return Builder.CreateOr(LHS: Result, RHS: Elt);
608 assert(IntrinsicId == Intrinsic::dx_all);
609 return Builder.CreateAnd(LHS: Result, RHS: Elt);
610 };
611
612 Value *Result = nullptr;
613 if (!Ty->isVectorTy()) {
614 Result = EltTy->isFloatingPointTy()
615 ? Builder.CreateFCmpUNE(LHS: X, RHS: ConstantFP::get(Ty: EltTy, V: 0))
616 : Builder.CreateICmpNE(LHS: X, RHS: ConstantInt::get(Ty: EltTy, V: 0));
617 } else {
618 auto *XVec = dyn_cast<FixedVectorType>(Val: Ty);
619 Value *Cond =
620 EltTy->isFloatingPointTy()
621 ? Builder.CreateFCmpUNE(
622 LHS: X, RHS: ConstantVector::getSplat(
623 EC: ElementCount::getFixed(MinVal: XVec->getNumElements()),
624 Elt: ConstantFP::get(Ty: EltTy, V: 0)))
625 : Builder.CreateICmpNE(
626 LHS: X, RHS: ConstantVector::getSplat(
627 EC: ElementCount::getFixed(MinVal: XVec->getNumElements()),
628 Elt: ConstantInt::get(Ty: EltTy, V: 0)));
629 Result = Builder.CreateExtractElement(Vec: Cond, Idx: (uint64_t)0);
630 for (unsigned I = 1; I < XVec->getNumElements(); I++) {
631 Value *Elt = Builder.CreateExtractElement(Vec: Cond, Idx: I);
632 Result = ApplyOp(IntrinsicId, Result, Elt);
633 }
634 }
635 return Result;
636}
637
638static Value *expandLogIntrinsic(CallInst *Orig,
639 float LogConstVal = numbers::ln2f) {
640 Value *X = Orig->getOperand(i_nocapture: 0);
641 IRBuilder<> Builder(Orig);
642 Type *Ty = X->getType();
643 Type *EltTy = Ty->getScalarType();
644 Constant *Ln2Const =
645 Ty->isVectorTy() ? ConstantVector::getSplat(
646 EC: ElementCount::getFixed(
647 MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()),
648 Elt: ConstantFP::get(Ty: EltTy, V: LogConstVal))
649 : ConstantFP::get(Ty: EltTy, V: LogConstVal);
650 CallInst *Log2Call = Builder.CreateIntrinsicWithoutFolding(
651 RetTy: Ty, ID: Intrinsic::log2, Args: {X}, FMFSource: nullptr, Name: "elt.log2");
652 Log2Call->setTailCall(Orig->isTailCall());
653 Log2Call->setAttributes(Orig->getAttributes());
654 return Builder.CreateFMul(L: Ln2Const, R: Log2Call);
655}
656static Value *expandLog10Intrinsic(CallInst *Orig) {
657 return expandLogIntrinsic(Orig, LogConstVal: numbers::ln2f / numbers::ln10f);
658}
659
660static Value *expandAtan2Intrinsic(CallInst *Orig) {
661 Value *Y = Orig->getOperand(i_nocapture: 0);
662 Value *X = Orig->getOperand(i_nocapture: 1);
663 Type *Ty = X->getType();
664 IRBuilder<> Builder(Orig);
665 Builder.setFastMathFlags(Orig->getFastMathFlags());
666
667 Value *Tan = Builder.CreateFDiv(L: Y, R: X);
668
669 CallInst *Atan = Builder.CreateIntrinsicWithoutFolding(
670 RetTy: Ty, ID: Intrinsic::atan, Args: {Tan}, FMFSource: nullptr, Name: "Elt.Atan");
671 Atan->setTailCall(Orig->isTailCall());
672 Atan->setAttributes(Orig->getAttributes());
673
674 // Modify atan result based on https://en.wikipedia.org/wiki/Atan2.
675 Constant *Pi = ConstantFP::get(Ty, V: llvm::numbers::pi);
676 Constant *HalfPi = ConstantFP::get(Ty, V: llvm::numbers::pi / 2);
677 Constant *NegHalfPi = ConstantFP::get(Ty, V: -llvm::numbers::pi / 2);
678 Constant *Zero = ConstantFP::get(Ty, V: 0);
679 Value *AtanAddPi = Builder.CreateFAdd(L: Atan, R: Pi);
680 Value *AtanSubPi = Builder.CreateFSub(L: Atan, R: Pi);
681
682 // x > 0 -> atan.
683 Value *Result = Atan;
684 Value *XLt0 = Builder.CreateFCmpOLT(LHS: X, RHS: Zero);
685 Value *XEq0 = Builder.CreateFCmpOEQ(LHS: X, RHS: Zero);
686 Value *YGe0 = Builder.CreateFCmpOGE(LHS: Y, RHS: Zero);
687 Value *YLt0 = Builder.CreateFCmpOLT(LHS: Y, RHS: Zero);
688
689 // x < 0, y >= 0 -> atan + pi.
690 Value *XLt0AndYGe0 = Builder.CreateAnd(LHS: XLt0, RHS: YGe0);
691 Result = Builder.CreateSelect(C: XLt0AndYGe0, True: AtanAddPi, False: Result);
692
693 // x < 0, y < 0 -> atan - pi.
694 Value *XLt0AndYLt0 = Builder.CreateAnd(LHS: XLt0, RHS: YLt0);
695 Result = Builder.CreateSelect(C: XLt0AndYLt0, True: AtanSubPi, False: Result);
696
697 // x == 0, y < 0 -> -pi/2
698 Value *XEq0AndYLt0 = Builder.CreateAnd(LHS: XEq0, RHS: YLt0);
699 Result = Builder.CreateSelect(C: XEq0AndYLt0, True: NegHalfPi, False: Result);
700
701 // x == 0, y > 0 -> pi/2
702 Value *XEq0AndYGe0 = Builder.CreateAnd(LHS: XEq0, RHS: YGe0);
703 Result = Builder.CreateSelect(C: XEq0AndYGe0, True: HalfPi, False: Result);
704
705 return Result;
706}
707
708template <bool LeftFunnel>
709static Value *expandFunnelShiftIntrinsic(CallInst *Orig) {
710 Type *Ty = Orig->getType();
711 Value *A = Orig->getOperand(i_nocapture: 0);
712 Value *B = Orig->getOperand(i_nocapture: 1);
713 Value *Shift = Orig->getOperand(i_nocapture: 2);
714
715 IRBuilder<> Builder(Orig);
716
717 assert(llvm::isPowerOf2_32(Ty->getScalarSizeInBits()) &&
718 "Can't use Mask to compute modulo and inverse");
719
720 // Note: if (Shift % BitWidth) == 0 then (BitWidth - Shift) == BitWidth,
721 // shifting by the bitwidth for shl/lshr returns a poisoned result. As such,
722 // we implement the same formula as LegalizerHelper::lowerFunnelShiftAsShifts.
723 //
724 // The funnel shift is expanded like so:
725 // fshl
726 // -> msb_extract((concat(A, B) << (Shift % BitWidth)), BitWidth)
727 // -> A << (Shift % BitWidth) | B >> 1 >> (BitWidth - 1 - (Shift % BitWidth))
728 // fshr
729 // -> lsb_extract((concat(A, B) >> (Shift % BitWidth), BitWidth))
730 // -> A << 1 << (BitWidth - 1 - (Shift % BitWidth)) | B >> (Shift % BitWidth)
731
732 // (BitWidth - 1) -> Mask
733 Constant *Mask = ConstantInt::get(Ty, V: Ty->getScalarSizeInBits() - 1);
734
735 // Shift % BitWidth
736 // -> Shift & (BitWidth - 1)
737 // -> Shift & Mask
738 Value *MaskedShift = Builder.CreateAnd(LHS: Shift, RHS: Mask);
739
740 // (BitWidth - 1) - (Shift % BitWidth)
741 // -> ~Shift & (BitWidth - 1)
742 // -> ~Shift & Mask
743 Value *NotShift = Builder.CreateNot(V: Shift);
744 Value *InverseShift = Builder.CreateAnd(LHS: NotShift, RHS: Mask);
745
746 Constant *One = ConstantInt::get(Ty, V: 1);
747 Value *ShiftedA;
748 Value *ShiftedB;
749
750 if (LeftFunnel) {
751 ShiftedA = Builder.CreateShl(LHS: A, RHS: MaskedShift);
752 Value *ShiftB1 = Builder.CreateLShr(LHS: B, RHS: One);
753 ShiftedB = Builder.CreateLShr(LHS: ShiftB1, RHS: InverseShift);
754 } else {
755 Value *ShiftA1 = Builder.CreateShl(LHS: A, RHS: One);
756 ShiftedA = Builder.CreateShl(LHS: ShiftA1, RHS: InverseShift);
757 ShiftedB = Builder.CreateLShr(LHS: B, RHS: MaskedShift);
758 }
759
760 Value *Result = Builder.CreateOr(LHS: ShiftedA, RHS: ShiftedB);
761 return Result;
762}
763
764static Value *expandPowIntrinsic(CallInst *Orig, Intrinsic::ID IntrinsicId) {
765
766 Value *X = Orig->getOperand(i_nocapture: 0);
767 Value *Y = Orig->getOperand(i_nocapture: 1);
768 Type *Ty = X->getType();
769 IRBuilder<> Builder(Orig);
770
771 if (IntrinsicId == Intrinsic::powi)
772 Y = Builder.CreateSIToFP(V: Y, DestTy: Ty);
773
774 Value *Log2Call =
775 Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::log2, Args: {X}, FMFSource: nullptr, Name: "elt.log2");
776 auto *Mul = Builder.CreateFMul(L: Log2Call, R: Y);
777 CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding(
778 RetTy: Ty, ID: Intrinsic::exp2, Args: {Mul}, FMFSource: nullptr, Name: "elt.exp2");
779 Exp2Call->setTailCall(Orig->isTailCall());
780 Exp2Call->setAttributes(Orig->getAttributes());
781 return Exp2Call;
782}
783
784static bool expandBufferLoadIntrinsic(CallInst *Orig, bool IsRaw) {
785 IRBuilder<> Builder(Orig);
786
787 Type *BufferTy = Orig->getType()->getStructElementType(N: 0);
788 Type *ScalarTy = BufferTy->getScalarType();
789 bool IsDouble = ScalarTy->isDoubleTy();
790 assert(IsDouble || ScalarTy->isIntegerTy(64) &&
791 "Only expand double or int64 scalars or vectors");
792 bool IsVector = false;
793 unsigned ExtractNum = 2;
794 if (auto *VT = dyn_cast<FixedVectorType>(Val: BufferTy)) {
795 ExtractNum = 2 * VT->getNumElements();
796 IsVector = true;
797 assert(IsRaw || ExtractNum == 4 && "TypedBufferLoad vector must be size 2");
798 }
799
800 SmallVector<Value *, 2> Loads;
801 Value *Result = PoisonValue::get(T: BufferTy);
802 unsigned Base = 0;
803 // If we need to extract more than 4 i32; we need to break it up into
804 // more than one load. LoadNum tells us how many i32s we are loading in
805 // each load
806 while (ExtractNum > 0) {
807 unsigned LoadNum = std::min(a: ExtractNum, b: 4u);
808 Type *Ty = VectorType::get(ElementType: Builder.getInt32Ty(), NumElements: LoadNum, Scalable: false);
809
810 Type *LoadType = StructType::get(elt1: Ty, elts: Builder.getInt1Ty());
811 Intrinsic::ID LoadIntrinsic = Intrinsic::dx_resource_load_typedbuffer;
812 SmallVector<Value *, 3> Args = {Orig->getOperand(i_nocapture: 0), Orig->getOperand(i_nocapture: 1)};
813 if (IsRaw) {
814 LoadIntrinsic = Intrinsic::dx_resource_load_rawbuffer;
815 Value *Tmp = Builder.getInt32(C: 4 * Base * 2);
816 Args.push_back(Elt: Builder.CreateAdd(LHS: Orig->getOperand(i_nocapture: 2), RHS: Tmp));
817 }
818
819 Value *Load = Builder.CreateIntrinsic(RetTy: LoadType, ID: LoadIntrinsic, Args);
820 Loads.push_back(Elt: Load);
821
822 // extract the buffer load's result
823 Value *Extract = Builder.CreateExtractValue(Agg: Load, Idxs: {0});
824
825 SmallVector<Value *> ExtractElements;
826 for (unsigned I = 0; I < LoadNum; ++I)
827 ExtractElements.push_back(
828 Elt: Builder.CreateExtractElement(Vec: Extract, Idx: Builder.getInt32(C: I)));
829
830 // combine into double(s) or int64(s)
831 for (unsigned I = 0; I < LoadNum; I += 2) {
832 Value *Combined = nullptr;
833 if (IsDouble)
834 // For doubles, use dx_asdouble intrinsic
835 Combined = Builder.CreateIntrinsic(
836 RetTy: Builder.getDoubleTy(), ID: Intrinsic::dx_asdouble,
837 Args: {ExtractElements[I], ExtractElements[I + 1]});
838 else {
839 // For int64, manually combine two int32s
840 // First, zero-extend both values to i64
841 Value *Lo =
842 Builder.CreateZExt(V: ExtractElements[I], DestTy: Builder.getInt64Ty());
843 Value *Hi =
844 Builder.CreateZExt(V: ExtractElements[I + 1], DestTy: Builder.getInt64Ty());
845 // Shift the high bits left by 32 bits
846 Value *ShiftedHi = Builder.CreateShl(LHS: Hi, RHS: Builder.getInt64(C: 32));
847 // OR the high and low bits together
848 Combined = Builder.CreateOr(LHS: Lo, RHS: ShiftedHi);
849 }
850
851 if (IsVector)
852 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Combined,
853 Idx: Builder.getInt32(C: (I / 2) + Base));
854 else
855 Result = Combined;
856 }
857
858 ExtractNum -= LoadNum;
859 Base += LoadNum / 2;
860 }
861
862 Value *CheckBit = nullptr;
863 for (User *U : make_early_inc_range(Range: Orig->users())) {
864 // If it's not a ExtractValueInst, we don't know how to
865 // handle it
866 auto *EVI = dyn_cast<ExtractValueInst>(Val: U);
867 if (!EVI)
868 llvm_unreachable("Unexpected user of typedbufferload");
869
870 ArrayRef<unsigned> Indices = EVI->getIndices();
871 assert(Indices.size() == 1);
872
873 if (Indices[0] == 0) {
874 // Use of the value(s)
875 EVI->replaceAllUsesWith(V: Result);
876 } else {
877 // Use of the check bit
878 assert(Indices[0] == 1 && "Unexpected type for typedbufferload");
879 // Note: This does not always match the historical behaviour of DXC.
880 // See https://github.com/microsoft/DirectXShaderCompiler/issues/7622
881 if (!CheckBit) {
882 SmallVector<Value *, 2> CheckBits;
883 for (Value *L : Loads)
884 CheckBits.push_back(Elt: Builder.CreateExtractValue(Agg: L, Idxs: {1}));
885 CheckBit = Builder.CreateAnd(Ops: CheckBits);
886 }
887 EVI->replaceAllUsesWith(V: CheckBit);
888 }
889 EVI->eraseFromParent();
890 }
891 Orig->eraseFromParent();
892 return true;
893}
894
895static bool expandBufferStoreIntrinsic(CallInst *Orig, bool IsRaw) {
896 IRBuilder<> Builder(Orig);
897
898 unsigned ValIndex = IsRaw ? 3 : 2;
899 Type *BufferTy = Orig->getFunctionType()->getParamType(i: ValIndex);
900 Type *ScalarTy = BufferTy->getScalarType();
901 bool IsDouble = ScalarTy->isDoubleTy();
902 assert((IsDouble || ScalarTy->isIntegerTy(64)) &&
903 "Only expand double or int64 scalars or vectors");
904
905 // Determine if we're dealing with a vector or scalar
906 bool IsVector = false;
907 unsigned ExtractNum = 2;
908 unsigned VecLen = 0;
909 if (auto *VT = dyn_cast<FixedVectorType>(Val: BufferTy)) {
910 VecLen = VT->getNumElements();
911 assert(IsRaw || VecLen == 2 && "TypedBufferStore vector must be size 2");
912 ExtractNum = VecLen * 2;
913 IsVector = true;
914 }
915
916 // Create the appropriate vector type for the result
917 Type *Int32Ty = Builder.getInt32Ty();
918 Type *ResultTy = VectorType::get(ElementType: Int32Ty, NumElements: ExtractNum, Scalable: false);
919 Value *Val = PoisonValue::get(T: ResultTy);
920
921 Type *SplitElementTy = Int32Ty;
922 if (IsVector)
923 SplitElementTy = VectorType::get(ElementType: SplitElementTy, NumElements: VecLen, Scalable: false);
924
925 Value *LowBits = nullptr;
926 Value *HighBits = nullptr;
927 // Split the 64-bit values into 32-bit components
928 if (IsDouble) {
929 auto *SplitTy = llvm::StructType::get(elt1: SplitElementTy, elts: SplitElementTy);
930 Value *Split = Builder.CreateIntrinsic(RetTy: SplitTy, ID: Intrinsic::dx_splitdouble,
931 Args: {Orig->getOperand(i_nocapture: ValIndex)});
932 LowBits = Builder.CreateExtractValue(Agg: Split, Idxs: 0);
933 HighBits = Builder.CreateExtractValue(Agg: Split, Idxs: 1);
934 } else {
935 // Handle int64 type(s)
936 Value *InputVal = Orig->getOperand(i_nocapture: ValIndex);
937 Constant *ShiftAmt = Builder.getInt64(C: 32);
938 if (IsVector)
939 ShiftAmt =
940 ConstantVector::getSplat(EC: ElementCount::getFixed(MinVal: VecLen), Elt: ShiftAmt);
941
942 // Split into low and high 32-bit parts
943 LowBits = Builder.CreateTrunc(V: InputVal, DestTy: SplitElementTy);
944 Value *ShiftedVal = Builder.CreateLShr(LHS: InputVal, RHS: ShiftAmt);
945 HighBits = Builder.CreateTrunc(V: ShiftedVal, DestTy: SplitElementTy);
946 }
947
948 if (IsVector) {
949 SmallVector<int, 8> Mask;
950 for (unsigned I = 0; I < VecLen; ++I) {
951 Mask.push_back(Elt: I);
952 Mask.push_back(Elt: I + VecLen);
953 }
954 Val = Builder.CreateShuffleVector(V1: LowBits, V2: HighBits, Mask);
955 } else {
956 Val = Builder.CreateInsertElement(Vec: Val, NewElt: LowBits, Idx: Builder.getInt32(C: 0));
957 Val = Builder.CreateInsertElement(Vec: Val, NewElt: HighBits, Idx: Builder.getInt32(C: 1));
958 }
959
960 // If we need to extract more than 4 i32; we need to break it up into
961 // more than one store. StoreNum tells us how many i32s we are storing in
962 // each store
963 unsigned Base = 0;
964 while (ExtractNum > 0) {
965 unsigned StoreNum = std::min(a: ExtractNum, b: 4u);
966
967 Intrinsic::ID StoreIntrinsic = Intrinsic::dx_resource_store_typedbuffer;
968 SmallVector<Value *, 4> Args = {Orig->getOperand(i_nocapture: 0), Orig->getOperand(i_nocapture: 1)};
969 if (IsRaw) {
970 StoreIntrinsic = Intrinsic::dx_resource_store_rawbuffer;
971 Value *Tmp = Builder.getInt32(C: 4 * Base);
972 Value *Offset = Orig->getOperand(i_nocapture: 2);
973 Args.push_back(Elt: Offset);
974 unsigned AddressArg = isa<PoisonValue>(Val: Offset) ? 1 : 2;
975 if (Base != 0)
976 Args[AddressArg] = Builder.CreateAdd(LHS: Args[AddressArg], RHS: Tmp);
977 }
978
979 SmallVector<int, 4> Mask;
980 for (unsigned I = 0; I < StoreNum; ++I) {
981 Mask.push_back(Elt: Base + I);
982 }
983
984 Value *SubVal = Val;
985 if (VecLen > 2)
986 SubVal = Builder.CreateShuffleVector(V: Val, Mask);
987
988 Args.push_back(Elt: SubVal);
989 // Create the final intrinsic call
990 Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(), ID: StoreIntrinsic, Args);
991
992 ExtractNum -= StoreNum;
993 Base += StoreNum;
994 }
995 Orig->eraseFromParent();
996 return true;
997}
998
999static Intrinsic::ID getMaxForClamp(Intrinsic::ID ClampIntrinsic) {
1000 if (ClampIntrinsic == Intrinsic::dx_uclamp)
1001 return Intrinsic::umax;
1002 if (ClampIntrinsic == Intrinsic::dx_sclamp)
1003 return Intrinsic::smax;
1004 assert(ClampIntrinsic == Intrinsic::dx_nclamp);
1005 return Intrinsic::maxnum;
1006}
1007
1008static Intrinsic::ID getMinForClamp(Intrinsic::ID ClampIntrinsic) {
1009 if (ClampIntrinsic == Intrinsic::dx_uclamp)
1010 return Intrinsic::umin;
1011 if (ClampIntrinsic == Intrinsic::dx_sclamp)
1012 return Intrinsic::smin;
1013 assert(ClampIntrinsic == Intrinsic::dx_nclamp);
1014 return Intrinsic::minnum;
1015}
1016
1017static Value *expandClampIntrinsic(CallInst *Orig,
1018 Intrinsic::ID ClampIntrinsic) {
1019 Value *X = Orig->getOperand(i_nocapture: 0);
1020 Value *Min = Orig->getOperand(i_nocapture: 1);
1021 Value *Max = Orig->getOperand(i_nocapture: 2);
1022 Type *Ty = X->getType();
1023 IRBuilder<> Builder(Orig);
1024 auto *MaxCall = Builder.CreateIntrinsic(RetTy: Ty, ID: getMaxForClamp(ClampIntrinsic),
1025 Args: {X, Min}, FMFSource: nullptr, Name: "dx.max");
1026 return Builder.CreateIntrinsic(RetTy: Ty, ID: getMinForClamp(ClampIntrinsic),
1027 Args: {MaxCall, Max}, FMFSource: nullptr, Name: "dx.min");
1028}
1029
1030static Value *expandSignIntrinsic(CallInst *Orig) {
1031 Value *X = Orig->getOperand(i_nocapture: 0);
1032 Type *Ty = X->getType();
1033 Type *ScalarTy = Ty->getScalarType();
1034 Type *RetTy = Orig->getType();
1035 Constant *Zero = Constant::getNullValue(Ty);
1036
1037 IRBuilder<> Builder(Orig);
1038
1039 Value *GT;
1040 Value *LT;
1041 if (ScalarTy->isFloatingPointTy()) {
1042 GT = Builder.CreateFCmpOLT(LHS: Zero, RHS: X);
1043 LT = Builder.CreateFCmpOLT(LHS: X, RHS: Zero);
1044 } else {
1045 assert(ScalarTy->isIntegerTy());
1046 GT = Builder.CreateICmpSLT(LHS: Zero, RHS: X);
1047 LT = Builder.CreateICmpSLT(LHS: X, RHS: Zero);
1048 }
1049
1050 Value *ZextGT = Builder.CreateZExt(V: GT, DestTy: RetTy);
1051 Value *ZextLT = Builder.CreateZExt(V: LT, DestTy: RetTy);
1052
1053 return Builder.CreateSub(LHS: ZextGT, RHS: ZextLT);
1054}
1055
1056// Expand llvm.copysign by combining the sign bit with the magnitude bits using
1057// bitwise operations.
1058static Value *expandCopySignIntrinsic(CallInst *Orig) {
1059 Value *Magnitude = Orig->getOperand(i_nocapture: 0);
1060 Value *Sign = Orig->getOperand(i_nocapture: 1);
1061 Type *Ty = Orig->getType();
1062
1063 IRBuilder<> Builder(Orig);
1064
1065 bool IsDouble = Ty->getScalarType()->isDoubleTy();
1066 unsigned BitWidth = IsDouble ? 32 : Ty->getScalarSizeInBits();
1067 Type *IntTy = Ty->getWithNewType(EltTy: Builder.getIntNTy(N: BitWidth));
1068
1069 auto CopySignBit = [&](Value *MagnitudeInt, Value *SignInt) {
1070 APInt SignMaskVal = APInt::getSignMask(BitWidth);
1071 // `ConstantInt::get` broadcasts to a splat when `IntTy` is a vector.
1072 Constant *SignMask = ConstantInt::get(Ty: IntTy, V: SignMaskVal);
1073 Constant *NotSignMask = ConstantInt::get(Ty: IntTy, V: ~SignMaskVal);
1074
1075 Value *MagnitudeBits = Builder.CreateAnd(LHS: MagnitudeInt, RHS: NotSignMask);
1076 Value *SignBits = Builder.CreateAnd(LHS: SignInt, RHS: SignMask);
1077 return Builder.CreateOr(LHS: MagnitudeBits, RHS: SignBits);
1078 };
1079
1080 // Avoid i64 bitwise ops, which require the Int64Ops shader feature.
1081 if (IsDouble) {
1082 auto *SplitTy = StructType::get(elt1: IntTy, elts: IntTy);
1083 Value *MagnitudeHalves = Builder.CreateIntrinsic(
1084 RetTy: SplitTy, ID: Intrinsic::dx_splitdouble, Args: {Magnitude});
1085 Value *SignHalves =
1086 Builder.CreateIntrinsic(RetTy: SplitTy, ID: Intrinsic::dx_splitdouble, Args: {Sign});
1087 Value *MagnitudeLow = Builder.CreateExtractValue(Agg: MagnitudeHalves, Idxs: 0);
1088 Value *MagnitudeHigh = Builder.CreateExtractValue(Agg: MagnitudeHalves, Idxs: 1);
1089 Value *SignHigh = Builder.CreateExtractValue(Agg: SignHalves, Idxs: 1);
1090
1091 Value *CombinedHigh = CopySignBit(MagnitudeHigh, SignHigh);
1092 return Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::dx_asdouble,
1093 Args: {MagnitudeLow, CombinedHigh});
1094 }
1095
1096 Value *MagnitudeInt = Builder.CreateBitCast(V: Magnitude, DestTy: IntTy);
1097 Value *SignInt = Builder.CreateBitCast(V: Sign, DestTy: IntTy);
1098 Value *CombinedInt = CopySignBit(MagnitudeInt, SignInt);
1099 return Builder.CreateBitCast(V: CombinedInt, DestTy: Ty);
1100}
1101
1102// Expand llvm.matrix.multiply by extracting row/column vectors and computing
1103// dot products.
1104// Result[r,c] = dot(row_r(LHS), col_c(RHS))
1105// Element (r,c) is at index c*NumRows + r (column-major).
1106static Value *expandMatrixMultiply(CallInst *Orig) {
1107 Value *LHS = Orig->getArgOperand(i: 0);
1108 Value *RHS = Orig->getArgOperand(i: 1);
1109 unsigned LHSRows = cast<ConstantInt>(Val: Orig->getArgOperand(i: 2))->getZExtValue();
1110 unsigned LHSCols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 3))->getZExtValue();
1111 unsigned RHSCols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 4))->getZExtValue();
1112
1113 auto *RetTy = cast<FixedVectorType>(Val: Orig->getType());
1114 Type *EltTy = RetTy->getElementType();
1115 bool IsFP = EltTy->isFloatingPointTy();
1116
1117 IRBuilder<> Builder(Orig);
1118
1119 // Column-major indexing:
1120 // LHS row R, element K: index = K * LHSRows + R
1121 // RHS col C, element K: index = C * LHSCols + K
1122 Value *Result = PoisonValue::get(T: RetTy);
1123
1124 // Extract all scalar elements from LHS and RHS once, then reuse them.
1125 unsigned LHSSize = LHSRows * LHSCols;
1126 unsigned RHSSize = LHSCols * RHSCols;
1127 SmallVector<Value *, 16> LHSElts(LHSSize);
1128 SmallVector<Value *, 16> RHSElts(RHSSize);
1129 for (unsigned I = 0; I < LHSSize; ++I)
1130 LHSElts[I] = Builder.CreateExtractElement(Vec: LHS, Idx: I);
1131 for (unsigned I = 0; I < RHSSize; ++I)
1132 RHSElts[I] = Builder.CreateExtractElement(Vec: RHS, Idx: I);
1133
1134 // Choose the appropriate scalar-arg dot intrinsic for floats.
1135 // K=1 and double types use scalar expansion instead.
1136 Intrinsic::ID FloatDotID = Intrinsic::not_intrinsic;
1137 bool UseScalarFP = IsFP && (EltTy->isDoubleTy() || LHSCols == 1);
1138 if (IsFP && !UseScalarFP) {
1139 switch (LHSCols) {
1140 case 2:
1141 FloatDotID = Intrinsic::dx_dot2;
1142 break;
1143 case 3:
1144 FloatDotID = Intrinsic::dx_dot3;
1145 break;
1146 case 4:
1147 FloatDotID = Intrinsic::dx_dot4;
1148 break;
1149 default:
1150 reportFatalUsageError(
1151 reason: "Invalid matrix inner dimension for dot product: must be 2-4");
1152 return nullptr;
1153 }
1154 }
1155
1156 for (unsigned C = 0; C < RHSCols; ++C) {
1157 for (unsigned R = 0; R < LHSRows; ++R) {
1158 // Gather row R from LHS and column C from RHS.
1159 SmallVector<Value *, 4> RowElts, ColElts;
1160 for (unsigned K = 0; K < LHSCols; ++K) {
1161 RowElts.push_back(Elt: LHSElts[K * LHSRows + R]);
1162 ColElts.push_back(Elt: RHSElts[C * LHSCols + K]);
1163 }
1164
1165 Value *Dot;
1166 if (UseScalarFP) {
1167 // Scalar fmul+fmuladd expansion for double types and K=1.
1168 Dot = Builder.CreateFMul(L: RowElts[0], R: ColElts[0]);
1169 for (unsigned K = 1; K < LHSCols; ++K)
1170 Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: Intrinsic::fmuladd,
1171 Args: {RowElts[K], ColElts[K], Dot});
1172 } else if (IsFP) {
1173 // Emit scalar-arg DXIL dot directly (dx.dot2/dx.dot3/dx.dot4).
1174 SmallVector<Value *, 8> Args;
1175 Args.append(in_start: RowElts.begin(), in_end: RowElts.end());
1176 Args.append(in_start: ColElts.begin(), in_end: ColElts.end());
1177 Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: FloatDotID, Args);
1178 } else {
1179 // Integer: emit multiply + imad chain.
1180 Dot = Builder.CreateMul(LHS: RowElts[0], RHS: ColElts[0]);
1181 for (unsigned K = 1; K < LHSCols; ++K)
1182 Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: Intrinsic::dx_imad,
1183 Args: {RowElts[K], ColElts[K], Dot});
1184 }
1185 unsigned ResIdx = C * LHSRows + R;
1186 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Dot, Idx: ResIdx);
1187 }
1188 }
1189 return Result;
1190}
1191
1192// Expand llvm.matrix.transpose as a shufflevector that permutes elements
1193// from column-major source to column-major transposed layout.
1194// Element (r,c) at index c*Rows + r moves to index r*Cols + c.
1195static Value *expandMatrixTranspose(CallInst *Orig) {
1196 Value *Mat = Orig->getArgOperand(i: 0);
1197 unsigned Rows = cast<ConstantInt>(Val: Orig->getArgOperand(i: 1))->getZExtValue();
1198 unsigned Cols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 2))->getZExtValue();
1199
1200 unsigned NumElts = Rows * Cols;
1201 SmallVector<int, 16> Mask(NumElts);
1202 for (unsigned I = 0; I < NumElts; ++I)
1203 Mask[I] = (I % Cols) * Rows + (I / Cols);
1204
1205 IRBuilder<> Builder(Orig);
1206 return Builder.CreateShuffleVector(V: Mat, Mask);
1207}
1208
1209// Scalarize a vector int_dx_store_output call into per-component scalar calls.
1210// The DXIL StoreOutput op is per-component; vector intrinsics are split here
1211// so that DXILOpLowering sees only scalar variants.
1212static bool expandStoreOutput(CallInst *Orig) {
1213 auto *VT = dyn_cast<FixedVectorType>(Val: Orig->getArgOperand(i: 3)->getType());
1214 if (!VT)
1215 return false; // already scalar, nothing to expand
1216
1217 IRBuilder<> Builder(Orig);
1218 Module *M = Orig->getModule();
1219 Type *Int8Ty = Builder.getInt8Ty();
1220 Type *Int32Ty = Builder.getInt32Ty();
1221 Type *ScalarTy = VT->getElementType();
1222 unsigned NumElems = VT->getNumElements();
1223
1224 Value *SigElementId = Orig->getArgOperand(i: 0);
1225 Value *RowIndex = Orig->getArgOperand(i: 1);
1226 Value *StartCol = Orig->getArgOperand(i: 2); // i8
1227 Value *Data = Orig->getArgOperand(i: 3);
1228 Value *StartColI32 = Builder.CreateZExt(V: StartCol, DestTy: Int32Ty);
1229
1230 Function *ScalarFn = Intrinsic::getOrInsertDeclaration(
1231 M, id: Intrinsic::dx_store_output, OverloadTys: {ScalarTy});
1232
1233 for (unsigned I = 0; I < NumElems; ++I) {
1234 Value *Scalar =
1235 Builder.CreateExtractElement(Vec: Data, Idx: ConstantInt::get(Ty: Int32Ty, V: I));
1236 Value *ColIdx =
1237 Builder.CreateAdd(LHS: StartColI32, RHS: ConstantInt::get(Ty: Int32Ty, V: I));
1238 Value *ColI8 = Builder.CreateTrunc(V: ColIdx, DestTy: Int8Ty);
1239 Builder.CreateCall(Callee: ScalarFn, Args: {SigElementId, RowIndex, ColI8, Scalar});
1240 }
1241
1242 Orig->eraseFromParent();
1243 return true;
1244}
1245
1246// Scalarize a vector int_dx_load_input call into per-component scalar calls
1247// and reassemble the vector. The DXIL LoadInput op is per-component.
1248static Value *expandLoadInput(CallInst *Orig) {
1249 auto *VT = dyn_cast<FixedVectorType>(Val: Orig->getType());
1250 if (!VT)
1251 return nullptr; // already scalar, nothing to expand
1252
1253 IRBuilder<> Builder(Orig);
1254 Module *M = Orig->getModule();
1255 Type *Int8Ty = Builder.getInt8Ty();
1256 Type *Int32Ty = Builder.getInt32Ty();
1257 Type *ScalarTy = VT->getElementType();
1258 unsigned NumElems = VT->getNumElements();
1259
1260 Value *SigElementId = Orig->getArgOperand(i: 0);
1261 Value *RowIndex = Orig->getArgOperand(i: 1);
1262 Value *StartCol = Orig->getArgOperand(i: 2); // i8
1263 Value *GsVertexOrPrimIndex = Orig->getArgOperand(i: 3);
1264 Value *StartColI32 = Builder.CreateZExt(V: StartCol, DestTy: Int32Ty);
1265
1266 Function *ScalarFn = Intrinsic::getOrInsertDeclaration(
1267 M, id: Intrinsic::dx_load_input, OverloadTys: {ScalarTy});
1268
1269 Value *Vec = PoisonValue::get(T: VT);
1270 for (unsigned I = 0; I < NumElems; ++I) {
1271 Value *ColIdx =
1272 Builder.CreateAdd(LHS: StartColI32, RHS: ConstantInt::get(Ty: Int32Ty, V: I));
1273 Value *ColI8 = Builder.CreateTrunc(V: ColIdx, DestTy: Int8Ty);
1274 Value *Scalar = Builder.CreateCall(
1275 Callee: ScalarFn, Args: {SigElementId, RowIndex, ColI8, GsVertexOrPrimIndex});
1276 Vec =
1277 Builder.CreateInsertElement(Vec, NewElt: Scalar, Idx: ConstantInt::get(Ty: Int32Ty, V: I));
1278 }
1279
1280 return Vec;
1281}
1282
1283static bool expandIntrinsic(Function &F, CallInst *Orig) {
1284 Value *Result = nullptr;
1285 Intrinsic::ID IntrinsicId = F.getIntrinsicID();
1286 switch (IntrinsicId) {
1287 case Intrinsic::abs:
1288 Result = expandAbs(Orig);
1289 break;
1290 case Intrinsic::assume:
1291 Orig->eraseFromParent();
1292 return true;
1293 case Intrinsic::atan2:
1294 Result = expandAtan2Intrinsic(Orig);
1295 break;
1296 case Intrinsic::copysign:
1297 Result = expandCopySignIntrinsic(Orig);
1298 break;
1299 case Intrinsic::fshl:
1300 Result = expandFunnelShiftIntrinsic<true>(Orig);
1301 break;
1302 case Intrinsic::fshr:
1303 Result = expandFunnelShiftIntrinsic<false>(Orig);
1304 break;
1305 case Intrinsic::exp:
1306 Result = expandExpIntrinsic(Orig);
1307 break;
1308 case Intrinsic::is_fpclass:
1309 Result = expandIsFPClass(Orig);
1310 break;
1311 case Intrinsic::log:
1312 Result = expandLogIntrinsic(Orig);
1313 break;
1314 case Intrinsic::log10:
1315 Result = expandLog10Intrinsic(Orig);
1316 break;
1317 case Intrinsic::pow:
1318 case Intrinsic::powi:
1319 Result = expandPowIntrinsic(Orig, IntrinsicId);
1320 break;
1321 case Intrinsic::dx_all:
1322 case Intrinsic::dx_any:
1323 Result = expandAnyOrAllIntrinsic(Orig, IntrinsicId);
1324 break;
1325 case Intrinsic::dx_uclamp:
1326 case Intrinsic::dx_sclamp:
1327 case Intrinsic::dx_nclamp:
1328 Result = expandClampIntrinsic(Orig, ClampIntrinsic: IntrinsicId);
1329 break;
1330 case Intrinsic::dx_isinf:
1331 Result = expand16BitIsInf(Orig);
1332 break;
1333 case Intrinsic::dx_isnan:
1334 Result = expand16BitIsNaN(Orig);
1335 break;
1336 case Intrinsic::dx_fdot:
1337 Result = expandFloatDotIntrinsic(Orig);
1338 break;
1339 case Intrinsic::dx_sdot:
1340 case Intrinsic::dx_udot:
1341 Result = expandIntegerDotIntrinsic(Orig, DotIntrinsic: IntrinsicId);
1342 break;
1343 case Intrinsic::dx_sign:
1344 Result = expandSignIntrinsic(Orig);
1345 break;
1346 case Intrinsic::dx_load_input:
1347 Result = expandLoadInput(Orig);
1348 break;
1349 case Intrinsic::dx_store_output:
1350 if (expandStoreOutput(Orig))
1351 return true;
1352 break;
1353 case Intrinsic::dx_resource_load_rawbuffer:
1354 if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ true))
1355 return true;
1356 break;
1357 case Intrinsic::dx_resource_store_rawbuffer:
1358 if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ true))
1359 return true;
1360 break;
1361 case Intrinsic::dx_resource_load_typedbuffer:
1362 if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ false))
1363 return true;
1364 break;
1365 case Intrinsic::dx_resource_store_typedbuffer:
1366 if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ false))
1367 return true;
1368 break;
1369 case Intrinsic::usub_sat:
1370 Result = expandUsubSat(Orig);
1371 break;
1372 case Intrinsic::umul_with_overflow:
1373 case Intrinsic::smul_with_overflow:
1374 Result = expandMulWithOverflow(Orig, /*Signed=*/IntrinsicId ==
1375 Intrinsic::smul_with_overflow);
1376 break;
1377 case Intrinsic::vector_reduce_add:
1378 case Intrinsic::vector_reduce_fadd:
1379 Result = expandVecReduceAdd(Orig, IntrinsicId);
1380 break;
1381 case Intrinsic::matrix_multiply:
1382 Result = expandMatrixMultiply(Orig);
1383 break;
1384 case Intrinsic::matrix_transpose:
1385 Result = expandMatrixTranspose(Orig);
1386 break;
1387 }
1388 if (Result) {
1389 Orig->replaceAllUsesWith(V: Result);
1390 Orig->eraseFromParent();
1391 return true;
1392 }
1393 return false;
1394}
1395
1396static bool expansionIntrinsics(Module &M) {
1397 for (auto &F : make_early_inc_range(Range: M.functions())) {
1398 if (!isIntrinsicExpansion(F))
1399 continue;
1400 bool IntrinsicExpanded = false;
1401 for (User *U : make_early_inc_range(Range: F.users())) {
1402 auto *IntrinsicCall = dyn_cast<CallInst>(Val: U);
1403 if (!IntrinsicCall)
1404 continue;
1405 IntrinsicExpanded = expandIntrinsic(F, Orig: IntrinsicCall);
1406 }
1407 if (F.user_empty() && IntrinsicExpanded)
1408 F.eraseFromParent();
1409 }
1410 return true;
1411}
1412
1413PreservedAnalyses DXILIntrinsicExpansion::run(Module &M,
1414 ModuleAnalysisManager &) {
1415 if (expansionIntrinsics(M))
1416 return PreservedAnalyses::none();
1417 return PreservedAnalyses::all();
1418}
1419
1420bool DXILIntrinsicExpansionLegacy::runOnModule(Module &M) {
1421 return expansionIntrinsics(M);
1422}
1423
1424char DXILIntrinsicExpansionLegacy::ID = 0;
1425
1426INITIALIZE_PASS_BEGIN(DXILIntrinsicExpansionLegacy, DEBUG_TYPE,
1427 "DXIL Intrinsic Expansion", false, false)
1428INITIALIZE_PASS_END(DXILIntrinsicExpansionLegacy, DEBUG_TYPE,
1429 "DXIL Intrinsic Expansion", false, false)
1430
1431ModulePass *llvm::createDXILIntrinsicExpansionLegacyPass() {
1432 return new DXILIntrinsicExpansionLegacy();
1433}
1434