| 1 | //===- DXILIntrinsicExpansion.cpp - Prepare LLVM Module for DXIL encoding--===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file This file contains DXIL intrinsic expansions for those that don't have |
| 10 | // opcodes in DirectX Intermediate Language (DXIL). |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "DirectX.h" |
| 14 | #include "llvm/ADT/APInt.h" |
| 15 | #include "llvm/ADT/STLExtras.h" |
| 16 | #include "llvm/ADT/SmallVector.h" |
| 17 | #include "llvm/CodeGen/Passes.h" |
| 18 | #include "llvm/IR/Constants.h" |
| 19 | #include "llvm/IR/IRBuilder.h" |
| 20 | #include "llvm/IR/InstrTypes.h" |
| 21 | #include "llvm/IR/Instruction.h" |
| 22 | #include "llvm/IR/Instructions.h" |
| 23 | #include "llvm/IR/Intrinsics.h" |
| 24 | #include "llvm/IR/IntrinsicsDirectX.h" |
| 25 | #include "llvm/IR/MatrixBuilder.h" |
| 26 | #include "llvm/IR/Module.h" |
| 27 | #include "llvm/IR/PassManager.h" |
| 28 | #include "llvm/IR/Type.h" |
| 29 | #include "llvm/Pass.h" |
| 30 | #include "llvm/Support/Casting.h" |
| 31 | #include "llvm/Support/ErrorHandling.h" |
| 32 | #include "llvm/Support/MathExtras.h" |
| 33 | |
| 34 | #define DEBUG_TYPE "dxil-intrinsic-expansion" |
| 35 | |
| 36 | using namespace llvm; |
| 37 | |
| 38 | class DXILIntrinsicExpansionLegacy : public ModulePass { |
| 39 | |
| 40 | public: |
| 41 | bool runOnModule(Module &M) override; |
| 42 | DXILIntrinsicExpansionLegacy() : ModulePass(ID) {} |
| 43 | |
| 44 | static char ID; // Pass identification. |
| 45 | }; |
| 46 | |
| 47 | static bool resourceAccessNeeds64BitExpansion(Module *M, Type *OverloadTy, |
| 48 | bool IsRaw) { |
| 49 | if (IsRaw && M->getTargetTriple().getDXILVersion() > VersionTuple(1, 2)) |
| 50 | return false; |
| 51 | |
| 52 | Type *ScalarTy = OverloadTy->getScalarType(); |
| 53 | return ScalarTy->isDoubleTy() || ScalarTy->isIntegerTy(BitWidth: 64); |
| 54 | } |
| 55 | |
| 56 | static Value *expand16BitIsInf(CallInst *Orig) { |
| 57 | Module *M = Orig->getModule(); |
| 58 | if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9)) |
| 59 | return nullptr; |
| 60 | |
| 61 | Value *Val = Orig->getOperand(i_nocapture: 0); |
| 62 | Type *ValTy = Val->getType(); |
| 63 | if (!ValTy->getScalarType()->isHalfTy()) |
| 64 | return nullptr; |
| 65 | |
| 66 | IRBuilder<> Builder(Orig); |
| 67 | Type *IType = Type::getInt16Ty(C&: M->getContext()); |
| 68 | Constant *PosInf = |
| 69 | ValTy->isVectorTy() |
| 70 | ? ConstantVector::getSplat( |
| 71 | EC: ElementCount::getFixed( |
| 72 | MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()), |
| 73 | Elt: ConstantInt::get(Ty: IType, V: 0x7c00)) |
| 74 | : ConstantInt::get(Ty: IType, V: 0x7c00); |
| 75 | |
| 76 | Constant *NegInf = |
| 77 | ValTy->isVectorTy() |
| 78 | ? ConstantVector::getSplat( |
| 79 | EC: ElementCount::getFixed( |
| 80 | MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()), |
| 81 | Elt: ConstantInt::get(Ty: IType, V: 0xfc00)) |
| 82 | : ConstantInt::get(Ty: IType, V: 0xfc00); |
| 83 | |
| 84 | Value *IVal = Builder.CreateBitCast(V: Val, DestTy: PosInf->getType()); |
| 85 | Value *B1 = Builder.CreateICmpEQ(LHS: IVal, RHS: PosInf); |
| 86 | Value *B2 = Builder.CreateICmpEQ(LHS: IVal, RHS: NegInf); |
| 87 | Value *B3 = Builder.CreateOr(LHS: B1, RHS: B2); |
| 88 | return B3; |
| 89 | } |
| 90 | |
| 91 | static Value *expand16BitIsNaN(CallInst *Orig) { |
| 92 | Module *M = Orig->getModule(); |
| 93 | if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9)) |
| 94 | return nullptr; |
| 95 | |
| 96 | Value *Val = Orig->getOperand(i_nocapture: 0); |
| 97 | Type *ValTy = Val->getType(); |
| 98 | if (!ValTy->getScalarType()->isHalfTy()) |
| 99 | return nullptr; |
| 100 | |
| 101 | IRBuilder<> Builder(Orig); |
| 102 | Type *IType = Type::getInt16Ty(C&: M->getContext()); |
| 103 | |
| 104 | Constant *ExpBitMask = |
| 105 | ValTy->isVectorTy() |
| 106 | ? ConstantVector::getSplat( |
| 107 | EC: ElementCount::getFixed( |
| 108 | MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()), |
| 109 | Elt: ConstantInt::get(Ty: IType, V: 0x7c00)) |
| 110 | : ConstantInt::get(Ty: IType, V: 0x7c00); |
| 111 | Constant *SigBitMask = |
| 112 | ValTy->isVectorTy() |
| 113 | ? ConstantVector::getSplat( |
| 114 | EC: ElementCount::getFixed( |
| 115 | MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()), |
| 116 | Elt: ConstantInt::get(Ty: IType, V: 0x3ff)) |
| 117 | : ConstantInt::get(Ty: IType, V: 0x3ff); |
| 118 | |
| 119 | Constant *Zero = |
| 120 | ValTy->isVectorTy() |
| 121 | ? ConstantVector::getSplat( |
| 122 | EC: ElementCount::getFixed( |
| 123 | MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()), |
| 124 | Elt: ConstantInt::get(Ty: IType, V: 0)) |
| 125 | : ConstantInt::get(Ty: IType, V: 0); |
| 126 | |
| 127 | Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType()); |
| 128 | Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask); |
| 129 | Value *B1 = Builder.CreateICmpEQ(LHS: Exp, RHS: ExpBitMask); |
| 130 | |
| 131 | Value *Sig = Builder.CreateAnd(LHS: IVal, RHS: SigBitMask); |
| 132 | Value *B2 = Builder.CreateICmpNE(LHS: Sig, RHS: Zero); |
| 133 | Value *B3 = Builder.CreateAnd(LHS: B1, RHS: B2); |
| 134 | return B3; |
| 135 | } |
| 136 | |
| 137 | static Value *expand16BitIsFinite(CallInst *Orig) { |
| 138 | Module *M = Orig->getModule(); |
| 139 | if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9)) |
| 140 | return nullptr; |
| 141 | |
| 142 | Value *Val = Orig->getOperand(i_nocapture: 0); |
| 143 | Type *ValTy = Val->getType(); |
| 144 | if (!ValTy->getScalarType()->isHalfTy()) |
| 145 | return nullptr; |
| 146 | |
| 147 | IRBuilder<> Builder(Orig); |
| 148 | Type *IType = Type::getInt16Ty(C&: M->getContext()); |
| 149 | |
| 150 | Constant *ExpBitMask = |
| 151 | ValTy->isVectorTy() |
| 152 | ? ConstantVector::getSplat( |
| 153 | EC: ElementCount::getFixed( |
| 154 | MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()), |
| 155 | Elt: ConstantInt::get(Ty: IType, V: 0x7c00)) |
| 156 | : ConstantInt::get(Ty: IType, V: 0x7c00); |
| 157 | |
| 158 | Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType()); |
| 159 | Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask); |
| 160 | Value *B1 = Builder.CreateICmpNE(LHS: Exp, RHS: ExpBitMask); |
| 161 | return B1; |
| 162 | } |
| 163 | |
| 164 | static Value *expand16BitIsNormal(CallInst *Orig) { |
| 165 | Module *M = Orig->getModule(); |
| 166 | if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9)) |
| 167 | return nullptr; |
| 168 | |
| 169 | Value *Val = Orig->getOperand(i_nocapture: 0); |
| 170 | Type *ValTy = Val->getType(); |
| 171 | if (!ValTy->getScalarType()->isHalfTy()) |
| 172 | return nullptr; |
| 173 | |
| 174 | IRBuilder<> Builder(Orig); |
| 175 | Type *IType = Type::getInt16Ty(C&: M->getContext()); |
| 176 | |
| 177 | Constant *ExpBitMask = |
| 178 | ValTy->isVectorTy() |
| 179 | ? ConstantVector::getSplat( |
| 180 | EC: ElementCount::getFixed( |
| 181 | MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()), |
| 182 | Elt: ConstantInt::get(Ty: IType, V: 0x7c00)) |
| 183 | : ConstantInt::get(Ty: IType, V: 0x7c00); |
| 184 | Constant *Zero = |
| 185 | ValTy->isVectorTy() |
| 186 | ? ConstantVector::getSplat( |
| 187 | EC: ElementCount::getFixed( |
| 188 | MinVal: cast<FixedVectorType>(Val: ValTy)->getNumElements()), |
| 189 | Elt: ConstantInt::get(Ty: IType, V: 0)) |
| 190 | : ConstantInt::get(Ty: IType, V: 0); |
| 191 | |
| 192 | Value *IVal = Builder.CreateBitCast(V: Val, DestTy: ExpBitMask->getType()); |
| 193 | Value *Exp = Builder.CreateAnd(LHS: IVal, RHS: ExpBitMask); |
| 194 | Value *NotAllZeroes = Builder.CreateICmpNE(LHS: Exp, RHS: Zero); |
| 195 | Value *NotAllOnes = Builder.CreateICmpNE(LHS: Exp, RHS: ExpBitMask); |
| 196 | Value *B1 = Builder.CreateAnd(LHS: NotAllZeroes, RHS: NotAllOnes); |
| 197 | return B1; |
| 198 | } |
| 199 | |
| 200 | static bool shouldExpandFloatDotIntrinsic(Function &F) { |
| 201 | assert(F.getIntrinsicID() == Intrinsic::dx_fdot && |
| 202 | "Function is not a dx.fdot intrinsic" ); |
| 203 | auto *ParamTy = cast<FixedVectorType>(Val: F.getFunctionType()->getParamType(i: 0)); |
| 204 | return ParamTy->getNumElements() <= 4 || |
| 205 | F.getParent()->getTargetTriple().getOSVersion() < VersionTuple(6, 9); |
| 206 | } |
| 207 | |
| 208 | static bool isIntrinsicExpansion(Function &F) { |
| 209 | switch (F.getIntrinsicID()) { |
| 210 | case Intrinsic::assume: |
| 211 | case Intrinsic::abs: |
| 212 | case Intrinsic::atan2: |
| 213 | case Intrinsic::copysign: |
| 214 | case Intrinsic::fshl: |
| 215 | case Intrinsic::fshr: |
| 216 | case Intrinsic::exp: |
| 217 | case Intrinsic::is_fpclass: |
| 218 | case Intrinsic::log: |
| 219 | case Intrinsic::log10: |
| 220 | case Intrinsic::pow: |
| 221 | case Intrinsic::powi: |
| 222 | case Intrinsic::dx_uclamp: |
| 223 | case Intrinsic::dx_sclamp: |
| 224 | case Intrinsic::dx_nclamp: |
| 225 | case Intrinsic::dx_isfinite: |
| 226 | case Intrinsic::dx_isinf: |
| 227 | case Intrinsic::dx_isnan: |
| 228 | case Intrinsic::dx_sdot: |
| 229 | case Intrinsic::dx_udot: |
| 230 | case Intrinsic::dx_sign: |
| 231 | case Intrinsic::usub_sat: |
| 232 | case Intrinsic::vector_reduce_add: |
| 233 | case Intrinsic::vector_reduce_fadd: |
| 234 | case Intrinsic::matrix_multiply: |
| 235 | case Intrinsic::matrix_transpose: |
| 236 | case Intrinsic::umul_with_overflow: |
| 237 | case Intrinsic::smul_with_overflow: |
| 238 | case Intrinsic::dx_load_input: |
| 239 | case Intrinsic::dx_store_output: |
| 240 | return true; |
| 241 | case Intrinsic::vector_reduce_and: |
| 242 | case Intrinsic::vector_reduce_or: |
| 243 | return F.getParent()->getTargetTriple().getDXILVersion() < |
| 244 | VersionTuple(1, 9); |
| 245 | case Intrinsic::dx_quad_all: |
| 246 | return F.getParent()->getTargetTriple().getDXILVersion() < |
| 247 | VersionTuple(1, 7); |
| 248 | case Intrinsic::dx_fdot: |
| 249 | return shouldExpandFloatDotIntrinsic(F); |
| 250 | case Intrinsic::dx_resource_load_rawbuffer: |
| 251 | return resourceAccessNeeds64BitExpansion( |
| 252 | M: F.getParent(), OverloadTy: F.getReturnType()->getStructElementType(N: 0), |
| 253 | /*IsRaw*/ true); |
| 254 | case Intrinsic::dx_resource_load_typedbuffer: |
| 255 | return resourceAccessNeeds64BitExpansion( |
| 256 | M: F.getParent(), OverloadTy: F.getReturnType()->getStructElementType(N: 0), |
| 257 | /*IsRaw*/ false); |
| 258 | case Intrinsic::dx_resource_store_rawbuffer: |
| 259 | return resourceAccessNeeds64BitExpansion( |
| 260 | M: F.getParent(), OverloadTy: F.getFunctionType()->getParamType(i: 3), /*IsRaw*/ true); |
| 261 | case Intrinsic::dx_resource_store_typedbuffer: |
| 262 | return resourceAccessNeeds64BitExpansion( |
| 263 | M: F.getParent(), OverloadTy: F.getFunctionType()->getParamType(i: 2), /*IsRaw*/ false); |
| 264 | } |
| 265 | return false; |
| 266 | } |
| 267 | |
| 268 | static Value *expandUsubSat(CallInst *Orig) { |
| 269 | Value *A = Orig->getArgOperand(i: 0); |
| 270 | Value *B = Orig->getArgOperand(i: 1); |
| 271 | Type *Ty = A->getType(); |
| 272 | |
| 273 | IRBuilder<> Builder(Orig); |
| 274 | |
| 275 | Value *Cmp = Builder.CreateICmpULT(LHS: A, RHS: B, Name: "usub.cmp" ); |
| 276 | Value *Sub = Builder.CreateSub(LHS: A, RHS: B, Name: "usub.sub" ); |
| 277 | Value *Zero = ConstantInt::get(Ty, V: 0); |
| 278 | return Builder.CreateSelect(C: Cmp, True: Zero, False: Sub, Name: "usub.sat" ); |
| 279 | } |
| 280 | |
| 281 | // Compute the high N bits of the 2N-bit unsigned product of two N-bit values |
| 282 | // using only N-bit arithmetic, so we don't introduce a wider integer type that |
| 283 | // may be unsupported in DXIL. |
| 284 | static Value *createMulHighUnsigned(IRBuilder<> &Builder, Value *A, Value *B, |
| 285 | Type *Ty, unsigned BW) { |
| 286 | assert(BW % 2 == 0 && "high-half split needs symmetric halves" ); |
| 287 | unsigned Half = BW / 2; |
| 288 | Value *HalfShift = ConstantInt::get(Ty, V: Half); |
| 289 | Value *LoMask = ConstantInt::get(Ty, V: APInt::getLowBitsSet(numBits: BW, loBitsSet: Half)); |
| 290 | |
| 291 | Value *U0 = Builder.CreateAnd(LHS: A, RHS: LoMask); |
| 292 | Value *U1 = Builder.CreateLShr(LHS: A, RHS: HalfShift); |
| 293 | Value *V0 = Builder.CreateAnd(LHS: B, RHS: LoMask); |
| 294 | Value *V1 = Builder.CreateLShr(LHS: B, RHS: HalfShift); |
| 295 | |
| 296 | Value *W0 = Builder.CreateMul(LHS: U0, RHS: V0); |
| 297 | Value *T = Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U1, RHS: V0), |
| 298 | RHS: Builder.CreateLShr(LHS: W0, RHS: HalfShift)); |
| 299 | Value *W1 = Builder.CreateAnd(LHS: T, RHS: LoMask); |
| 300 | Value *W2 = Builder.CreateLShr(LHS: T, RHS: HalfShift); |
| 301 | W1 = Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U0, RHS: V1), RHS: W1); |
| 302 | return Builder.CreateAdd(LHS: Builder.CreateAdd(LHS: Builder.CreateMul(LHS: U1, RHS: V1), RHS: W2), |
| 303 | RHS: Builder.CreateLShr(LHS: W1, RHS: HalfShift)); |
| 304 | } |
| 305 | |
| 306 | // Expand a {u,s}mul.with.overflow intrinsic. The low half of the result is a |
| 307 | // plain multiply; overflow is derived from the high half of the double-width |
| 308 | // product. |
| 309 | static Value *expandMulWithOverflow(CallInst *Orig, bool Signed) { |
| 310 | IRBuilder<> Builder(Orig); |
| 311 | Value *A = Orig->getArgOperand(i: 0); |
| 312 | Value *B = Orig->getArgOperand(i: 1); |
| 313 | Type *Ty = A->getType(); |
| 314 | unsigned BW = Ty->getScalarSizeInBits(); |
| 315 | |
| 316 | Value *Lo; |
| 317 | Value *Ov; |
| 318 | |
| 319 | // A plain double-width multiply is simplest, but we avoid it once it would |
| 320 | // introduce a 64-bit (or wider) integer, which DXIL does not always support. |
| 321 | // For i32 we use the native DXIL IMul/UMul ops, which return the full product |
| 322 | // as two i32s; wider types fall back to a same-width high-half computation. |
| 323 | if (2 * BW <= 32) { |
| 324 | Lo = Builder.CreateMul(LHS: A, RHS: B); |
| 325 | Type *WideTy = Ty->getWithNewBitWidth(NewBitWidth: 2 * BW); |
| 326 | Value *WideA = |
| 327 | Signed ? Builder.CreateSExt(V: A, DestTy: WideTy) : Builder.CreateZExt(V: A, DestTy: WideTy); |
| 328 | Value *WideB = |
| 329 | Signed ? Builder.CreateSExt(V: B, DestTy: WideTy) : Builder.CreateZExt(V: B, DestTy: WideTy); |
| 330 | Value *Wide = Builder.CreateMul(LHS: WideA, RHS: WideB); |
| 331 | if (Signed) { |
| 332 | // Overflow when the full product doesn't fit back into BW signed bits. |
| 333 | Ov = Builder.CreateICmpNE(LHS: Wide, RHS: Builder.CreateSExt(V: Lo, DestTy: WideTy)); |
| 334 | } else { |
| 335 | Value *Hi = Builder.CreateLShr(LHS: Wide, RHS: ConstantInt::get(Ty: WideTy, V: BW)); |
| 336 | Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty: WideTy, V: 0)); |
| 337 | } |
| 338 | } else if (BW == 32) { |
| 339 | // IMul/UMul return {high, low}; index 0 is the high 32 bits. |
| 340 | Type *ResTy = StructType::get(elt1: Ty, elts: Ty); |
| 341 | Intrinsic::ID IntrinsicID = |
| 342 | Signed ? Intrinsic::dx_imul : Intrinsic::dx_umul; |
| 343 | Value *Mul = Builder.CreateIntrinsic(RetTy: ResTy, ID: IntrinsicID, Args: {A, B}); |
| 344 | Value *Hi = Builder.CreateExtractValue(Agg: Mul, Idxs: 0); |
| 345 | Lo = Builder.CreateExtractValue(Agg: Mul, Idxs: 1); |
| 346 | if (Signed) |
| 347 | Ov = Builder.CreateICmpNE( |
| 348 | LHS: Hi, RHS: Builder.CreateAShr(LHS: Lo, RHS: ConstantInt::get(Ty, V: BW - 1))); |
| 349 | else |
| 350 | Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty, V: 0)); |
| 351 | } else { |
| 352 | Lo = Builder.CreateMul(LHS: A, RHS: B); |
| 353 | Value *Hi = createMulHighUnsigned(Builder, A, B, Ty, BW); |
| 354 | if (Signed) { |
| 355 | // Turn the unsigned high half into the signed one, then overflow means it |
| 356 | // isn't the sign extension of the low half. |
| 357 | Value *SignShift = ConstantInt::get(Ty, V: BW - 1); |
| 358 | Value *ASign = Builder.CreateAShr(LHS: A, RHS: SignShift); |
| 359 | Value *BSign = Builder.CreateAShr(LHS: B, RHS: SignShift); |
| 360 | Hi = Builder.CreateSub(LHS: Hi, RHS: Builder.CreateAnd(LHS: ASign, RHS: B)); |
| 361 | Hi = Builder.CreateSub(LHS: Hi, RHS: Builder.CreateAnd(LHS: BSign, RHS: A)); |
| 362 | Ov = Builder.CreateICmpNE(LHS: Hi, RHS: Builder.CreateAShr(LHS: Lo, RHS: SignShift)); |
| 363 | } else { |
| 364 | Ov = Builder.CreateICmpNE(LHS: Hi, RHS: ConstantInt::get(Ty, V: 0)); |
| 365 | } |
| 366 | } |
| 367 | |
| 368 | Value *Agg = PoisonValue::get(T: Orig->getType()); |
| 369 | Agg = Builder.CreateInsertValue(Agg, Val: Lo, Idxs: 0); |
| 370 | return Builder.CreateInsertValue(Agg, Val: Ov, Idxs: 1); |
| 371 | } |
| 372 | |
| 373 | static Value *expandVecReduceAdd(CallInst *Orig, Intrinsic::ID IntrinsicId) { |
| 374 | assert(IntrinsicId == Intrinsic::vector_reduce_add || |
| 375 | IntrinsicId == Intrinsic::vector_reduce_fadd); |
| 376 | |
| 377 | IRBuilder<> Builder(Orig); |
| 378 | bool IsFAdd = (IntrinsicId == Intrinsic::vector_reduce_fadd); |
| 379 | |
| 380 | Value *X = Orig->getOperand(i_nocapture: IsFAdd ? 1 : 0); |
| 381 | Type *Ty = X->getType(); |
| 382 | auto *XVec = dyn_cast<FixedVectorType>(Val: Ty); |
| 383 | unsigned XVecSize = XVec->getNumElements(); |
| 384 | Value *Sum = Builder.CreateExtractElement(Vec: X, Idx: static_cast<uint64_t>(0)); |
| 385 | |
| 386 | // Handle the initial start value for floating-point addition. |
| 387 | if (IsFAdd) { |
| 388 | Constant *StartValue = dyn_cast<Constant>(Val: Orig->getOperand(i_nocapture: 0)); |
| 389 | if (StartValue && !StartValue->isNullValue()) |
| 390 | Sum = Builder.CreateFAdd(L: Sum, R: StartValue); |
| 391 | } |
| 392 | |
| 393 | // Accumulate the remaining vector elements. |
| 394 | for (unsigned I = 1; I < XVecSize; I++) { |
| 395 | Value *Elt = Builder.CreateExtractElement(Vec: X, Idx: I); |
| 396 | if (IsFAdd) |
| 397 | Sum = Builder.CreateFAdd(L: Sum, R: Elt); |
| 398 | else |
| 399 | Sum = Builder.CreateAdd(LHS: Sum, RHS: Elt); |
| 400 | } |
| 401 | |
| 402 | return Sum; |
| 403 | } |
| 404 | |
| 405 | static Value *expandVecReduceAndOr(CallInst *Orig, Intrinsic::ID IntrinsicId) { |
| 406 | assert(IntrinsicId == Intrinsic::vector_reduce_and || |
| 407 | IntrinsicId == Intrinsic::vector_reduce_or); |
| 408 | |
| 409 | IRBuilder<> Builder(Orig); |
| 410 | Value *X = Orig->getArgOperand(i: 0); |
| 411 | auto *XVec = cast<FixedVectorType>(Val: X->getType()); |
| 412 | Value *Result = Builder.CreateExtractElement(Vec: X, Idx: static_cast<uint64_t>(0)); |
| 413 | for (unsigned I = 1; I < XVec->getNumElements(); ++I) { |
| 414 | Value *Elt = Builder.CreateExtractElement(Vec: X, Idx: I); |
| 415 | Result = IntrinsicId == Intrinsic::vector_reduce_or |
| 416 | ? Builder.CreateOr(LHS: Result, RHS: Elt) |
| 417 | : Builder.CreateAnd(LHS: Result, RHS: Elt); |
| 418 | } |
| 419 | return Result; |
| 420 | } |
| 421 | |
| 422 | static Value *expandAbs(CallInst *Orig) { |
| 423 | Value *X = Orig->getOperand(i_nocapture: 0); |
| 424 | IRBuilder<> Builder(Orig); |
| 425 | Type *Ty = X->getType(); |
| 426 | Type *EltTy = Ty->getScalarType(); |
| 427 | Constant *Zero = Ty->isVectorTy() |
| 428 | ? ConstantVector::getSplat( |
| 429 | EC: ElementCount::getFixed( |
| 430 | MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()), |
| 431 | Elt: ConstantInt::get(Ty: EltTy, V: 0)) |
| 432 | : ConstantInt::get(Ty: EltTy, V: 0); |
| 433 | auto *V = Builder.CreateSub(LHS: Zero, RHS: X); |
| 434 | return Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::smax, Args: {X, V}, FMFSource: nullptr, |
| 435 | Name: "dx.max" ); |
| 436 | } |
| 437 | |
| 438 | // Create a DXIL dot2, dot3, or dot4 for the given operands. |
| 439 | static Value *expandFloatDotChunk(CallInst *Orig, Value *A, Value *B) { |
| 440 | Type *ATy = A->getType(); |
| 441 | [[maybe_unused]] Type *BTy = B->getType(); |
| 442 | assert(ATy->isVectorTy() && BTy->isVectorTy()); |
| 443 | |
| 444 | IRBuilder<> Builder(Orig); |
| 445 | |
| 446 | auto *AVec = dyn_cast<FixedVectorType>(Val: ATy); |
| 447 | |
| 448 | assert(ATy->getScalarType()->isFloatingPointTy()); |
| 449 | |
| 450 | unsigned NumElts = AVec->getNumElements(); |
| 451 | Intrinsic::ID DotIntrinsic; |
| 452 | switch (NumElts) { |
| 453 | case 2: |
| 454 | DotIntrinsic = Intrinsic::dx_dot2; |
| 455 | break; |
| 456 | case 3: |
| 457 | DotIntrinsic = Intrinsic::dx_dot3; |
| 458 | break; |
| 459 | case 4: |
| 460 | DotIntrinsic = Intrinsic::dx_dot4; |
| 461 | break; |
| 462 | default: |
| 463 | reportFatalUsageError( |
| 464 | reason: "Invalid dot product input vector: length is outside 2-4" ); |
| 465 | } |
| 466 | |
| 467 | SmallVector<Value *> Args; |
| 468 | for (unsigned I = 0; I < NumElts; ++I) |
| 469 | Args.push_back(Elt: Builder.CreateExtractElement(Vec: A, Idx: Builder.getInt32(C: I))); |
| 470 | for (unsigned I = 0; I < NumElts; ++I) |
| 471 | Args.push_back(Elt: Builder.CreateExtractElement(Vec: B, Idx: Builder.getInt32(C: I))); |
| 472 | return Builder.CreateIntrinsic(RetTy: ATy->getScalarType(), ID: DotIntrinsic, Args, |
| 473 | FMFSource: nullptr, Name: "dot" ); |
| 474 | } |
| 475 | |
| 476 | // Expand an arbitrary-width float dot into the minimum number of legal DXIL |
| 477 | // dot2, dot3, and dot4 operations. |
| 478 | static Value *expandFloatDotIntrinsic(CallInst *Orig) { |
| 479 | Value *A = Orig->getOperand(i_nocapture: 0); |
| 480 | Value *B = Orig->getOperand(i_nocapture: 1); |
| 481 | unsigned NumElts = cast<FixedVectorType>(Val: A->getType())->getNumElements(); |
| 482 | |
| 483 | // We return early here to avoid constructing unnecessary identity shuffles. |
| 484 | if (NumElts <= 4) |
| 485 | return expandFloatDotChunk(Orig, A, B); |
| 486 | |
| 487 | assert(Orig->getModule()->getTargetTriple().getOSVersion() < |
| 488 | VersionTuple(6, 9) && |
| 489 | "long fdot must not be expanded for shader model 6.9 or later" ); |
| 490 | |
| 491 | IRBuilder<> Builder(Orig); |
| 492 | Value *Result = nullptr; |
| 493 | for (unsigned Offset = 0; Offset < NumElts;) { |
| 494 | unsigned Remaining = NumElts - Offset; |
| 495 | // Taking four is optimal unless it would leave an illegal one-element |
| 496 | // tail. In that case, take three and finish with dot2. |
| 497 | unsigned ChunkSize = Remaining == 5 ? 3 : std::min(a: Remaining, b: 4u); |
| 498 | SmallVector<int, 4> Mask; |
| 499 | for (unsigned I = 0; I < ChunkSize; ++I) |
| 500 | Mask.push_back(Elt: Offset + I); |
| 501 | Value *AChunk = Builder.CreateShuffleVector(V: A, Mask); |
| 502 | Value *BChunk = Builder.CreateShuffleVector(V: B, Mask); |
| 503 | Value *Chunk = expandFloatDotChunk(Orig, A: AChunk, B: BChunk); |
| 504 | Result = Result ? Builder.CreateFAdd(L: Result, R: Chunk, Name: "dot.add" ) : Chunk; |
| 505 | Offset += ChunkSize; |
| 506 | } |
| 507 | return Result; |
| 508 | } |
| 509 | |
| 510 | // Expand integer dot product to multiply and add ops |
| 511 | static Value *expandIntegerDotIntrinsic(CallInst *Orig, |
| 512 | Intrinsic::ID DotIntrinsic) { |
| 513 | assert(DotIntrinsic == Intrinsic::dx_sdot || |
| 514 | DotIntrinsic == Intrinsic::dx_udot); |
| 515 | Value *A = Orig->getOperand(i_nocapture: 0); |
| 516 | Value *B = Orig->getOperand(i_nocapture: 1); |
| 517 | Type *ATy = A->getType(); |
| 518 | [[maybe_unused]] Type *BTy = B->getType(); |
| 519 | assert(ATy->isVectorTy() && BTy->isVectorTy()); |
| 520 | |
| 521 | IRBuilder<> Builder(Orig); |
| 522 | |
| 523 | auto *AVec = dyn_cast<FixedVectorType>(Val: ATy); |
| 524 | |
| 525 | assert(ATy->getScalarType()->isIntegerTy()); |
| 526 | |
| 527 | Value *Result; |
| 528 | Intrinsic::ID MadIntrinsic = DotIntrinsic == Intrinsic::dx_sdot |
| 529 | ? Intrinsic::dx_imad |
| 530 | : Intrinsic::dx_umad; |
| 531 | Value *Elt0 = Builder.CreateExtractElement(Vec: A, Idx: (uint64_t)0); |
| 532 | Value *Elt1 = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0); |
| 533 | Result = Builder.CreateMul(LHS: Elt0, RHS: Elt1); |
| 534 | for (unsigned I = 1; I < AVec->getNumElements(); I++) { |
| 535 | Elt0 = Builder.CreateExtractElement(Vec: A, Idx: I); |
| 536 | Elt1 = Builder.CreateExtractElement(Vec: B, Idx: I); |
| 537 | Result = Builder.CreateIntrinsic(RetTy: Result->getType(), ID: MadIntrinsic, |
| 538 | Args: ArrayRef<Value *>{Elt0, Elt1, Result}, |
| 539 | FMFSource: nullptr, Name: "dx.mad" ); |
| 540 | } |
| 541 | return Result; |
| 542 | } |
| 543 | |
| 544 | static Value *expandExpIntrinsic(CallInst *Orig) { |
| 545 | Value *X = Orig->getOperand(i_nocapture: 0); |
| 546 | IRBuilder<> Builder(Orig); |
| 547 | Type *Ty = X->getType(); |
| 548 | Type *EltTy = Ty->getScalarType(); |
| 549 | Constant *Log2eConst = |
| 550 | Ty->isVectorTy() ? ConstantVector::getSplat( |
| 551 | EC: ElementCount::getFixed( |
| 552 | MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()), |
| 553 | Elt: ConstantFP::get(Ty: EltTy, V: numbers::log2ef)) |
| 554 | : ConstantFP::get(Ty: EltTy, V: numbers::log2ef); |
| 555 | Value *NewX = Builder.CreateFMul(L: Log2eConst, R: X); |
| 556 | CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding( |
| 557 | RetTy: Ty, ID: Intrinsic::exp2, Args: {NewX}, FMFSource: nullptr, Name: "dx.exp2" ); |
| 558 | Exp2Call->setTailCall(Orig->isTailCall()); |
| 559 | Exp2Call->setAttributes(Orig->getAttributes()); |
| 560 | return Exp2Call; |
| 561 | } |
| 562 | |
| 563 | static Value *expandIsFPClass(CallInst *Orig) { |
| 564 | Value *T = Orig->getArgOperand(i: 1); |
| 565 | auto *TCI = dyn_cast<ConstantInt>(Val: T); |
| 566 | |
| 567 | // These FPClassTest cases have DXIL opcodes, so they will be handled in |
| 568 | // DXIL Op Lowering instead for all non f16 cases. |
| 569 | switch (TCI->getZExtValue()) { |
| 570 | case FPClassTest::fcInf: |
| 571 | return expand16BitIsInf(Orig); |
| 572 | case FPClassTest::fcNan: |
| 573 | return expand16BitIsNaN(Orig); |
| 574 | case FPClassTest::fcNormal: |
| 575 | return expand16BitIsNormal(Orig); |
| 576 | case FPClassTest::fcFinite: |
| 577 | return expand16BitIsFinite(Orig); |
| 578 | } |
| 579 | |
| 580 | IRBuilder<> Builder(Orig); |
| 581 | |
| 582 | Value *F = Orig->getArgOperand(i: 0); |
| 583 | Type *FTy = F->getType(); |
| 584 | unsigned FNumElem = 0; // 0 => F is not a vector |
| 585 | |
| 586 | unsigned BitWidth; // Bit width of F or the ElemTy of F |
| 587 | Type *BitCastTy; // An IntNTy of the same bitwidth as F or ElemTy of F |
| 588 | |
| 589 | if (auto *FVecTy = dyn_cast<FixedVectorType>(Val: FTy)) { |
| 590 | Type *ElemTy = FVecTy->getElementType(); |
| 591 | FNumElem = FVecTy->getNumElements(); |
| 592 | BitWidth = ElemTy->getPrimitiveSizeInBits(); |
| 593 | BitCastTy = FixedVectorType::get(ElementType: Builder.getIntNTy(N: BitWidth), NumElts: FNumElem); |
| 594 | } else { |
| 595 | BitWidth = FTy->getPrimitiveSizeInBits(); |
| 596 | BitCastTy = Builder.getIntNTy(N: BitWidth); |
| 597 | } |
| 598 | |
| 599 | Value *FBitCast = Builder.CreateBitCast(V: F, DestTy: BitCastTy); |
| 600 | switch (TCI->getZExtValue()) { |
| 601 | case FPClassTest::fcNegZero: { |
| 602 | Value *NegZero = ConstantInt::get(Ty: Builder.getIntNTy(N: BitWidth), |
| 603 | V: APInt::getSignMask(BitWidth)); |
| 604 | Value *RetVal; |
| 605 | if (FNumElem) { |
| 606 | Value *NegZeroSplat = Builder.CreateVectorSplat(NumElts: FNumElem, V: NegZero); |
| 607 | RetVal = |
| 608 | Builder.CreateICmpEQ(LHS: FBitCast, RHS: NegZeroSplat, Name: "is.fpclass.negzero" ); |
| 609 | } else |
| 610 | RetVal = Builder.CreateICmpEQ(LHS: FBitCast, RHS: NegZero, Name: "is.fpclass.negzero" ); |
| 611 | return RetVal; |
| 612 | } |
| 613 | default: |
| 614 | reportFatalUsageError(reason: "Unsupported FPClassTest" ); |
| 615 | } |
| 616 | } |
| 617 | |
| 618 | static Value *expandLogIntrinsic(CallInst *Orig, |
| 619 | float LogConstVal = numbers::ln2f) { |
| 620 | Value *X = Orig->getOperand(i_nocapture: 0); |
| 621 | IRBuilder<> Builder(Orig); |
| 622 | Type *Ty = X->getType(); |
| 623 | Type *EltTy = Ty->getScalarType(); |
| 624 | Constant *Ln2Const = |
| 625 | Ty->isVectorTy() ? ConstantVector::getSplat( |
| 626 | EC: ElementCount::getFixed( |
| 627 | MinVal: cast<FixedVectorType>(Val: Ty)->getNumElements()), |
| 628 | Elt: ConstantFP::get(Ty: EltTy, V: LogConstVal)) |
| 629 | : ConstantFP::get(Ty: EltTy, V: LogConstVal); |
| 630 | CallInst *Log2Call = Builder.CreateIntrinsicWithoutFolding( |
| 631 | RetTy: Ty, ID: Intrinsic::log2, Args: {X}, FMFSource: nullptr, Name: "elt.log2" ); |
| 632 | Log2Call->setTailCall(Orig->isTailCall()); |
| 633 | Log2Call->setAttributes(Orig->getAttributes()); |
| 634 | return Builder.CreateFMul(L: Ln2Const, R: Log2Call); |
| 635 | } |
| 636 | static Value *expandLog10Intrinsic(CallInst *Orig) { |
| 637 | return expandLogIntrinsic(Orig, LogConstVal: numbers::ln2f / numbers::ln10f); |
| 638 | } |
| 639 | |
| 640 | static Value *expandAtan2Intrinsic(CallInst *Orig) { |
| 641 | Value *Y = Orig->getOperand(i_nocapture: 0); |
| 642 | Value *X = Orig->getOperand(i_nocapture: 1); |
| 643 | Type *Ty = X->getType(); |
| 644 | IRBuilder<> Builder(Orig); |
| 645 | Builder.setFastMathFlags(Orig->getFastMathFlags()); |
| 646 | |
| 647 | Value *Tan = Builder.CreateFDiv(L: Y, R: X); |
| 648 | |
| 649 | CallInst *Atan = Builder.CreateIntrinsicWithoutFolding( |
| 650 | RetTy: Ty, ID: Intrinsic::atan, Args: {Tan}, FMFSource: nullptr, Name: "Elt.Atan" ); |
| 651 | Atan->setTailCall(Orig->isTailCall()); |
| 652 | Atan->setAttributes(Orig->getAttributes()); |
| 653 | |
| 654 | // Modify atan result based on https://en.wikipedia.org/wiki/Atan2. |
| 655 | Constant *Pi = ConstantFP::get(Ty, V: llvm::numbers::pi); |
| 656 | Constant *HalfPi = ConstantFP::get(Ty, V: llvm::numbers::pi / 2); |
| 657 | Constant *NegHalfPi = ConstantFP::get(Ty, V: -llvm::numbers::pi / 2); |
| 658 | Constant *Zero = ConstantFP::get(Ty, V: 0); |
| 659 | Value *AtanAddPi = Builder.CreateFAdd(L: Atan, R: Pi); |
| 660 | Value *AtanSubPi = Builder.CreateFSub(L: Atan, R: Pi); |
| 661 | |
| 662 | // x > 0 -> atan. |
| 663 | Value *Result = Atan; |
| 664 | Value *XLt0 = Builder.CreateFCmpOLT(LHS: X, RHS: Zero); |
| 665 | Value *XEq0 = Builder.CreateFCmpOEQ(LHS: X, RHS: Zero); |
| 666 | Value *YGe0 = Builder.CreateFCmpOGE(LHS: Y, RHS: Zero); |
| 667 | Value *YLt0 = Builder.CreateFCmpOLT(LHS: Y, RHS: Zero); |
| 668 | |
| 669 | // x < 0, y >= 0 -> atan + pi. |
| 670 | Value *XLt0AndYGe0 = Builder.CreateAnd(LHS: XLt0, RHS: YGe0); |
| 671 | Result = Builder.CreateSelect(C: XLt0AndYGe0, True: AtanAddPi, False: Result); |
| 672 | |
| 673 | // x < 0, y < 0 -> atan - pi. |
| 674 | Value *XLt0AndYLt0 = Builder.CreateAnd(LHS: XLt0, RHS: YLt0); |
| 675 | Result = Builder.CreateSelect(C: XLt0AndYLt0, True: AtanSubPi, False: Result); |
| 676 | |
| 677 | // x == 0, y < 0 -> -pi/2 |
| 678 | Value *XEq0AndYLt0 = Builder.CreateAnd(LHS: XEq0, RHS: YLt0); |
| 679 | Result = Builder.CreateSelect(C: XEq0AndYLt0, True: NegHalfPi, False: Result); |
| 680 | |
| 681 | // x == 0, y > 0 -> pi/2 |
| 682 | Value *XEq0AndYGe0 = Builder.CreateAnd(LHS: XEq0, RHS: YGe0); |
| 683 | Result = Builder.CreateSelect(C: XEq0AndYGe0, True: HalfPi, False: Result); |
| 684 | |
| 685 | return Result; |
| 686 | } |
| 687 | |
| 688 | template <bool LeftFunnel> |
| 689 | static Value *expandFunnelShiftIntrinsic(CallInst *Orig) { |
| 690 | Type *Ty = Orig->getType(); |
| 691 | Value *A = Orig->getOperand(i_nocapture: 0); |
| 692 | Value *B = Orig->getOperand(i_nocapture: 1); |
| 693 | Value *Shift = Orig->getOperand(i_nocapture: 2); |
| 694 | |
| 695 | IRBuilder<> Builder(Orig); |
| 696 | |
| 697 | assert(llvm::isPowerOf2_32(Ty->getScalarSizeInBits()) && |
| 698 | "Can't use Mask to compute modulo and inverse" ); |
| 699 | |
| 700 | // Note: if (Shift % BitWidth) == 0 then (BitWidth - Shift) == BitWidth, |
| 701 | // shifting by the bitwidth for shl/lshr returns a poisoned result. As such, |
| 702 | // we implement the same formula as LegalizerHelper::lowerFunnelShiftAsShifts. |
| 703 | // |
| 704 | // The funnel shift is expanded like so: |
| 705 | // fshl |
| 706 | // -> msb_extract((concat(A, B) << (Shift % BitWidth)), BitWidth) |
| 707 | // -> A << (Shift % BitWidth) | B >> 1 >> (BitWidth - 1 - (Shift % BitWidth)) |
| 708 | // fshr |
| 709 | // -> lsb_extract((concat(A, B) >> (Shift % BitWidth), BitWidth)) |
| 710 | // -> A << 1 << (BitWidth - 1 - (Shift % BitWidth)) | B >> (Shift % BitWidth) |
| 711 | |
| 712 | // (BitWidth - 1) -> Mask |
| 713 | Constant *Mask = ConstantInt::get(Ty, V: Ty->getScalarSizeInBits() - 1); |
| 714 | |
| 715 | // Shift % BitWidth |
| 716 | // -> Shift & (BitWidth - 1) |
| 717 | // -> Shift & Mask |
| 718 | Value *MaskedShift = Builder.CreateAnd(LHS: Shift, RHS: Mask); |
| 719 | |
| 720 | // (BitWidth - 1) - (Shift % BitWidth) |
| 721 | // -> ~Shift & (BitWidth - 1) |
| 722 | // -> ~Shift & Mask |
| 723 | Value *NotShift = Builder.CreateNot(V: Shift); |
| 724 | Value *InverseShift = Builder.CreateAnd(LHS: NotShift, RHS: Mask); |
| 725 | |
| 726 | Constant *One = ConstantInt::get(Ty, V: 1); |
| 727 | Value *ShiftedA; |
| 728 | Value *ShiftedB; |
| 729 | |
| 730 | if (LeftFunnel) { |
| 731 | ShiftedA = Builder.CreateShl(LHS: A, RHS: MaskedShift); |
| 732 | Value *ShiftB1 = Builder.CreateLShr(LHS: B, RHS: One); |
| 733 | ShiftedB = Builder.CreateLShr(LHS: ShiftB1, RHS: InverseShift); |
| 734 | } else { |
| 735 | Value *ShiftA1 = Builder.CreateShl(LHS: A, RHS: One); |
| 736 | ShiftedA = Builder.CreateShl(LHS: ShiftA1, RHS: InverseShift); |
| 737 | ShiftedB = Builder.CreateLShr(LHS: B, RHS: MaskedShift); |
| 738 | } |
| 739 | |
| 740 | Value *Result = Builder.CreateOr(LHS: ShiftedA, RHS: ShiftedB); |
| 741 | return Result; |
| 742 | } |
| 743 | |
| 744 | static Value *expandPowIntrinsic(CallInst *Orig, Intrinsic::ID IntrinsicId) { |
| 745 | |
| 746 | Value *X = Orig->getOperand(i_nocapture: 0); |
| 747 | Value *Y = Orig->getOperand(i_nocapture: 1); |
| 748 | Type *Ty = X->getType(); |
| 749 | IRBuilder<> Builder(Orig); |
| 750 | |
| 751 | if (IntrinsicId == Intrinsic::powi) |
| 752 | Y = Builder.CreateSIToFP(V: Y, DestTy: Ty); |
| 753 | |
| 754 | Value *Log2Call = |
| 755 | Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::log2, Args: {X}, FMFSource: nullptr, Name: "elt.log2" ); |
| 756 | auto *Mul = Builder.CreateFMul(L: Log2Call, R: Y); |
| 757 | CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding( |
| 758 | RetTy: Ty, ID: Intrinsic::exp2, Args: {Mul}, FMFSource: nullptr, Name: "elt.exp2" ); |
| 759 | Exp2Call->setTailCall(Orig->isTailCall()); |
| 760 | Exp2Call->setAttributes(Orig->getAttributes()); |
| 761 | return Exp2Call; |
| 762 | } |
| 763 | |
| 764 | static bool expandBufferLoadIntrinsic(CallInst *Orig, bool IsRaw) { |
| 765 | IRBuilder<> Builder(Orig); |
| 766 | |
| 767 | Type *BufferTy = Orig->getType()->getStructElementType(N: 0); |
| 768 | Type *ScalarTy = BufferTy->getScalarType(); |
| 769 | bool IsDouble = ScalarTy->isDoubleTy(); |
| 770 | assert(IsDouble || ScalarTy->isIntegerTy(64) && |
| 771 | "Only expand double or int64 scalars or vectors" ); |
| 772 | bool IsVector = false; |
| 773 | unsigned = 2; |
| 774 | if (auto *VT = dyn_cast<FixedVectorType>(Val: BufferTy)) { |
| 775 | ExtractNum = 2 * VT->getNumElements(); |
| 776 | IsVector = true; |
| 777 | assert(IsRaw || ExtractNum == 4 && "TypedBufferLoad vector must be size 2" ); |
| 778 | } |
| 779 | |
| 780 | SmallVector<Value *, 2> Loads; |
| 781 | Value *Result = PoisonValue::get(T: BufferTy); |
| 782 | unsigned Base = 0; |
| 783 | // If we need to extract more than 4 i32; we need to break it up into |
| 784 | // more than one load. LoadNum tells us how many i32s we are loading in |
| 785 | // each load |
| 786 | while (ExtractNum > 0) { |
| 787 | unsigned LoadNum = std::min(a: ExtractNum, b: 4u); |
| 788 | Type *Ty = VectorType::get(ElementType: Builder.getInt32Ty(), NumElements: LoadNum, Scalable: false); |
| 789 | |
| 790 | Type *LoadType = StructType::get(elt1: Ty, elts: Builder.getInt1Ty()); |
| 791 | Intrinsic::ID LoadIntrinsic = Intrinsic::dx_resource_load_typedbuffer; |
| 792 | SmallVector<Value *, 3> Args = {Orig->getOperand(i_nocapture: 0), Orig->getOperand(i_nocapture: 1)}; |
| 793 | if (IsRaw) { |
| 794 | LoadIntrinsic = Intrinsic::dx_resource_load_rawbuffer; |
| 795 | Value *Tmp = Builder.getInt32(C: 4 * Base * 2); |
| 796 | Value *Offset = Orig->getOperand(i_nocapture: 2); |
| 797 | Args.push_back(Elt: Offset); |
| 798 | unsigned AddressArg = isa<PoisonValue>(Val: Offset) ? 1 : 2; |
| 799 | if (Base != 0) |
| 800 | Args[AddressArg] = Builder.CreateAdd(LHS: Args[AddressArg], RHS: Tmp); |
| 801 | } |
| 802 | |
| 803 | Value *Load = Builder.CreateIntrinsic(RetTy: LoadType, ID: LoadIntrinsic, Args); |
| 804 | Loads.push_back(Elt: Load); |
| 805 | |
| 806 | // extract the buffer load's result |
| 807 | Value * = Builder.CreateExtractValue(Agg: Load, Idxs: {0}); |
| 808 | |
| 809 | SmallVector<Value *> ; |
| 810 | for (unsigned I = 0; I < LoadNum; ++I) |
| 811 | ExtractElements.push_back( |
| 812 | Elt: Builder.CreateExtractElement(Vec: Extract, Idx: Builder.getInt32(C: I))); |
| 813 | |
| 814 | // combine into double(s) or int64(s) |
| 815 | for (unsigned I = 0; I < LoadNum; I += 2) { |
| 816 | Value *Combined = nullptr; |
| 817 | if (IsDouble) |
| 818 | // For doubles, use dx_asdouble intrinsic |
| 819 | Combined = Builder.CreateIntrinsic( |
| 820 | RetTy: Builder.getDoubleTy(), ID: Intrinsic::dx_asdouble, |
| 821 | Args: {ExtractElements[I], ExtractElements[I + 1]}); |
| 822 | else { |
| 823 | // For int64, manually combine two int32s |
| 824 | // First, zero-extend both values to i64 |
| 825 | Value *Lo = |
| 826 | Builder.CreateZExt(V: ExtractElements[I], DestTy: Builder.getInt64Ty()); |
| 827 | Value *Hi = |
| 828 | Builder.CreateZExt(V: ExtractElements[I + 1], DestTy: Builder.getInt64Ty()); |
| 829 | // Shift the high bits left by 32 bits |
| 830 | Value *ShiftedHi = Builder.CreateShl(LHS: Hi, RHS: Builder.getInt64(C: 32)); |
| 831 | // OR the high and low bits together |
| 832 | Combined = Builder.CreateOr(LHS: Lo, RHS: ShiftedHi); |
| 833 | } |
| 834 | |
| 835 | if (IsVector) |
| 836 | Result = Builder.CreateInsertElement(Vec: Result, NewElt: Combined, |
| 837 | Idx: Builder.getInt32(C: (I / 2) + Base)); |
| 838 | else |
| 839 | Result = Combined; |
| 840 | } |
| 841 | |
| 842 | ExtractNum -= LoadNum; |
| 843 | Base += LoadNum / 2; |
| 844 | } |
| 845 | |
| 846 | Value *CheckBit = nullptr; |
| 847 | for (User *U : make_early_inc_range(Range: Orig->users())) { |
| 848 | // If it's not a ExtractValueInst, we don't know how to |
| 849 | // handle it |
| 850 | auto *EVI = dyn_cast<ExtractValueInst>(Val: U); |
| 851 | if (!EVI) |
| 852 | llvm_unreachable("Unexpected user of typedbufferload" ); |
| 853 | |
| 854 | ArrayRef<unsigned> Indices = EVI->getIndices(); |
| 855 | assert(Indices.size() == 1); |
| 856 | |
| 857 | if (Indices[0] == 0) { |
| 858 | // Use of the value(s) |
| 859 | EVI->replaceAllUsesWith(V: Result); |
| 860 | } else { |
| 861 | // Use of the check bit |
| 862 | assert(Indices[0] == 1 && "Unexpected type for typedbufferload" ); |
| 863 | // Note: This does not always match the historical behaviour of DXC. |
| 864 | // See https://github.com/microsoft/DirectXShaderCompiler/issues/7622 |
| 865 | if (!CheckBit) { |
| 866 | SmallVector<Value *, 2> CheckBits; |
| 867 | for (Value *L : Loads) |
| 868 | CheckBits.push_back(Elt: Builder.CreateExtractValue(Agg: L, Idxs: {1})); |
| 869 | CheckBit = Builder.CreateAnd(Ops: CheckBits); |
| 870 | } |
| 871 | EVI->replaceAllUsesWith(V: CheckBit); |
| 872 | } |
| 873 | EVI->eraseFromParent(); |
| 874 | } |
| 875 | Orig->eraseFromParent(); |
| 876 | return true; |
| 877 | } |
| 878 | |
| 879 | static bool expandBufferStoreIntrinsic(CallInst *Orig, bool IsRaw) { |
| 880 | IRBuilder<> Builder(Orig); |
| 881 | |
| 882 | unsigned ValIndex = IsRaw ? 3 : 2; |
| 883 | Type *BufferTy = Orig->getFunctionType()->getParamType(i: ValIndex); |
| 884 | Type *ScalarTy = BufferTy->getScalarType(); |
| 885 | bool IsDouble = ScalarTy->isDoubleTy(); |
| 886 | assert((IsDouble || ScalarTy->isIntegerTy(64)) && |
| 887 | "Only expand double or int64 scalars or vectors" ); |
| 888 | |
| 889 | // Determine if we're dealing with a vector or scalar |
| 890 | bool IsVector = false; |
| 891 | unsigned = 2; |
| 892 | unsigned VecLen = 0; |
| 893 | if (auto *VT = dyn_cast<FixedVectorType>(Val: BufferTy)) { |
| 894 | VecLen = VT->getNumElements(); |
| 895 | assert(IsRaw || VecLen == 2 && "TypedBufferStore vector must be size 2" ); |
| 896 | ExtractNum = VecLen * 2; |
| 897 | IsVector = true; |
| 898 | } |
| 899 | |
| 900 | // Create the appropriate vector type for the result |
| 901 | Type *Int32Ty = Builder.getInt32Ty(); |
| 902 | Type *ResultTy = VectorType::get(ElementType: Int32Ty, NumElements: ExtractNum, Scalable: false); |
| 903 | Value *Val = PoisonValue::get(T: ResultTy); |
| 904 | |
| 905 | Type *SplitElementTy = Int32Ty; |
| 906 | if (IsVector) |
| 907 | SplitElementTy = VectorType::get(ElementType: SplitElementTy, NumElements: VecLen, Scalable: false); |
| 908 | |
| 909 | Value *LowBits = nullptr; |
| 910 | Value *HighBits = nullptr; |
| 911 | // Split the 64-bit values into 32-bit components |
| 912 | if (IsDouble) { |
| 913 | auto *SplitTy = llvm::StructType::get(elt1: SplitElementTy, elts: SplitElementTy); |
| 914 | Value *Split = Builder.CreateIntrinsic(RetTy: SplitTy, ID: Intrinsic::dx_splitdouble, |
| 915 | Args: {Orig->getOperand(i_nocapture: ValIndex)}); |
| 916 | LowBits = Builder.CreateExtractValue(Agg: Split, Idxs: 0); |
| 917 | HighBits = Builder.CreateExtractValue(Agg: Split, Idxs: 1); |
| 918 | } else { |
| 919 | // Handle int64 type(s) |
| 920 | Value *InputVal = Orig->getOperand(i_nocapture: ValIndex); |
| 921 | Constant *ShiftAmt = Builder.getInt64(C: 32); |
| 922 | if (IsVector) |
| 923 | ShiftAmt = |
| 924 | ConstantVector::getSplat(EC: ElementCount::getFixed(MinVal: VecLen), Elt: ShiftAmt); |
| 925 | |
| 926 | // Split into low and high 32-bit parts |
| 927 | LowBits = Builder.CreateTrunc(V: InputVal, DestTy: SplitElementTy); |
| 928 | Value *ShiftedVal = Builder.CreateLShr(LHS: InputVal, RHS: ShiftAmt); |
| 929 | HighBits = Builder.CreateTrunc(V: ShiftedVal, DestTy: SplitElementTy); |
| 930 | } |
| 931 | |
| 932 | if (IsVector) { |
| 933 | SmallVector<int, 8> Mask; |
| 934 | for (unsigned I = 0; I < VecLen; ++I) { |
| 935 | Mask.push_back(Elt: I); |
| 936 | Mask.push_back(Elt: I + VecLen); |
| 937 | } |
| 938 | Val = Builder.CreateShuffleVector(V1: LowBits, V2: HighBits, Mask); |
| 939 | } else { |
| 940 | Val = Builder.CreateInsertElement(Vec: Val, NewElt: LowBits, Idx: Builder.getInt32(C: 0)); |
| 941 | Val = Builder.CreateInsertElement(Vec: Val, NewElt: HighBits, Idx: Builder.getInt32(C: 1)); |
| 942 | } |
| 943 | |
| 944 | // If we need to extract more than 4 i32; we need to break it up into |
| 945 | // more than one store. StoreNum tells us how many i32s we are storing in |
| 946 | // each store |
| 947 | unsigned Base = 0; |
| 948 | while (ExtractNum > 0) { |
| 949 | unsigned StoreNum = std::min(a: ExtractNum, b: 4u); |
| 950 | |
| 951 | Intrinsic::ID StoreIntrinsic = Intrinsic::dx_resource_store_typedbuffer; |
| 952 | SmallVector<Value *, 4> Args = {Orig->getOperand(i_nocapture: 0), Orig->getOperand(i_nocapture: 1)}; |
| 953 | if (IsRaw) { |
| 954 | StoreIntrinsic = Intrinsic::dx_resource_store_rawbuffer; |
| 955 | Value *Tmp = Builder.getInt32(C: 4 * Base); |
| 956 | Value *Offset = Orig->getOperand(i_nocapture: 2); |
| 957 | Args.push_back(Elt: Offset); |
| 958 | unsigned AddressArg = isa<PoisonValue>(Val: Offset) ? 1 : 2; |
| 959 | if (Base != 0) |
| 960 | Args[AddressArg] = Builder.CreateAdd(LHS: Args[AddressArg], RHS: Tmp); |
| 961 | } |
| 962 | |
| 963 | SmallVector<int, 4> Mask; |
| 964 | for (unsigned I = 0; I < StoreNum; ++I) { |
| 965 | Mask.push_back(Elt: Base + I); |
| 966 | } |
| 967 | |
| 968 | Value *SubVal = Val; |
| 969 | if (VecLen > 2) |
| 970 | SubVal = Builder.CreateShuffleVector(V: Val, Mask); |
| 971 | |
| 972 | Args.push_back(Elt: SubVal); |
| 973 | // Create the final intrinsic call |
| 974 | Builder.CreateIntrinsic(RetTy: Builder.getVoidTy(), ID: StoreIntrinsic, Args); |
| 975 | |
| 976 | ExtractNum -= StoreNum; |
| 977 | Base += StoreNum; |
| 978 | } |
| 979 | Orig->eraseFromParent(); |
| 980 | return true; |
| 981 | } |
| 982 | |
| 983 | static Intrinsic::ID getMaxForClamp(Intrinsic::ID ClampIntrinsic) { |
| 984 | if (ClampIntrinsic == Intrinsic::dx_uclamp) |
| 985 | return Intrinsic::umax; |
| 986 | if (ClampIntrinsic == Intrinsic::dx_sclamp) |
| 987 | return Intrinsic::smax; |
| 988 | assert(ClampIntrinsic == Intrinsic::dx_nclamp); |
| 989 | return Intrinsic::maxnum; |
| 990 | } |
| 991 | |
| 992 | static Intrinsic::ID getMinForClamp(Intrinsic::ID ClampIntrinsic) { |
| 993 | if (ClampIntrinsic == Intrinsic::dx_uclamp) |
| 994 | return Intrinsic::umin; |
| 995 | if (ClampIntrinsic == Intrinsic::dx_sclamp) |
| 996 | return Intrinsic::smin; |
| 997 | assert(ClampIntrinsic == Intrinsic::dx_nclamp); |
| 998 | return Intrinsic::minnum; |
| 999 | } |
| 1000 | |
| 1001 | static Value *expandClampIntrinsic(CallInst *Orig, |
| 1002 | Intrinsic::ID ClampIntrinsic) { |
| 1003 | Value *X = Orig->getOperand(i_nocapture: 0); |
| 1004 | Value *Min = Orig->getOperand(i_nocapture: 1); |
| 1005 | Value *Max = Orig->getOperand(i_nocapture: 2); |
| 1006 | Type *Ty = X->getType(); |
| 1007 | IRBuilder<> Builder(Orig); |
| 1008 | auto *MaxCall = Builder.CreateIntrinsic(RetTy: Ty, ID: getMaxForClamp(ClampIntrinsic), |
| 1009 | Args: {X, Min}, FMFSource: nullptr, Name: "dx.max" ); |
| 1010 | return Builder.CreateIntrinsic(RetTy: Ty, ID: getMinForClamp(ClampIntrinsic), |
| 1011 | Args: {MaxCall, Max}, FMFSource: nullptr, Name: "dx.min" ); |
| 1012 | } |
| 1013 | |
| 1014 | static Value *expandSignIntrinsic(CallInst *Orig) { |
| 1015 | Value *X = Orig->getOperand(i_nocapture: 0); |
| 1016 | Type *Ty = X->getType(); |
| 1017 | Type *ScalarTy = Ty->getScalarType(); |
| 1018 | Type *RetTy = Orig->getType(); |
| 1019 | Constant *Zero = Constant::getNullValue(Ty); |
| 1020 | |
| 1021 | IRBuilder<> Builder(Orig); |
| 1022 | |
| 1023 | Value *GT; |
| 1024 | Value *LT; |
| 1025 | if (ScalarTy->isFloatingPointTy()) { |
| 1026 | GT = Builder.CreateFCmpOLT(LHS: Zero, RHS: X); |
| 1027 | LT = Builder.CreateFCmpOLT(LHS: X, RHS: Zero); |
| 1028 | } else { |
| 1029 | assert(ScalarTy->isIntegerTy()); |
| 1030 | GT = Builder.CreateICmpSLT(LHS: Zero, RHS: X); |
| 1031 | LT = Builder.CreateICmpSLT(LHS: X, RHS: Zero); |
| 1032 | } |
| 1033 | |
| 1034 | Value *ZextGT = Builder.CreateZExt(V: GT, DestTy: RetTy); |
| 1035 | Value *ZextLT = Builder.CreateZExt(V: LT, DestTy: RetTy); |
| 1036 | |
| 1037 | return Builder.CreateSub(LHS: ZextGT, RHS: ZextLT); |
| 1038 | } |
| 1039 | |
| 1040 | // Expand llvm.copysign by combining the sign bit with the magnitude bits using |
| 1041 | // bitwise operations. |
| 1042 | static Value *expandCopySignIntrinsic(CallInst *Orig) { |
| 1043 | Value *Magnitude = Orig->getOperand(i_nocapture: 0); |
| 1044 | Value *Sign = Orig->getOperand(i_nocapture: 1); |
| 1045 | Type *Ty = Orig->getType(); |
| 1046 | |
| 1047 | IRBuilder<> Builder(Orig); |
| 1048 | |
| 1049 | bool IsDouble = Ty->getScalarType()->isDoubleTy(); |
| 1050 | unsigned BitWidth = IsDouble ? 32 : Ty->getScalarSizeInBits(); |
| 1051 | Type *IntTy = Ty->getWithNewType(EltTy: Builder.getIntNTy(N: BitWidth)); |
| 1052 | |
| 1053 | auto CopySignBit = [&](Value *MagnitudeInt, Value *SignInt) { |
| 1054 | APInt SignMaskVal = APInt::getSignMask(BitWidth); |
| 1055 | // `ConstantInt::get` broadcasts to a splat when `IntTy` is a vector. |
| 1056 | Constant *SignMask = ConstantInt::get(Ty: IntTy, V: SignMaskVal); |
| 1057 | Constant *NotSignMask = ConstantInt::get(Ty: IntTy, V: ~SignMaskVal); |
| 1058 | |
| 1059 | Value *MagnitudeBits = Builder.CreateAnd(LHS: MagnitudeInt, RHS: NotSignMask); |
| 1060 | Value *SignBits = Builder.CreateAnd(LHS: SignInt, RHS: SignMask); |
| 1061 | return Builder.CreateOr(LHS: MagnitudeBits, RHS: SignBits); |
| 1062 | }; |
| 1063 | |
| 1064 | // Avoid i64 bitwise ops, which require the Int64Ops shader feature. |
| 1065 | if (IsDouble) { |
| 1066 | auto *SplitTy = StructType::get(elt1: IntTy, elts: IntTy); |
| 1067 | Value *MagnitudeHalves = Builder.CreateIntrinsic( |
| 1068 | RetTy: SplitTy, ID: Intrinsic::dx_splitdouble, Args: {Magnitude}); |
| 1069 | Value *SignHalves = |
| 1070 | Builder.CreateIntrinsic(RetTy: SplitTy, ID: Intrinsic::dx_splitdouble, Args: {Sign}); |
| 1071 | Value *MagnitudeLow = Builder.CreateExtractValue(Agg: MagnitudeHalves, Idxs: 0); |
| 1072 | Value *MagnitudeHigh = Builder.CreateExtractValue(Agg: MagnitudeHalves, Idxs: 1); |
| 1073 | Value *SignHigh = Builder.CreateExtractValue(Agg: SignHalves, Idxs: 1); |
| 1074 | |
| 1075 | Value *CombinedHigh = CopySignBit(MagnitudeHigh, SignHigh); |
| 1076 | return Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::dx_asdouble, |
| 1077 | Args: {MagnitudeLow, CombinedHigh}); |
| 1078 | } |
| 1079 | |
| 1080 | Value *MagnitudeInt = Builder.CreateBitCast(V: Magnitude, DestTy: IntTy); |
| 1081 | Value *SignInt = Builder.CreateBitCast(V: Sign, DestTy: IntTy); |
| 1082 | Value *CombinedInt = CopySignBit(MagnitudeInt, SignInt); |
| 1083 | return Builder.CreateBitCast(V: CombinedInt, DestTy: Ty); |
| 1084 | } |
| 1085 | |
| 1086 | // Expand llvm.matrix.multiply by extracting row/column vectors and computing |
| 1087 | // dot products. |
| 1088 | // Result[r,c] = dot(row_r(LHS), col_c(RHS)) |
| 1089 | // Element (r,c) is at index c*NumRows + r (column-major). |
| 1090 | static Value *expandMatrixMultiply(CallInst *Orig) { |
| 1091 | Value *LHS = Orig->getArgOperand(i: 0); |
| 1092 | Value *RHS = Orig->getArgOperand(i: 1); |
| 1093 | unsigned LHSRows = cast<ConstantInt>(Val: Orig->getArgOperand(i: 2))->getZExtValue(); |
| 1094 | unsigned LHSCols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 3))->getZExtValue(); |
| 1095 | unsigned RHSCols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 4))->getZExtValue(); |
| 1096 | |
| 1097 | auto *RetTy = cast<FixedVectorType>(Val: Orig->getType()); |
| 1098 | Type *EltTy = RetTy->getElementType(); |
| 1099 | bool IsFP = EltTy->isFloatingPointTy(); |
| 1100 | |
| 1101 | IRBuilder<> Builder(Orig); |
| 1102 | |
| 1103 | // Column-major indexing: |
| 1104 | // LHS row R, element K: index = K * LHSRows + R |
| 1105 | // RHS col C, element K: index = C * LHSCols + K |
| 1106 | Value *Result = PoisonValue::get(T: RetTy); |
| 1107 | |
| 1108 | // Extract all scalar elements from LHS and RHS once, then reuse them. |
| 1109 | unsigned LHSSize = LHSRows * LHSCols; |
| 1110 | unsigned RHSSize = LHSCols * RHSCols; |
| 1111 | SmallVector<Value *, 16> LHSElts(LHSSize); |
| 1112 | SmallVector<Value *, 16> RHSElts(RHSSize); |
| 1113 | for (unsigned I = 0; I < LHSSize; ++I) |
| 1114 | LHSElts[I] = Builder.CreateExtractElement(Vec: LHS, Idx: I); |
| 1115 | for (unsigned I = 0; I < RHSSize; ++I) |
| 1116 | RHSElts[I] = Builder.CreateExtractElement(Vec: RHS, Idx: I); |
| 1117 | |
| 1118 | // Choose the appropriate scalar-arg dot intrinsic for floats. |
| 1119 | // K=1 and double types use scalar expansion instead. |
| 1120 | Intrinsic::ID FloatDotID = Intrinsic::not_intrinsic; |
| 1121 | bool UseScalarFP = IsFP && (EltTy->isDoubleTy() || LHSCols == 1); |
| 1122 | if (IsFP && !UseScalarFP) { |
| 1123 | switch (LHSCols) { |
| 1124 | case 2: |
| 1125 | FloatDotID = Intrinsic::dx_dot2; |
| 1126 | break; |
| 1127 | case 3: |
| 1128 | FloatDotID = Intrinsic::dx_dot3; |
| 1129 | break; |
| 1130 | case 4: |
| 1131 | FloatDotID = Intrinsic::dx_dot4; |
| 1132 | break; |
| 1133 | default: |
| 1134 | reportFatalUsageError( |
| 1135 | reason: "Invalid matrix inner dimension for dot product: must be 2-4" ); |
| 1136 | return nullptr; |
| 1137 | } |
| 1138 | } |
| 1139 | |
| 1140 | for (unsigned C = 0; C < RHSCols; ++C) { |
| 1141 | for (unsigned R = 0; R < LHSRows; ++R) { |
| 1142 | // Gather row R from LHS and column C from RHS. |
| 1143 | SmallVector<Value *, 4> RowElts, ColElts; |
| 1144 | for (unsigned K = 0; K < LHSCols; ++K) { |
| 1145 | RowElts.push_back(Elt: LHSElts[K * LHSRows + R]); |
| 1146 | ColElts.push_back(Elt: RHSElts[C * LHSCols + K]); |
| 1147 | } |
| 1148 | |
| 1149 | Value *Dot; |
| 1150 | if (UseScalarFP) { |
| 1151 | // Scalar fmul+fmuladd expansion for double types and K=1. |
| 1152 | Dot = Builder.CreateFMul(L: RowElts[0], R: ColElts[0]); |
| 1153 | for (unsigned K = 1; K < LHSCols; ++K) |
| 1154 | Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: Intrinsic::fmuladd, |
| 1155 | Args: {RowElts[K], ColElts[K], Dot}); |
| 1156 | } else if (IsFP) { |
| 1157 | // Emit scalar-arg DXIL dot directly (dx.dot2/dx.dot3/dx.dot4). |
| 1158 | SmallVector<Value *, 8> Args; |
| 1159 | Args.append(in_start: RowElts.begin(), in_end: RowElts.end()); |
| 1160 | Args.append(in_start: ColElts.begin(), in_end: ColElts.end()); |
| 1161 | Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: FloatDotID, Args); |
| 1162 | } else { |
| 1163 | // Integer: emit multiply + imad chain. |
| 1164 | Dot = Builder.CreateMul(LHS: RowElts[0], RHS: ColElts[0]); |
| 1165 | for (unsigned K = 1; K < LHSCols; ++K) |
| 1166 | Dot = Builder.CreateIntrinsic(RetTy: EltTy, ID: Intrinsic::dx_imad, |
| 1167 | Args: {RowElts[K], ColElts[K], Dot}); |
| 1168 | } |
| 1169 | unsigned ResIdx = C * LHSRows + R; |
| 1170 | Result = Builder.CreateInsertElement(Vec: Result, NewElt: Dot, Idx: ResIdx); |
| 1171 | } |
| 1172 | } |
| 1173 | return Result; |
| 1174 | } |
| 1175 | |
| 1176 | // Expand llvm.matrix.transpose as a shufflevector that permutes elements |
| 1177 | // from column-major source to column-major transposed layout. |
| 1178 | // Element (r,c) at index c*Rows + r moves to index r*Cols + c. |
| 1179 | static Value *expandMatrixTranspose(CallInst *Orig) { |
| 1180 | Value *Mat = Orig->getArgOperand(i: 0); |
| 1181 | unsigned Rows = cast<ConstantInt>(Val: Orig->getArgOperand(i: 1))->getZExtValue(); |
| 1182 | unsigned Cols = cast<ConstantInt>(Val: Orig->getArgOperand(i: 2))->getZExtValue(); |
| 1183 | |
| 1184 | unsigned NumElts = Rows * Cols; |
| 1185 | SmallVector<int, 16> Mask(NumElts); |
| 1186 | for (unsigned I = 0; I < NumElts; ++I) |
| 1187 | Mask[I] = (I % Cols) * Rows + (I / Cols); |
| 1188 | |
| 1189 | IRBuilder<> Builder(Orig); |
| 1190 | return Builder.CreateShuffleVector(V: Mat, Mask); |
| 1191 | } |
| 1192 | |
| 1193 | // Scalarize a vector int_dx_store_output call into per-component scalar calls. |
| 1194 | // The DXIL StoreOutput op is per-component; vector intrinsics are split here |
| 1195 | // so that DXILOpLowering sees only scalar variants. |
| 1196 | static bool expandStoreOutput(CallInst *Orig) { |
| 1197 | auto *VT = dyn_cast<FixedVectorType>(Val: Orig->getArgOperand(i: 3)->getType()); |
| 1198 | if (!VT) |
| 1199 | return false; // already scalar, nothing to expand |
| 1200 | |
| 1201 | IRBuilder<> Builder(Orig); |
| 1202 | Module *M = Orig->getModule(); |
| 1203 | Type *Int8Ty = Builder.getInt8Ty(); |
| 1204 | Type *Int32Ty = Builder.getInt32Ty(); |
| 1205 | Type *ScalarTy = VT->getElementType(); |
| 1206 | unsigned NumElems = VT->getNumElements(); |
| 1207 | |
| 1208 | Value *SigElementId = Orig->getArgOperand(i: 0); |
| 1209 | Value *RowIndex = Orig->getArgOperand(i: 1); |
| 1210 | Value *StartCol = Orig->getArgOperand(i: 2); // i8 |
| 1211 | Value *Data = Orig->getArgOperand(i: 3); |
| 1212 | Value *StartColI32 = Builder.CreateZExt(V: StartCol, DestTy: Int32Ty); |
| 1213 | |
| 1214 | Function *ScalarFn = Intrinsic::getOrInsertDeclaration( |
| 1215 | M, id: Intrinsic::dx_store_output, OverloadTys: {ScalarTy}); |
| 1216 | |
| 1217 | for (unsigned I = 0; I < NumElems; ++I) { |
| 1218 | Value *Scalar = |
| 1219 | Builder.CreateExtractElement(Vec: Data, Idx: ConstantInt::get(Ty: Int32Ty, V: I)); |
| 1220 | Value *ColIdx = |
| 1221 | Builder.CreateAdd(LHS: StartColI32, RHS: ConstantInt::get(Ty: Int32Ty, V: I)); |
| 1222 | Value *ColI8 = Builder.CreateTrunc(V: ColIdx, DestTy: Int8Ty); |
| 1223 | Builder.CreateCall(Callee: ScalarFn, Args: {SigElementId, RowIndex, ColI8, Scalar}); |
| 1224 | } |
| 1225 | |
| 1226 | Orig->eraseFromParent(); |
| 1227 | return true; |
| 1228 | } |
| 1229 | |
| 1230 | // Scalarize a vector int_dx_load_input call into per-component scalar calls |
| 1231 | // and reassemble the vector. The DXIL LoadInput op is per-component. |
| 1232 | static Value *expandLoadInput(CallInst *Orig) { |
| 1233 | auto *VT = dyn_cast<FixedVectorType>(Val: Orig->getType()); |
| 1234 | if (!VT) |
| 1235 | return nullptr; // already scalar, nothing to expand |
| 1236 | |
| 1237 | IRBuilder<> Builder(Orig); |
| 1238 | Module *M = Orig->getModule(); |
| 1239 | Type *Int8Ty = Builder.getInt8Ty(); |
| 1240 | Type *Int32Ty = Builder.getInt32Ty(); |
| 1241 | Type *ScalarTy = VT->getElementType(); |
| 1242 | unsigned NumElems = VT->getNumElements(); |
| 1243 | |
| 1244 | Value *SigElementId = Orig->getArgOperand(i: 0); |
| 1245 | Value *RowIndex = Orig->getArgOperand(i: 1); |
| 1246 | Value *StartCol = Orig->getArgOperand(i: 2); // i8 |
| 1247 | Value *GsVertexOrPrimIndex = Orig->getArgOperand(i: 3); |
| 1248 | Value *StartColI32 = Builder.CreateZExt(V: StartCol, DestTy: Int32Ty); |
| 1249 | |
| 1250 | Function *ScalarFn = Intrinsic::getOrInsertDeclaration( |
| 1251 | M, id: Intrinsic::dx_load_input, OverloadTys: {ScalarTy}); |
| 1252 | |
| 1253 | Value *Vec = PoisonValue::get(T: VT); |
| 1254 | for (unsigned I = 0; I < NumElems; ++I) { |
| 1255 | Value *ColIdx = |
| 1256 | Builder.CreateAdd(LHS: StartColI32, RHS: ConstantInt::get(Ty: Int32Ty, V: I)); |
| 1257 | Value *ColI8 = Builder.CreateTrunc(V: ColIdx, DestTy: Int8Ty); |
| 1258 | Value *Scalar = Builder.CreateCall( |
| 1259 | Callee: ScalarFn, Args: {SigElementId, RowIndex, ColI8, GsVertexOrPrimIndex}); |
| 1260 | Vec = |
| 1261 | Builder.CreateInsertElement(Vec, NewElt: Scalar, Idx: ConstantInt::get(Ty: Int32Ty, V: I)); |
| 1262 | } |
| 1263 | |
| 1264 | return Vec; |
| 1265 | } |
| 1266 | |
| 1267 | static Value *expandQuadAll(CallInst *Orig) { |
| 1268 | IRBuilder<> Builder(Orig); |
| 1269 | SmallVector<OperandBundleDef, 1> Bundles; |
| 1270 | Orig->getOperandBundlesAsDefs(Defs&: Bundles); |
| 1271 | |
| 1272 | Value *Cond = |
| 1273 | Builder.CreateSExt(V: Orig->getArgOperand(i: 0), DestTy: Builder.getInt32Ty()); |
| 1274 | Value *Result = Cond; |
| 1275 | for (Intrinsic::ID ID : |
| 1276 | {Intrinsic::dx_quad_read_across_x, Intrinsic::dx_quad_read_across_y, |
| 1277 | Intrinsic::dx_quad_read_across_diagonal}) { |
| 1278 | Value *QuadValue = Builder.CreateIntrinsic(ID, OverloadTypes: {Builder.getInt32Ty()}, Args: Cond, |
| 1279 | FMFSource: nullptr, Name: "" , OpBundles: Bundles); |
| 1280 | Result = Builder.CreateAnd(LHS: Result, RHS: QuadValue); |
| 1281 | } |
| 1282 | return Builder.CreateTrunc(V: Result, DestTy: Builder.getInt1Ty()); |
| 1283 | } |
| 1284 | |
| 1285 | static bool expandIntrinsic(Function &F, CallInst *Orig) { |
| 1286 | Value *Result = nullptr; |
| 1287 | Intrinsic::ID IntrinsicId = F.getIntrinsicID(); |
| 1288 | switch (IntrinsicId) { |
| 1289 | case Intrinsic::abs: |
| 1290 | Result = expandAbs(Orig); |
| 1291 | break; |
| 1292 | case Intrinsic::assume: |
| 1293 | Orig->eraseFromParent(); |
| 1294 | return true; |
| 1295 | case Intrinsic::atan2: |
| 1296 | Result = expandAtan2Intrinsic(Orig); |
| 1297 | break; |
| 1298 | case Intrinsic::copysign: |
| 1299 | Result = expandCopySignIntrinsic(Orig); |
| 1300 | break; |
| 1301 | case Intrinsic::fshl: |
| 1302 | Result = expandFunnelShiftIntrinsic<true>(Orig); |
| 1303 | break; |
| 1304 | case Intrinsic::fshr: |
| 1305 | Result = expandFunnelShiftIntrinsic<false>(Orig); |
| 1306 | break; |
| 1307 | case Intrinsic::exp: |
| 1308 | Result = expandExpIntrinsic(Orig); |
| 1309 | break; |
| 1310 | case Intrinsic::is_fpclass: |
| 1311 | Result = expandIsFPClass(Orig); |
| 1312 | break; |
| 1313 | case Intrinsic::log: |
| 1314 | Result = expandLogIntrinsic(Orig); |
| 1315 | break; |
| 1316 | case Intrinsic::log10: |
| 1317 | Result = expandLog10Intrinsic(Orig); |
| 1318 | break; |
| 1319 | case Intrinsic::pow: |
| 1320 | case Intrinsic::powi: |
| 1321 | Result = expandPowIntrinsic(Orig, IntrinsicId); |
| 1322 | break; |
| 1323 | case Intrinsic::vector_reduce_and: |
| 1324 | case Intrinsic::vector_reduce_or: |
| 1325 | Result = expandVecReduceAndOr(Orig, IntrinsicId); |
| 1326 | break; |
| 1327 | case Intrinsic::dx_uclamp: |
| 1328 | case Intrinsic::dx_sclamp: |
| 1329 | case Intrinsic::dx_nclamp: |
| 1330 | Result = expandClampIntrinsic(Orig, ClampIntrinsic: IntrinsicId); |
| 1331 | break; |
| 1332 | case Intrinsic::dx_isfinite: |
| 1333 | Result = expand16BitIsFinite(Orig); |
| 1334 | break; |
| 1335 | case Intrinsic::dx_isinf: |
| 1336 | Result = expand16BitIsInf(Orig); |
| 1337 | break; |
| 1338 | case Intrinsic::dx_isnan: |
| 1339 | Result = expand16BitIsNaN(Orig); |
| 1340 | break; |
| 1341 | case Intrinsic::dx_fdot: |
| 1342 | Result = expandFloatDotIntrinsic(Orig); |
| 1343 | break; |
| 1344 | case Intrinsic::dx_sdot: |
| 1345 | case Intrinsic::dx_udot: |
| 1346 | Result = expandIntegerDotIntrinsic(Orig, DotIntrinsic: IntrinsicId); |
| 1347 | break; |
| 1348 | case Intrinsic::dx_sign: |
| 1349 | Result = expandSignIntrinsic(Orig); |
| 1350 | break; |
| 1351 | case Intrinsic::dx_quad_all: |
| 1352 | Result = expandQuadAll(Orig); |
| 1353 | break; |
| 1354 | case Intrinsic::dx_load_input: |
| 1355 | Result = expandLoadInput(Orig); |
| 1356 | break; |
| 1357 | case Intrinsic::dx_store_output: |
| 1358 | if (expandStoreOutput(Orig)) |
| 1359 | return true; |
| 1360 | break; |
| 1361 | case Intrinsic::dx_resource_load_rawbuffer: |
| 1362 | if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ true)) |
| 1363 | return true; |
| 1364 | break; |
| 1365 | case Intrinsic::dx_resource_store_rawbuffer: |
| 1366 | if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ true)) |
| 1367 | return true; |
| 1368 | break; |
| 1369 | case Intrinsic::dx_resource_load_typedbuffer: |
| 1370 | if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ false)) |
| 1371 | return true; |
| 1372 | break; |
| 1373 | case Intrinsic::dx_resource_store_typedbuffer: |
| 1374 | if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ false)) |
| 1375 | return true; |
| 1376 | break; |
| 1377 | case Intrinsic::usub_sat: |
| 1378 | Result = expandUsubSat(Orig); |
| 1379 | break; |
| 1380 | case Intrinsic::umul_with_overflow: |
| 1381 | case Intrinsic::smul_with_overflow: |
| 1382 | Result = expandMulWithOverflow(Orig, /*Signed=*/IntrinsicId == |
| 1383 | Intrinsic::smul_with_overflow); |
| 1384 | break; |
| 1385 | case Intrinsic::vector_reduce_add: |
| 1386 | case Intrinsic::vector_reduce_fadd: |
| 1387 | Result = expandVecReduceAdd(Orig, IntrinsicId); |
| 1388 | break; |
| 1389 | case Intrinsic::matrix_multiply: |
| 1390 | Result = expandMatrixMultiply(Orig); |
| 1391 | break; |
| 1392 | case Intrinsic::matrix_transpose: |
| 1393 | Result = expandMatrixTranspose(Orig); |
| 1394 | break; |
| 1395 | } |
| 1396 | if (Result) { |
| 1397 | Orig->replaceAllUsesWith(V: Result); |
| 1398 | Orig->eraseFromParent(); |
| 1399 | return true; |
| 1400 | } |
| 1401 | return false; |
| 1402 | } |
| 1403 | |
| 1404 | static bool expansionIntrinsics(Module &M) { |
| 1405 | for (auto &F : make_early_inc_range(Range: M.functions())) { |
| 1406 | if (!isIntrinsicExpansion(F)) |
| 1407 | continue; |
| 1408 | bool IntrinsicExpanded = false; |
| 1409 | for (User *U : make_early_inc_range(Range: F.users())) { |
| 1410 | auto *IntrinsicCall = dyn_cast<CallInst>(Val: U); |
| 1411 | if (!IntrinsicCall) |
| 1412 | continue; |
| 1413 | IntrinsicExpanded = expandIntrinsic(F, Orig: IntrinsicCall); |
| 1414 | } |
| 1415 | if (F.user_empty() && IntrinsicExpanded) |
| 1416 | F.eraseFromParent(); |
| 1417 | } |
| 1418 | return true; |
| 1419 | } |
| 1420 | |
| 1421 | PreservedAnalyses DXILIntrinsicExpansionPass::run(Module &M, |
| 1422 | ModuleAnalysisManager &) { |
| 1423 | if (expansionIntrinsics(M)) |
| 1424 | return PreservedAnalyses::none(); |
| 1425 | return PreservedAnalyses::all(); |
| 1426 | } |
| 1427 | |
| 1428 | bool DXILIntrinsicExpansionLegacy::runOnModule(Module &M) { |
| 1429 | return expansionIntrinsics(M); |
| 1430 | } |
| 1431 | |
| 1432 | char DXILIntrinsicExpansionLegacy::ID = 0; |
| 1433 | |
| 1434 | INITIALIZE_PASS_BEGIN(DXILIntrinsicExpansionLegacy, DEBUG_TYPE, |
| 1435 | "DXIL Intrinsic Expansion" , false, false) |
| 1436 | INITIALIZE_PASS_END(DXILIntrinsicExpansionLegacy, DEBUG_TYPE, |
| 1437 | "DXIL Intrinsic Expansion" , false, false) |
| 1438 | |
| 1439 | ModulePass *llvm::createDXILIntrinsicExpansionLegacyPass() { |
| 1440 | return new DXILIntrinsicExpansionLegacy(); |
| 1441 | } |
| 1442 | |