| 1 | //===- DXILLegalizePass.cpp - Legalizes llvm IR for DXIL ------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===---------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "DXILLegalizePass.h" |
| 10 | #include "DirectX.h" |
| 11 | #include "llvm/ADT/APInt.h" |
| 12 | #include "llvm/IR/Constants.h" |
| 13 | #include "llvm/IR/Function.h" |
| 14 | #include "llvm/IR/IRBuilder.h" |
| 15 | #include "llvm/IR/InstIterator.h" |
| 16 | #include "llvm/IR/Instruction.h" |
| 17 | #include "llvm/IR/Instructions.h" |
| 18 | #include "llvm/IR/Module.h" |
| 19 | #include "llvm/Pass.h" |
| 20 | #include "llvm/Support/ErrorHandling.h" |
| 21 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
| 22 | #include "llvm/Transforms/Utils/Local.h" |
| 23 | #include <functional> |
| 24 | |
| 25 | #define DEBUG_TYPE "dxil-legalize" |
| 26 | |
| 27 | using namespace llvm; |
| 28 | |
| 29 | static bool legalizeFreeze(Instruction &I, |
| 30 | SmallVectorImpl<Instruction *> &ToRemove, |
| 31 | DenseMap<Value *, Value *>) { |
| 32 | auto *FI = dyn_cast<FreezeInst>(Val: &I); |
| 33 | if (!FI) |
| 34 | return false; |
| 35 | |
| 36 | FI->replaceAllUsesWith(V: FI->getOperand(i_nocapture: 0)); |
| 37 | ToRemove.push_back(Elt: FI); |
| 38 | return true; |
| 39 | } |
| 40 | |
| 41 | static bool fixI8UseChain(Instruction &I, |
| 42 | SmallVectorImpl<Instruction *> &ToRemove, |
| 43 | DenseMap<Value *, Value *> &ReplacedValues) { |
| 44 | |
| 45 | auto ProcessOperands = [&](SmallVector<Value *> &NewOperands) { |
| 46 | Type *InstrType = IntegerType::get(C&: I.getContext(), NumBits: 32); |
| 47 | |
| 48 | for (unsigned OpIdx = 0; OpIdx < I.getNumOperands(); ++OpIdx) { |
| 49 | Value *Op = I.getOperand(i: OpIdx); |
| 50 | if (ReplacedValues.count(Val: Op) && |
| 51 | ReplacedValues[Op]->getType()->isIntegerTy()) |
| 52 | InstrType = ReplacedValues[Op]->getType(); |
| 53 | } |
| 54 | |
| 55 | for (unsigned OpIdx = 0; OpIdx < I.getNumOperands(); ++OpIdx) { |
| 56 | Value *Op = I.getOperand(i: OpIdx); |
| 57 | if (ReplacedValues.count(Val: Op)) |
| 58 | NewOperands.push_back(Elt: ReplacedValues[Op]); |
| 59 | else if (auto *Imm = dyn_cast<ConstantInt>(Val: Op)) { |
| 60 | APInt Value = Imm->getValue(); |
| 61 | unsigned NewBitWidth = InstrType->getIntegerBitWidth(); |
| 62 | // Note: options here are sext or sextOrTrunc. |
| 63 | // Since i8 isn't supported, we assume new values |
| 64 | // will always have a higher bitness. |
| 65 | assert(NewBitWidth > Value.getBitWidth() && |
| 66 | "Replacement's BitWidth should be larger than Current." ); |
| 67 | APInt NewValue = Value.sext(width: NewBitWidth); |
| 68 | NewOperands.push_back(Elt: ConstantInt::get(Ty: InstrType, V: NewValue)); |
| 69 | } else { |
| 70 | assert(!Op->getType()->isIntegerTy(8)); |
| 71 | NewOperands.push_back(Elt: Op); |
| 72 | } |
| 73 | } |
| 74 | }; |
| 75 | IRBuilder<> Builder(&I); |
| 76 | if (auto *Trunc = dyn_cast<TruncInst>(Val: &I)) { |
| 77 | if (Trunc->getDestTy()->isIntegerTy(BitWidth: 8)) { |
| 78 | ReplacedValues[Trunc] = Trunc->getOperand(i_nocapture: 0); |
| 79 | ToRemove.push_back(Elt: Trunc); |
| 80 | return true; |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | if (auto *Store = dyn_cast<StoreInst>(Val: &I)) { |
| 85 | if (!Store->getValueOperand()->getType()->isIntegerTy(BitWidth: 8)) |
| 86 | return false; |
| 87 | SmallVector<Value *> NewOperands; |
| 88 | ProcessOperands(NewOperands); |
| 89 | Value *NewStore = Builder.CreateStore(Val: NewOperands[0], Ptr: NewOperands[1]); |
| 90 | ReplacedValues[Store] = NewStore; |
| 91 | ToRemove.push_back(Elt: Store); |
| 92 | return true; |
| 93 | } |
| 94 | |
| 95 | if (auto *Load = dyn_cast<LoadInst>(Val: &I); |
| 96 | Load && I.getType()->isIntegerTy(BitWidth: 8)) { |
| 97 | SmallVector<Value *> NewOperands; |
| 98 | ProcessOperands(NewOperands); |
| 99 | Type *ElementType = NewOperands[0]->getType(); |
| 100 | if (auto *AI = dyn_cast<AllocaInst>(Val: NewOperands[0])) |
| 101 | ElementType = AI->getAllocatedType(); |
| 102 | if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: NewOperands[0])) { |
| 103 | ElementType = GEP->getSourceElementType(); |
| 104 | } |
| 105 | if (ElementType->isArrayTy()) |
| 106 | ElementType = ElementType->getArrayElementType(); |
| 107 | LoadInst *NewLoad = Builder.CreateLoad(Ty: ElementType, Ptr: NewOperands[0]); |
| 108 | ReplacedValues[Load] = NewLoad; |
| 109 | ToRemove.push_back(Elt: Load); |
| 110 | return true; |
| 111 | } |
| 112 | |
| 113 | if (auto *Load = dyn_cast<LoadInst>(Val: &I); |
| 114 | Load && isa<ConstantExpr>(Val: Load->getPointerOperand())) { |
| 115 | auto *CE = dyn_cast<ConstantExpr>(Val: Load->getPointerOperand()); |
| 116 | if (!(CE->getOpcode() == Instruction::GetElementPtr)) |
| 117 | return false; |
| 118 | auto *GEP = dyn_cast<GEPOperator>(Val: CE); |
| 119 | if (!GEP->getSourceElementType()->isIntegerTy(BitWidth: 8)) |
| 120 | return false; |
| 121 | |
| 122 | Type *ElementType = Load->getType(); |
| 123 | ConstantInt *Offset = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1)); |
| 124 | uint32_t ByteOffset = Offset->getZExtValue(); |
| 125 | uint32_t ElemSize = Load->getDataLayout().getTypeAllocSize(Ty: ElementType); |
| 126 | uint32_t Index = ByteOffset / ElemSize; |
| 127 | |
| 128 | Value *PtrOperand = GEP->getPointerOperand(); |
| 129 | Type *GEPType = GEP->getPointerOperandType(); |
| 130 | |
| 131 | if (auto *GV = dyn_cast<GlobalVariable>(Val: PtrOperand)) |
| 132 | GEPType = GV->getValueType(); |
| 133 | if (auto *AI = dyn_cast<AllocaInst>(Val: PtrOperand)) |
| 134 | GEPType = AI->getAllocatedType(); |
| 135 | |
| 136 | if (auto *ArrTy = dyn_cast<ArrayType>(Val: GEPType)) |
| 137 | GEPType = ArrTy; |
| 138 | else |
| 139 | GEPType = ArrayType::get(ElementType, NumElements: 1); // its a scalar |
| 140 | |
| 141 | Value *NewGEP = GetElementPtrInst::Create( |
| 142 | PointeeType: GEPType, Ptr: PtrOperand, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: Index)}, |
| 143 | NW: GEP->getNoWrapFlags(), NameStr: GEP->getName(), InsertBefore: I.getIterator()); |
| 144 | |
| 145 | LoadInst *NewLoad = Builder.CreateLoad(Ty: ElementType, Ptr: NewGEP); |
| 146 | ReplacedValues[Load] = NewLoad; |
| 147 | Load->replaceAllUsesWith(V: NewLoad); |
| 148 | ToRemove.push_back(Elt: Load); |
| 149 | return true; |
| 150 | } |
| 151 | |
| 152 | if (auto *BO = dyn_cast<BinaryOperator>(Val: &I)) { |
| 153 | if (!I.getType()->isIntegerTy(BitWidth: 8)) |
| 154 | return false; |
| 155 | SmallVector<Value *> NewOperands; |
| 156 | ProcessOperands(NewOperands); |
| 157 | Value *NewInst = |
| 158 | Builder.CreateBinOp(Opc: BO->getOpcode(), LHS: NewOperands[0], RHS: NewOperands[1]); |
| 159 | if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: &I)) { |
| 160 | auto *NewBO = dyn_cast<BinaryOperator>(Val: NewInst); |
| 161 | if (NewBO && OBO->hasNoSignedWrap()) |
| 162 | NewBO->setHasNoSignedWrap(); |
| 163 | if (NewBO && OBO->hasNoUnsignedWrap()) |
| 164 | NewBO->setHasNoUnsignedWrap(); |
| 165 | } |
| 166 | ReplacedValues[BO] = NewInst; |
| 167 | ToRemove.push_back(Elt: BO); |
| 168 | return true; |
| 169 | } |
| 170 | |
| 171 | if (auto *Sel = dyn_cast<SelectInst>(Val: &I)) { |
| 172 | if (!I.getType()->isIntegerTy(BitWidth: 8)) |
| 173 | return false; |
| 174 | SmallVector<Value *> NewOperands; |
| 175 | ProcessOperands(NewOperands); |
| 176 | Value *NewInst = Builder.CreateSelect(C: Sel->getCondition(), True: NewOperands[1], |
| 177 | False: NewOperands[2]); |
| 178 | ReplacedValues[Sel] = NewInst; |
| 179 | ToRemove.push_back(Elt: Sel); |
| 180 | return true; |
| 181 | } |
| 182 | |
| 183 | if (auto *Cmp = dyn_cast<CmpInst>(Val: &I)) { |
| 184 | if (!Cmp->getOperand(i_nocapture: 0)->getType()->isIntegerTy(BitWidth: 8)) |
| 185 | return false; |
| 186 | SmallVector<Value *> NewOperands; |
| 187 | ProcessOperands(NewOperands); |
| 188 | Value *NewInst = |
| 189 | Builder.CreateCmp(Pred: Cmp->getPredicate(), LHS: NewOperands[0], RHS: NewOperands[1]); |
| 190 | Cmp->replaceAllUsesWith(V: NewInst); |
| 191 | ReplacedValues[Cmp] = NewInst; |
| 192 | ToRemove.push_back(Elt: Cmp); |
| 193 | return true; |
| 194 | } |
| 195 | |
| 196 | if (auto *Cast = dyn_cast<CastInst>(Val: &I)) { |
| 197 | if (!Cast->getSrcTy()->isIntegerTy(BitWidth: 8)) |
| 198 | return false; |
| 199 | |
| 200 | ToRemove.push_back(Elt: Cast); |
| 201 | auto *Replacement = ReplacedValues[Cast->getOperand(i_nocapture: 0)]; |
| 202 | if (Cast->getType() == Replacement->getType()) { |
| 203 | Cast->replaceAllUsesWith(V: Replacement); |
| 204 | return true; |
| 205 | } |
| 206 | |
| 207 | Value *AdjustedCast = nullptr; |
| 208 | if (Cast->getOpcode() == Instruction::ZExt) |
| 209 | AdjustedCast = Builder.CreateZExtOrTrunc(V: Replacement, DestTy: Cast->getType()); |
| 210 | if (Cast->getOpcode() == Instruction::SExt) |
| 211 | AdjustedCast = Builder.CreateSExtOrTrunc(V: Replacement, DestTy: Cast->getType()); |
| 212 | |
| 213 | if (AdjustedCast) |
| 214 | Cast->replaceAllUsesWith(V: AdjustedCast); |
| 215 | } |
| 216 | if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I)) { |
| 217 | if (!GEP->getType()->isPointerTy() || |
| 218 | !GEP->getSourceElementType()->isIntegerTy(BitWidth: 8)) |
| 219 | return false; |
| 220 | |
| 221 | Value *BasePtr = GEP->getPointerOperand(); |
| 222 | if (ReplacedValues.count(Val: BasePtr)) |
| 223 | BasePtr = ReplacedValues[BasePtr]; |
| 224 | |
| 225 | Type *ElementType = BasePtr->getType(); |
| 226 | |
| 227 | if (auto *AI = dyn_cast<AllocaInst>(Val: BasePtr)) |
| 228 | ElementType = AI->getAllocatedType(); |
| 229 | if (auto *GV = dyn_cast<GlobalVariable>(Val: BasePtr)) |
| 230 | ElementType = GV->getValueType(); |
| 231 | |
| 232 | Type *GEPType = ElementType; |
| 233 | if (auto *ArrTy = dyn_cast<ArrayType>(Val: ElementType)) |
| 234 | ElementType = ArrTy->getArrayElementType(); |
| 235 | else |
| 236 | GEPType = ArrayType::get(ElementType, NumElements: 1); // its a scalar |
| 237 | |
| 238 | ConstantInt *Offset = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1)); |
| 239 | // Note: i8 to i32 offset conversion without emitting IR requires constant |
| 240 | // ints. Since offset conversion is common, we can safely assume Offset is |
| 241 | // always a ConstantInt, so no need to have a conditional bail out on |
| 242 | // nullptr, instead assert this is the case. |
| 243 | assert(Offset && "Offset is expected to be a ConstantInt" ); |
| 244 | uint32_t ByteOffset = Offset->getZExtValue(); |
| 245 | uint32_t ElemSize = GEP->getDataLayout().getTypeAllocSize(Ty: ElementType); |
| 246 | assert(ElemSize > 0 && "ElementSize must be set" ); |
| 247 | uint32_t Index = ByteOffset / ElemSize; |
| 248 | Value *NewGEP = GetElementPtrInst::Create( |
| 249 | PointeeType: GEPType, Ptr: BasePtr, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: Index)}, |
| 250 | NW: GEP->getNoWrapFlags(), NameStr: GEP->getName(), InsertBefore: GEP->getIterator()); |
| 251 | ReplacedValues[GEP] = NewGEP; |
| 252 | GEP->replaceAllUsesWith(V: NewGEP); |
| 253 | ToRemove.push_back(Elt: GEP); |
| 254 | return true; |
| 255 | } |
| 256 | return false; |
| 257 | } |
| 258 | |
| 259 | static bool upcastI8AllocasAndUses(Instruction &I, |
| 260 | SmallVectorImpl<Instruction *> &ToRemove, |
| 261 | DenseMap<Value *, Value *> &ReplacedValues) { |
| 262 | auto *AI = dyn_cast<AllocaInst>(Val: &I); |
| 263 | if (!AI || !AI->getAllocatedType()->isIntegerTy(BitWidth: 8)) |
| 264 | return false; |
| 265 | |
| 266 | Type *SmallestType = nullptr; |
| 267 | |
| 268 | auto ProcessLoad = [&](LoadInst *Load) { |
| 269 | for (User *LU : Load->users()) { |
| 270 | CastInst *Cast = dyn_cast<CastInst>(Val: LU); |
| 271 | if (!Cast) |
| 272 | continue; |
| 273 | Type *Ty = Cast->getType(); |
| 274 | |
| 275 | if (!SmallestType || |
| 276 | Ty->getPrimitiveSizeInBits() < SmallestType->getPrimitiveSizeInBits()) |
| 277 | SmallestType = Ty; |
| 278 | } |
| 279 | }; |
| 280 | |
| 281 | for (User *U : AI->users()) { |
| 282 | if (auto *Load = dyn_cast<LoadInst>(Val: U)) |
| 283 | ProcessLoad(Load); |
| 284 | else if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: U)) { |
| 285 | for (User *GU : GEP->users()) { |
| 286 | if (auto *Load = dyn_cast<LoadInst>(Val: GU)) |
| 287 | ProcessLoad(Load); |
| 288 | } |
| 289 | } |
| 290 | } |
| 291 | |
| 292 | if (!SmallestType) |
| 293 | return false; // no valid casts found |
| 294 | |
| 295 | // Replace alloca |
| 296 | IRBuilder<> Builder(AI); |
| 297 | auto *NewAlloca = Builder.CreateAlloca(Ty: SmallestType); |
| 298 | ReplacedValues[AI] = NewAlloca; |
| 299 | ToRemove.push_back(Elt: AI); |
| 300 | return true; |
| 301 | } |
| 302 | |
| 303 | static bool |
| 304 | (Instruction &I, |
| 305 | SmallVectorImpl<Instruction *> &ToRemove, |
| 306 | DenseMap<Value *, Value *> &) { |
| 307 | |
| 308 | if (auto * = dyn_cast<ExtractElementInst>(Val: &I)) { |
| 309 | Value *Idx = Extract->getIndexOperand(); |
| 310 | auto *CI = dyn_cast<ConstantInt>(Val: Idx); |
| 311 | if (CI && CI->getBitWidth() == 64) { |
| 312 | IRBuilder<> Builder(Extract); |
| 313 | int64_t IndexValue = CI->getSExtValue(); |
| 314 | auto *Idx32 = |
| 315 | ConstantInt::get(Ty: Type::getInt32Ty(C&: I.getContext()), V: IndexValue); |
| 316 | Value * = Builder.CreateExtractElement( |
| 317 | Vec: Extract->getVectorOperand(), Idx: Idx32, Name: Extract->getName()); |
| 318 | |
| 319 | Extract->replaceAllUsesWith(V: NewExtract); |
| 320 | ToRemove.push_back(Elt: Extract); |
| 321 | return true; |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | if (auto *Insert = dyn_cast<InsertElementInst>(Val: &I)) { |
| 326 | Value *Idx = Insert->getOperand(i_nocapture: 2); |
| 327 | auto *CI = dyn_cast<ConstantInt>(Val: Idx); |
| 328 | if (CI && CI->getBitWidth() == 64) { |
| 329 | int64_t IndexValue = CI->getSExtValue(); |
| 330 | auto *Idx32 = |
| 331 | ConstantInt::get(Ty: Type::getInt32Ty(C&: I.getContext()), V: IndexValue); |
| 332 | IRBuilder<> Builder(Insert); |
| 333 | Value *Insert32Index = Builder.CreateInsertElement( |
| 334 | Vec: Insert->getOperand(i_nocapture: 0), NewElt: Insert->getOperand(i_nocapture: 1), Idx: Idx32, |
| 335 | Name: Insert->getName()); |
| 336 | |
| 337 | Insert->replaceAllUsesWith(V: Insert32Index); |
| 338 | ToRemove.push_back(Elt: Insert); |
| 339 | return true; |
| 340 | } |
| 341 | } |
| 342 | return false; |
| 343 | } |
| 344 | |
| 345 | static bool updateFnegToFsub(Instruction &I, |
| 346 | SmallVectorImpl<Instruction *> &ToRemove, |
| 347 | DenseMap<Value *, Value *> &) { |
| 348 | const Intrinsic::ID ID = I.getOpcode(); |
| 349 | if (ID != Instruction::FNeg) |
| 350 | return false; |
| 351 | |
| 352 | IRBuilder<> Builder(&I); |
| 353 | Value *In = I.getOperand(i: 0); |
| 354 | Value *Zero = ConstantFP::get(Ty: In->getType(), V: -0.0); |
| 355 | I.replaceAllUsesWith(V: Builder.CreateFSub(L: Zero, R: In)); |
| 356 | ToRemove.push_back(Elt: &I); |
| 357 | return true; |
| 358 | } |
| 359 | |
| 360 | // DXIL has no floating-point atomic operation. A float exchange only moves the |
| 361 | // bit pattern, so exchange an integer of the same width instead. Opaque |
| 362 | // pointers keep the pointer operand type-agnostic, so only the value and the |
| 363 | // result need a cast. This matches what DXC emits for groupshared memory. |
| 364 | static bool |
| 365 | legalizeFloatAtomicExchange(Instruction &I, |
| 366 | SmallVectorImpl<Instruction *> &ToRemove, |
| 367 | DenseMap<Value *, Value *> &) { |
| 368 | auto *AI = dyn_cast<AtomicRMWInst>(Val: &I); |
| 369 | if (!AI || AI->getOperation() != AtomicRMWInst::Xchg) |
| 370 | return false; |
| 371 | |
| 372 | Type *ValTy = AI->getValOperand()->getType(); |
| 373 | if (!ValTy->isFloatingPointTy()) |
| 374 | return false; |
| 375 | |
| 376 | // DXIL has 32-bit and 64-bit atomics only. A float of any other width has no |
| 377 | // integer exchange to lower to. |
| 378 | unsigned Width = ValTy->getPrimitiveSizeInBits(); |
| 379 | if (Width != 32 && Width != 64) |
| 380 | reportFatalUsageError(reason: "DXIL atomic exchange requires a 32-bit or 64-bit " |
| 381 | "floating-point value" ); |
| 382 | |
| 383 | IRBuilder<> Builder(AI); |
| 384 | Type *IntTy = Builder.getIntNTy(N: Width); |
| 385 | Value *Val = Builder.CreateBitCast(V: AI->getValOperand(), DestTy: IntTy); |
| 386 | AtomicRMWInst *NewAI = Builder.CreateAtomicRMW( |
| 387 | Op: AtomicRMWInst::Xchg, Ptr: AI->getPointerOperand(), Val, Align: AI->getAlign(), |
| 388 | Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID()); |
| 389 | NewAI->copyMetadata(SrcInst: *AI); |
| 390 | AI->replaceAllUsesWith(V: Builder.CreateBitCast(V: NewAI, DestTy: ValTy)); |
| 391 | ToRemove.push_back(Elt: AI); |
| 392 | return true; |
| 393 | } |
| 394 | |
| 395 | static bool |
| 396 | resolveUnreachableSwitchDefault(Instruction &I, |
| 397 | SmallVectorImpl<Instruction *> &ToRemove, |
| 398 | DenseMap<Value *, Value *> &) { |
| 399 | auto *SI = dyn_cast<SwitchInst>(Val: &I); |
| 400 | if (!SI || SI->getNumCases() == 0) |
| 401 | return false; |
| 402 | |
| 403 | BasicBlock *DefaultBB = SI->getDefaultDest(); |
| 404 | |
| 405 | // Check if the default destination ends with an unreachable instruction. |
| 406 | if (DefaultBB->size() == 0 || |
| 407 | !isa<UnreachableInst>(Val: DefaultBB->getTerminator())) |
| 408 | return false; |
| 409 | |
| 410 | // Try to find a common successor of all case destinations. If all case |
| 411 | // blocks unconditionally branch to the same block, that is the common |
| 412 | // successor. This is just a best effort, and is done as the original form of |
| 413 | // the switch statement was likely in this form before being transformed to |
| 414 | // an unreachable branch. |
| 415 | BasicBlock *CommonSuccessor = nullptr; |
| 416 | for (auto &Case : SI->cases()) { |
| 417 | BasicBlock *CaseBB = Case.getCaseSuccessor(); |
| 418 | auto *BI = dyn_cast<UncondBrInst>(Val: CaseBB->getTerminator()); |
| 419 | if (!BI) { |
| 420 | CommonSuccessor = nullptr; |
| 421 | break; |
| 422 | } |
| 423 | BasicBlock *Succ = BI->getSuccessor(i: 0); |
| 424 | if (!CommonSuccessor) |
| 425 | CommonSuccessor = Succ; |
| 426 | else if (CommonSuccessor != Succ) { |
| 427 | CommonSuccessor = nullptr; |
| 428 | break; |
| 429 | } |
| 430 | } |
| 431 | |
| 432 | BasicBlock *NewDefault = |
| 433 | CommonSuccessor ? CommonSuccessor : SI->case_begin()->getCaseSuccessor(); |
| 434 | |
| 435 | BasicBlock *SwitchBB = SI->getParent(); |
| 436 | SI->setDefaultDest(NewDefault); |
| 437 | |
| 438 | // Ensure all phi nodes are legal by adding an incoming poison value from the |
| 439 | // unreachable branch. |
| 440 | for (PHINode &Phi : NewDefault->phis()) |
| 441 | Phi.addIncoming(V: PoisonValue::get(T: Phi.getType()), BB: SwitchBB); |
| 442 | |
| 443 | return true; |
| 444 | } |
| 445 | |
| 446 | static bool |
| 447 | legalizeScalarLoadStoreOnArrays(Instruction &I, |
| 448 | SmallVectorImpl<Instruction *> &ToRemove, |
| 449 | DenseMap<Value *, Value *> &) { |
| 450 | |
| 451 | Value *PtrOp; |
| 452 | unsigned PtrOpIndex; |
| 453 | [[maybe_unused]] Type *LoadStoreTy; |
| 454 | if (auto *LI = dyn_cast<LoadInst>(Val: &I)) { |
| 455 | PtrOp = LI->getPointerOperand(); |
| 456 | PtrOpIndex = LI->getPointerOperandIndex(); |
| 457 | LoadStoreTy = LI->getType(); |
| 458 | } else if (auto *SI = dyn_cast<StoreInst>(Val: &I)) { |
| 459 | PtrOp = SI->getPointerOperand(); |
| 460 | PtrOpIndex = SI->getPointerOperandIndex(); |
| 461 | LoadStoreTy = SI->getValueOperand()->getType(); |
| 462 | } else |
| 463 | return false; |
| 464 | |
| 465 | // If the load/store is not of a single-value type (i.e., scalar or vector) |
| 466 | // then we do not modify it. It shouldn't be a vector either because the |
| 467 | // dxil-data-scalarization pass is expected to run before this, but it's not |
| 468 | // incorrect to apply this transformation to vector load/stores. |
| 469 | if (!LoadStoreTy->isSingleValueType()) |
| 470 | return false; |
| 471 | |
| 472 | Type *ArrayTy; |
| 473 | if (auto *GlobalVarPtrOp = dyn_cast<GlobalVariable>(Val: PtrOp)) |
| 474 | ArrayTy = GlobalVarPtrOp->getValueType(); |
| 475 | else if (auto *AllocaPtrOp = dyn_cast<AllocaInst>(Val: PtrOp)) |
| 476 | ArrayTy = AllocaPtrOp->getAllocatedType(); |
| 477 | else |
| 478 | return false; |
| 479 | |
| 480 | if (!isa<ArrayType>(Val: ArrayTy)) |
| 481 | return false; |
| 482 | |
| 483 | assert(ArrayTy->getArrayElementType() == LoadStoreTy && |
| 484 | "Expected array element type to be the same as to the scalar load or " |
| 485 | "store type" ); |
| 486 | |
| 487 | Value *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: I.getContext()), V: 0); |
| 488 | Value *GEP = GetElementPtrInst::Create( |
| 489 | PointeeType: ArrayTy, Ptr: PtrOp, IdxList: {Zero, Zero}, NW: GEPNoWrapFlags::all(), NameStr: "" , InsertBefore: I.getIterator()); |
| 490 | I.setOperand(i: PtrOpIndex, Val: GEP); |
| 491 | return true; |
| 492 | } |
| 493 | |
| 494 | namespace { |
| 495 | class DXILLegalizationPipeline { |
| 496 | |
| 497 | public: |
| 498 | DXILLegalizationPipeline() { initializeLegalizationPipeline(); } |
| 499 | |
| 500 | bool runLegalizationPipeline(Function &F) { |
| 501 | bool MadeChange = false; |
| 502 | SmallVector<Instruction *> ToRemove; |
| 503 | DenseMap<Value *, Value *> ReplacedValues; |
| 504 | for (int Stage = 0; Stage < NumStages; ++Stage) { |
| 505 | ToRemove.clear(); |
| 506 | ReplacedValues.clear(); |
| 507 | for (auto &I : instructions(F)) { |
| 508 | for (auto &LegalizationFn : LegalizationPipeline[Stage]) |
| 509 | MadeChange |= LegalizationFn(I, ToRemove, ReplacedValues); |
| 510 | } |
| 511 | |
| 512 | for (auto *Inst : reverse(C&: ToRemove)) |
| 513 | Inst->eraseFromParent(); |
| 514 | } |
| 515 | |
| 516 | if (MadeChange) |
| 517 | MadeChange |= removeUnreachableBlocks(F); |
| 518 | return MadeChange; |
| 519 | } |
| 520 | |
| 521 | private: |
| 522 | enum LegalizationStage { Stage1 = 0, Stage2 = 1, NumStages }; |
| 523 | |
| 524 | using LegalizationFnTy = |
| 525 | std::function<bool(Instruction &, SmallVectorImpl<Instruction *> &, |
| 526 | DenseMap<Value *, Value *> &)>; |
| 527 | |
| 528 | SmallVector<LegalizationFnTy> LegalizationPipeline[NumStages]; |
| 529 | |
| 530 | void initializeLegalizationPipeline() { |
| 531 | LegalizationPipeline[Stage1].push_back(Elt: upcastI8AllocasAndUses); |
| 532 | LegalizationPipeline[Stage1].push_back(Elt: fixI8UseChain); |
| 533 | LegalizationPipeline[Stage1].push_back(Elt: legalizeFreeze); |
| 534 | LegalizationPipeline[Stage1].push_back(Elt: updateFnegToFsub); |
| 535 | LegalizationPipeline[Stage1].push_back(Elt: legalizeFloatAtomicExchange); |
| 536 | LegalizationPipeline[Stage1].push_back( |
| 537 | Elt: downcastI64toI32InsertExtractElements); |
| 538 | LegalizationPipeline[Stage2].push_back(Elt: legalizeScalarLoadStoreOnArrays); |
| 539 | LegalizationPipeline[Stage2].push_back(Elt: resolveUnreachableSwitchDefault); |
| 540 | } |
| 541 | }; |
| 542 | |
| 543 | class DXILLegalizeLegacy : public FunctionPass { |
| 544 | |
| 545 | public: |
| 546 | bool runOnFunction(Function &F) override; |
| 547 | DXILLegalizeLegacy() : FunctionPass(ID) {} |
| 548 | |
| 549 | static char ID; // Pass identification. |
| 550 | }; |
| 551 | } // namespace |
| 552 | |
| 553 | PreservedAnalyses DXILLegalizePass::run(Function &F, |
| 554 | FunctionAnalysisManager &FAM) { |
| 555 | DXILLegalizationPipeline DXLegalize; |
| 556 | bool MadeChanges = DXLegalize.runLegalizationPipeline(F); |
| 557 | if (!MadeChanges) |
| 558 | return PreservedAnalyses::all(); |
| 559 | PreservedAnalyses PA; |
| 560 | return PA; |
| 561 | } |
| 562 | |
| 563 | bool DXILLegalizeLegacy::runOnFunction(Function &F) { |
| 564 | DXILLegalizationPipeline DXLegalize; |
| 565 | return DXLegalize.runLegalizationPipeline(F); |
| 566 | } |
| 567 | |
| 568 | char DXILLegalizeLegacy::ID = 0; |
| 569 | |
| 570 | INITIALIZE_PASS_BEGIN(DXILLegalizeLegacy, DEBUG_TYPE, "DXIL Legalizer" , false, |
| 571 | false) |
| 572 | INITIALIZE_PASS_END(DXILLegalizeLegacy, DEBUG_TYPE, "DXIL Legalizer" , false, |
| 573 | false) |
| 574 | |
| 575 | FunctionPass *llvm::createDXILLegalizeLegacyPass() { |
| 576 | return new DXILLegalizeLegacy(); |
| 577 | } |
| 578 | |