1//===- DXILLegalizePass.cpp - Legalizes llvm IR for DXIL ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===---------------------------------------------------------------------===//
8
9#include "DirectX.h"
10#include "llvm/ADT/APInt.h"
11#include "llvm/ADT/PostOrderIterator.h"
12#include "llvm/Analysis/ValueTracking.h"
13#include "llvm/IR/Constants.h"
14#include "llvm/IR/Function.h"
15#include "llvm/IR/IRBuilder.h"
16#include "llvm/IR/Instruction.h"
17#include "llvm/IR/Instructions.h"
18#include "llvm/IR/Module.h"
19#include "llvm/IR/ProfDataUtils.h"
20#include "llvm/Pass.h"
21#include "llvm/Support/ErrorHandling.h"
22#include "llvm/Transforms/Utils/BasicBlockUtils.h"
23#include "llvm/Transforms/Utils/Local.h"
24#include <functional>
25
26#define DEBUG_TYPE "dxil-legalize"
27
28using namespace llvm;
29
30static IntegerType *getLegalIntegerType(Type *Ty) {
31 auto *IT = dyn_cast<IntegerType>(Val: Ty);
32 if (!IT)
33 return nullptr;
34 unsigned Width = IT->getBitWidth();
35 if (Width == 1 || Width == 16 || Width == 32 || Width == 64)
36 return nullptr;
37 if (Width > 64)
38 reportFatalUsageError(
39 reason: "DXIL does not support integer types wider than 64 bits");
40 return IntegerType::get(C&: Ty->getContext(), NumBits: Width <= 32 ? 32 : 64);
41}
42
43enum class IntegerExtension { None, Zero, Sign };
44
45static bool legalizeInteger(Instruction &I,
46 SmallVectorImpl<Instruction *> &ToRemove,
47 DenseMap<Value *, Value *> &ReplacedValues) {
48 if (ReplacedValues.contains(Val: &I))
49 return false;
50
51 IRBuilder<> B(&I);
52
53 auto GetOperand = [&](Value *V,
54 IntegerExtension Ext = IntegerExtension::None) {
55 auto *Ty = getLegalIntegerType(Ty: V->getType());
56 if (!Ty)
57 return V;
58 Value *R = ReplacedValues.lookup_or(Val: V, Default: dyn_cast<Constant>(Val: V));
59 if (!R)
60 reportFatalUsageError(reason: "DXIL legalization is missing an integer operand "
61 "replacement");
62 R = B.CreateZExtOrTrunc(V: R, DestTy: Ty);
63 unsigned Width = V->getType()->getIntegerBitWidth();
64 unsigned Shift = Ty->getBitWidth() - Width;
65 auto *ContextI = dyn_cast<Instruction>(Val: R);
66 if (Ext == IntegerExtension::Sign &&
67 ComputeNumSignBits(Op: R, DL: I.getDataLayout(), AC: nullptr, CtxI: ContextI) <= Shift)
68 R = B.CreateAShr(LHS: B.CreateShl(LHS: R, RHS: Shift), RHS: Shift);
69 if (Ext == IntegerExtension::Zero &&
70 !MaskedValueIsZero(V: R, Mask: APInt::getBitsSetFrom(numBits: Ty->getBitWidth(), loBit: Width),
71 SQ: SimplifyQuery(I.getDataLayout(), ContextI)))
72 R = B.CreateAnd(LHS: R, RHS: APInt::getLowBitsSet(numBits: Ty->getBitWidth(), loBitsSet: Width));
73 return R;
74 };
75
76 auto Replace = [&](Value *V) {
77 if (I.getType() == V->getType())
78 I.replaceAllUsesWith(V);
79 else
80 ReplacedValues[&I] = V;
81 ToRemove.push_back(Elt: &I);
82 };
83
84 if (auto *BC = dyn_cast<BitCastInst>(Val: &I)) {
85 auto *Dst = getLegalIntegerType(Ty: BC->getDestTy());
86 auto *Src = getLegalIntegerType(Ty: BC->getSrcTy());
87 if (!Dst && !Src)
88 return false;
89 auto *VT =
90 dyn_cast<FixedVectorType>(Val: Dst ? BC->getSrcTy() : BC->getDestTy());
91 if (!VT || !VT->getElementType()->isIntegerTy() ||
92 getLegalIntegerType(Ty: VT->getElementType()))
93 reportFatalUsageError(reason: "DXIL legalization does not support this integer "
94 "bitcast");
95 unsigned Width = VT->getScalarSizeInBits();
96 Value *Op = GetOperand(BC->getOperand(i_nocapture: 0));
97 if (Dst) {
98 Value *Result = B.getIntN(N: Dst->getBitWidth(), C: 0);
99 for (unsigned Idx = 0; Idx < VT->getNumElements(); ++Idx) {
100 Value *Elt =
101 B.CreateZExt(V: B.CreateExtractElement(Vec: Op, Idx: B.getInt32(C: Idx)), DestTy: Dst);
102 Result = B.CreateOr(LHS: Result, RHS: Idx ? B.CreateShl(LHS: Elt, RHS: Idx * Width) : Elt);
103 }
104 Replace(Result);
105 } else {
106 Value *Result = PoisonValue::get(T: VT);
107 for (unsigned Idx = 0; Idx < VT->getNumElements(); ++Idx) {
108 Value *Elt = B.CreateTrunc(V: Idx ? B.CreateLShr(LHS: Op, RHS: Idx * Width) : Op,
109 DestTy: VT->getElementType());
110 Result = B.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: B.getInt32(C: Idx));
111 }
112 Replace(Result);
113 }
114 return true;
115 }
116 if (auto *BO = dyn_cast<BinaryOperator>(Val: &I)) {
117 if (!getLegalIntegerType(Ty: I.getType()))
118 return false;
119 IntegerExtension Ext = IntegerExtension::None;
120 switch (BO->getOpcode()) {
121 case Instruction::SDiv:
122 case Instruction::SRem:
123 case Instruction::AShr:
124 Ext = IntegerExtension::Sign;
125 break;
126 case Instruction::UDiv:
127 case Instruction::URem:
128 case Instruction::LShr:
129 Ext = IntegerExtension::Zero;
130 break;
131 default:
132 break;
133 }
134 Value *LHS = GetOperand(BO->getOperand(i_nocapture: 0), Ext);
135 Value *RHS = GetOperand(BO->getOperand(i_nocapture: 1),
136 BO->isShift() ? IntegerExtension::Zero : Ext);
137 // Narrow no-wrap/exact flags do not describe the carrier operation.
138 Replace(B.CreateBinOp(Opc: BO->getOpcode(), LHS, RHS));
139 return true;
140 }
141 if (auto *SI = dyn_cast<SelectInst>(Val: &I)) {
142 if (!getLegalIntegerType(Ty: I.getType()))
143 return false;
144 Value *True = GetOperand(SI->getTrueValue());
145 Value *False = GetOperand(SI->getFalseValue());
146 Value *V =
147 B.CreateSelect(C: SI->getCondition(), True, False, Name: SI->getName(), MDFrom: SI);
148 if (auto *NewSI = dyn_cast<SelectInst>(Val: V);
149 NewSI && !NewSI->getMetadata(KindID: LLVMContext::MD_prof))
150 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *NewSI, DEBUG_TYPE);
151 Replace(V);
152 return true;
153 }
154 if (auto *CI = dyn_cast<ICmpInst>(Val: &I)) {
155 if (!getLegalIntegerType(Ty: CI->getOperand(i_nocapture: 0)->getType()))
156 return false;
157 auto Ext = CI->isSigned() ? IntegerExtension::Sign : IntegerExtension::Zero;
158 Value *LHS = GetOperand(CI->getOperand(i_nocapture: 0), Ext);
159 Value *RHS = GetOperand(CI->getOperand(i_nocapture: 1), Ext);
160 Replace(B.CreateICmp(P: CI->getPredicate(), LHS, RHS));
161 return true;
162 }
163 if (auto *CI = dyn_cast<CastInst>(Val: &I)) {
164 auto *Dst = getLegalIntegerType(Ty: CI->getDestTy());
165 if (!Dst && !getLegalIntegerType(Ty: CI->getSrcTy()))
166 return false;
167 auto Ext = IntegerExtension::Zero;
168 if (CI->getOpcode() == Instruction::Trunc)
169 Ext = IntegerExtension::None;
170 else if (CI->getOpcode() == Instruction::SExt ||
171 CI->getOpcode() == Instruction::SIToFP)
172 Ext = IntegerExtension::Sign;
173 Value *V = GetOperand(CI->getOperand(i_nocapture: 0), Ext);
174 Type *Ty = Dst ? Dst : CI->getDestTy();
175 Replace(CI->isIntegerCast()
176 ? B.CreateIntCast(V, DestTy: Ty, isSigned: Ext == IntegerExtension::Sign)
177 : B.CreateCast(Op: CI->getOpcode(), V, DestTy: Ty));
178 return true;
179 }
180 if (getLegalIntegerType(Ty: I.getType()))
181 reportFatalUsageError(reason: Twine("DXIL legalization does not support integer "
182 "result for instruction '") +
183 I.getOpcodeName() + "'");
184 for (Use &Op : I.operands()) {
185 // Fixed intrinsic immediates are ABI operands, not integer SSA.
186 if (auto *CB = dyn_cast<CallBase>(Val: &I); CB && CB->isArgOperand(U: &Op)) {
187 unsigned Arg = CB->getArgOperandNo(U: &Op);
188 if (CB->paramHasAttr(ArgNo: Arg, Kind: Attribute::ImmArg))
189 continue;
190 }
191 if (getLegalIntegerType(Ty: Op->getType()))
192 reportFatalUsageError(reason: Twine("DXIL legalization does not support integer "
193 "operand for instruction '") +
194 I.getOpcodeName() + "'");
195 }
196 return false;
197}
198
199static bool legalizeFreeze(Instruction &I,
200 SmallVectorImpl<Instruction *> &ToRemove,
201 DenseMap<Value *, Value *>) {
202 auto *FI = dyn_cast<FreezeInst>(Val: &I);
203 if (!FI)
204 return false;
205
206 FI->replaceAllUsesWith(V: FI->getOperand(i_nocapture: 0));
207 ToRemove.push_back(Elt: FI);
208 return true;
209}
210
211static bool legalizeI8Memory(Instruction &I,
212 SmallVectorImpl<Instruction *> &ToRemove,
213 DenseMap<Value *, Value *> &ReplacedValues) {
214
215 IRBuilder<> Builder(&I);
216 if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
217 if (!Store->getValueOperand()->getType()->isIntegerTy(BitWidth: 8))
218 return false;
219 Value *Pointer = ReplacedValues.lookup_or(Val: Store->getPointerOperand(),
220 Default: Store->getPointerOperand());
221 Type *Ty = nullptr;
222 if (auto *AI = dyn_cast<AllocaInst>(Val: Pointer))
223 Ty = AI->getAllocatedType();
224 else if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: Pointer))
225 Ty = GEP->getResultElementType();
226 else if (auto *GV = dyn_cast<GlobalVariable>(Val: Pointer))
227 Ty = GV->getValueType();
228 else if (isa<Argument>(Val: Pointer))
229 Ty = Builder.getInt32Ty();
230 if (auto *AT = dyn_cast_or_null<ArrayType>(Val: Ty))
231 Ty = AT->getElementType();
232 if (!Ty || !Ty->isIntegerTy() || getLegalIntegerType(Ty))
233 reportFatalUsageError(reason: "DXIL legalization cannot determine the i8 store's "
234 "storage type");
235 Value *V = Builder.CreateZExtOrTrunc(
236 V: ReplacedValues.lookup_or(Val: Store->getValueOperand(),
237 Default: Store->getValueOperand()),
238 DestTy: Builder.getInt32Ty());
239 V = Builder.CreateAnd(LHS: V, RHS: 255);
240 Value *NewStore =
241 Builder.CreateStore(Val: Builder.CreateZExtOrTrunc(V, DestTy: Ty), Ptr: Pointer);
242 ReplacedValues[Store] = NewStore;
243 ToRemove.push_back(Elt: Store);
244 return true;
245 }
246
247 if (auto *Load = dyn_cast<LoadInst>(Val: &I);
248 Load && I.getType()->isIntegerTy(BitWidth: 8)) {
249 Value *Pointer = ReplacedValues.lookup_or(Val: Load->getPointerOperand(),
250 Default: Load->getPointerOperand());
251 Type *ElementType = Pointer->getType();
252 if (auto *AI = dyn_cast<AllocaInst>(Val: Pointer))
253 ElementType = AI->getAllocatedType();
254 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: Pointer)) {
255 ElementType = GEP->getResultElementType();
256 }
257 if (ElementType->isArrayTy())
258 ElementType = ElementType->getArrayElementType();
259 LoadInst *NewLoad = Builder.CreateLoad(Ty: ElementType, Ptr: Pointer);
260 ReplacedValues[Load] = NewLoad;
261 ToRemove.push_back(Elt: Load);
262 return true;
263 }
264
265 if (auto *Load = dyn_cast<LoadInst>(Val: &I);
266 Load && isa<ConstantExpr>(Val: Load->getPointerOperand())) {
267 auto *CE = dyn_cast<ConstantExpr>(Val: Load->getPointerOperand());
268 if (!(CE->getOpcode() == Instruction::GetElementPtr))
269 return false;
270 auto *GEP = dyn_cast<GEPOperator>(Val: CE);
271 if (!GEP->getSourceElementType()->isIntegerTy(BitWidth: 8))
272 return false;
273
274 Type *ElementType = Load->getType();
275 ConstantInt *Offset = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1));
276 uint32_t ByteOffset = Offset->getZExtValue();
277 uint32_t ElemSize = Load->getDataLayout().getTypeAllocSize(Ty: ElementType);
278 uint32_t Index = ByteOffset / ElemSize;
279
280 Value *PtrOperand = GEP->getPointerOperand();
281 Type *GEPType = GEP->getPointerOperandType();
282
283 if (auto *GV = dyn_cast<GlobalVariable>(Val: PtrOperand))
284 GEPType = GV->getValueType();
285 if (auto *AI = dyn_cast<AllocaInst>(Val: PtrOperand))
286 GEPType = AI->getAllocatedType();
287
288 if (auto *ArrTy = dyn_cast<ArrayType>(Val: GEPType))
289 GEPType = ArrTy;
290 else
291 GEPType = ArrayType::get(ElementType, NumElements: 1); // its a scalar
292
293 Value *NewGEP = GetElementPtrInst::Create(
294 PointeeType: GEPType, Ptr: PtrOperand, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: Index)},
295 NW: GEP->getNoWrapFlags(), NameStr: GEP->getName(), InsertBefore: I.getIterator());
296
297 LoadInst *NewLoad = Builder.CreateLoad(Ty: ElementType, Ptr: NewGEP);
298 ReplacedValues[Load] = NewLoad;
299 Load->replaceAllUsesWith(V: NewLoad);
300 ToRemove.push_back(Elt: Load);
301 return true;
302 }
303
304 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I)) {
305 if (!GEP->getType()->isPointerTy() ||
306 !GEP->getSourceElementType()->isIntegerTy(BitWidth: 8))
307 return false;
308
309 Value *BasePtr = GEP->getPointerOperand();
310 if (ReplacedValues.count(Val: BasePtr))
311 BasePtr = ReplacedValues[BasePtr];
312
313 Type *ElementType = BasePtr->getType();
314
315 if (auto *AI = dyn_cast<AllocaInst>(Val: BasePtr))
316 ElementType = AI->getAllocatedType();
317 if (auto *GV = dyn_cast<GlobalVariable>(Val: BasePtr))
318 ElementType = GV->getValueType();
319
320 Type *GEPType = ElementType;
321 if (auto *ArrTy = dyn_cast<ArrayType>(Val: ElementType))
322 ElementType = ArrTy->getArrayElementType();
323 else
324 GEPType = ArrayType::get(ElementType, NumElements: 1); // its a scalar
325
326 ConstantInt *Offset = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1));
327 // Note: i8 to i32 offset conversion without emitting IR requires constant
328 // ints. Since offset conversion is common, we can safely assume Offset is
329 // always a ConstantInt, so no need to have a conditional bail out on
330 // nullptr, instead assert this is the case.
331 assert(Offset && "Offset is expected to be a ConstantInt");
332 uint32_t ByteOffset = Offset->getZExtValue();
333 uint32_t ElemSize = GEP->getDataLayout().getTypeAllocSize(Ty: ElementType);
334 assert(ElemSize > 0 && "ElementSize must be set");
335 uint32_t Index = ByteOffset / ElemSize;
336 Value *NewGEP = GetElementPtrInst::Create(
337 PointeeType: GEPType, Ptr: BasePtr, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: Index)},
338 NW: GEP->getNoWrapFlags(), NameStr: GEP->getName(), InsertBefore: GEP->getIterator());
339 ReplacedValues[GEP] = NewGEP;
340 GEP->replaceAllUsesWith(V: NewGEP);
341 ToRemove.push_back(Elt: GEP);
342 return true;
343 }
344 return false;
345}
346
347static bool upcastI8AllocasAndUses(Instruction &I,
348 SmallVectorImpl<Instruction *> &ToRemove,
349 DenseMap<Value *, Value *> &ReplacedValues) {
350 auto *AI = dyn_cast<AllocaInst>(Val: &I);
351 if (!AI || !AI->getAllocatedType()->isIntegerTy(BitWidth: 8))
352 return false;
353
354 Type *SmallestType = nullptr;
355
356 auto ProcessLoad = [&](LoadInst *Load) {
357 for (User *LU : Load->users()) {
358 CastInst *Cast = dyn_cast<CastInst>(Val: LU);
359 if (!Cast)
360 continue;
361 Type *Ty = Cast->getType();
362
363 if (!SmallestType ||
364 Ty->getPrimitiveSizeInBits() < SmallestType->getPrimitiveSizeInBits())
365 SmallestType = Ty;
366 }
367 };
368
369 for (User *U : AI->users()) {
370 if (auto *Load = dyn_cast<LoadInst>(Val: U))
371 ProcessLoad(Load);
372 else if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: U)) {
373 for (User *GU : GEP->users()) {
374 if (auto *Load = dyn_cast<LoadInst>(Val: GU))
375 ProcessLoad(Load);
376 }
377 }
378 }
379
380 if (!SmallestType)
381 return false; // no valid casts found
382
383 // Replace alloca
384 IRBuilder<> Builder(AI);
385 auto *NewAlloca = Builder.CreateAlloca(Ty: SmallestType);
386 ReplacedValues[AI] = NewAlloca;
387 ToRemove.push_back(Elt: AI);
388 return true;
389}
390
391static bool
392downcastI64toI32InsertExtractElements(Instruction &I,
393 SmallVectorImpl<Instruction *> &ToRemove,
394 DenseMap<Value *, Value *> &) {
395
396 if (auto *Extract = dyn_cast<ExtractElementInst>(Val: &I)) {
397 Value *Idx = Extract->getIndexOperand();
398 auto *CI = dyn_cast<ConstantInt>(Val: Idx);
399 if (CI && CI->getBitWidth() == 64) {
400 IRBuilder<> Builder(Extract);
401 int64_t IndexValue = CI->getSExtValue();
402 auto *Idx32 =
403 ConstantInt::get(Ty: Type::getInt32Ty(C&: I.getContext()), V: IndexValue);
404 Value *NewExtract = Builder.CreateExtractElement(
405 Vec: Extract->getVectorOperand(), Idx: Idx32, Name: Extract->getName());
406
407 Extract->replaceAllUsesWith(V: NewExtract);
408 ToRemove.push_back(Elt: Extract);
409 return true;
410 }
411 }
412
413 if (auto *Insert = dyn_cast<InsertElementInst>(Val: &I)) {
414 Value *Idx = Insert->getOperand(i_nocapture: 2);
415 auto *CI = dyn_cast<ConstantInt>(Val: Idx);
416 if (CI && CI->getBitWidth() == 64) {
417 int64_t IndexValue = CI->getSExtValue();
418 auto *Idx32 =
419 ConstantInt::get(Ty: Type::getInt32Ty(C&: I.getContext()), V: IndexValue);
420 IRBuilder<> Builder(Insert);
421 Value *Insert32Index = Builder.CreateInsertElement(
422 Vec: Insert->getOperand(i_nocapture: 0), NewElt: Insert->getOperand(i_nocapture: 1), Idx: Idx32,
423 Name: Insert->getName());
424
425 Insert->replaceAllUsesWith(V: Insert32Index);
426 ToRemove.push_back(Elt: Insert);
427 return true;
428 }
429 }
430 return false;
431}
432
433static bool updateFnegToFsub(Instruction &I,
434 SmallVectorImpl<Instruction *> &ToRemove,
435 DenseMap<Value *, Value *> &) {
436 const Intrinsic::ID ID = I.getOpcode();
437 if (ID != Instruction::FNeg)
438 return false;
439
440 IRBuilder<> Builder(&I);
441 Value *In = I.getOperand(i: 0);
442 Value *Zero = ConstantFP::get(Ty: In->getType(), V: -0.0);
443 I.replaceAllUsesWith(V: Builder.CreateFSub(L: Zero, R: In));
444 ToRemove.push_back(Elt: &I);
445 return true;
446}
447
448// DXIL has no floating-point atomic operation. A float exchange only moves the
449// bit pattern, so exchange an integer of the same width instead. Opaque
450// pointers keep the pointer operand type-agnostic, so only the value and the
451// result need a cast. This matches what DXC emits for groupshared memory.
452static bool
453legalizeFloatAtomicExchange(Instruction &I,
454 SmallVectorImpl<Instruction *> &ToRemove,
455 DenseMap<Value *, Value *> &) {
456 auto *AI = dyn_cast<AtomicRMWInst>(Val: &I);
457 if (!AI || AI->getOperation() != AtomicRMWInst::Xchg)
458 return false;
459
460 Type *ValTy = AI->getValOperand()->getType();
461 if (!ValTy->isFloatingPointTy())
462 return false;
463
464 // DXIL has 32-bit and 64-bit atomics only. A float of any other width has no
465 // integer exchange to lower to.
466 unsigned Width = ValTy->getPrimitiveSizeInBits();
467 if (Width != 32 && Width != 64)
468 reportFatalUsageError(reason: "DXIL atomic exchange requires a 32-bit or 64-bit "
469 "floating-point value");
470
471 IRBuilder<> Builder(AI);
472 Type *IntTy = Builder.getIntNTy(N: Width);
473 Value *Val = Builder.CreateBitCast(V: AI->getValOperand(), DestTy: IntTy);
474 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
475 Op: AtomicRMWInst::Xchg, Ptr: AI->getPointerOperand(), Val, Align: AI->getAlign(),
476 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
477 NewAI->copyMetadata(SrcInst: *AI);
478 AI->replaceAllUsesWith(V: Builder.CreateBitCast(V: NewAI, DestTy: ValTy));
479 ToRemove.push_back(Elt: AI);
480 return true;
481}
482
483static bool
484resolveUnreachableSwitchDefault(Instruction &I,
485 SmallVectorImpl<Instruction *> &ToRemove,
486 DenseMap<Value *, Value *> &) {
487 auto *SI = dyn_cast<SwitchInst>(Val: &I);
488 if (!SI || SI->getNumCases() == 0)
489 return false;
490
491 BasicBlock *DefaultBB = SI->getDefaultDest();
492
493 // Check if the default destination ends with an unreachable instruction.
494 if (DefaultBB->size() == 0 ||
495 !isa<UnreachableInst>(Val: DefaultBB->getTerminator()))
496 return false;
497
498 // Try to find a common successor of all case destinations. If all case
499 // blocks unconditionally branch to the same block, that is the common
500 // successor. This is just a best effort, and is done as the original form of
501 // the switch statement was likely in this form before being transformed to
502 // an unreachable branch.
503 BasicBlock *CommonSuccessor = nullptr;
504 for (auto &Case : SI->cases()) {
505 BasicBlock *CaseBB = Case.getCaseSuccessor();
506 auto *BI = dyn_cast<UncondBrInst>(Val: CaseBB->getTerminator());
507 if (!BI) {
508 CommonSuccessor = nullptr;
509 break;
510 }
511 BasicBlock *Succ = BI->getSuccessor(i: 0);
512 if (!CommonSuccessor)
513 CommonSuccessor = Succ;
514 else if (CommonSuccessor != Succ) {
515 CommonSuccessor = nullptr;
516 break;
517 }
518 }
519
520 BasicBlock *NewDefault =
521 CommonSuccessor ? CommonSuccessor : SI->case_begin()->getCaseSuccessor();
522
523 BasicBlock *SwitchBB = SI->getParent();
524 SI->setDefaultDest(NewDefault);
525
526 // Ensure all phi nodes are legal by adding an incoming poison value from the
527 // unreachable branch.
528 for (PHINode &Phi : NewDefault->phis())
529 Phi.addIncoming(V: PoisonValue::get(T: Phi.getType()), BB: SwitchBB);
530
531 return true;
532}
533
534static bool
535legalizeScalarLoadStoreOnArrays(Instruction &I,
536 SmallVectorImpl<Instruction *> &ToRemove,
537 DenseMap<Value *, Value *> &) {
538
539 Value *PtrOp;
540 unsigned PtrOpIndex;
541 [[maybe_unused]] Type *LoadStoreTy;
542 if (auto *LI = dyn_cast<LoadInst>(Val: &I)) {
543 PtrOp = LI->getPointerOperand();
544 PtrOpIndex = LI->getPointerOperandIndex();
545 LoadStoreTy = LI->getType();
546 } else if (auto *SI = dyn_cast<StoreInst>(Val: &I)) {
547 PtrOp = SI->getPointerOperand();
548 PtrOpIndex = SI->getPointerOperandIndex();
549 LoadStoreTy = SI->getValueOperand()->getType();
550 } else
551 return false;
552
553 // If the load/store is not of a single-value type (i.e., scalar or vector)
554 // then we do not modify it. It shouldn't be a vector either because the
555 // dxil-data-scalarization pass is expected to run before this, but it's not
556 // incorrect to apply this transformation to vector load/stores.
557 if (!LoadStoreTy->isSingleValueType())
558 return false;
559
560 Type *ArrayTy;
561 if (auto *GlobalVarPtrOp = dyn_cast<GlobalVariable>(Val: PtrOp))
562 ArrayTy = GlobalVarPtrOp->getValueType();
563 else if (auto *AllocaPtrOp = dyn_cast<AllocaInst>(Val: PtrOp))
564 ArrayTy = AllocaPtrOp->getAllocatedType();
565 else
566 return false;
567
568 if (!isa<ArrayType>(Val: ArrayTy))
569 return false;
570
571 assert(ArrayTy->getArrayElementType() == LoadStoreTy &&
572 "Expected array element type to be the same as to the scalar load or "
573 "store type");
574
575 Value *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: I.getContext()), V: 0);
576 Value *GEP = GetElementPtrInst::Create(
577 PointeeType: ArrayTy, Ptr: PtrOp, IdxList: {Zero, Zero}, NW: GEPNoWrapFlags::all(), NameStr: "", InsertBefore: I.getIterator());
578 I.setOperand(i: PtrOpIndex, Val: GEP);
579 return true;
580}
581
582namespace {
583class DXILLegalizationPipeline {
584
585public:
586 DXILLegalizationPipeline() { initializeLegalizationPipeline(); }
587
588 bool runLegalizationPipeline(Function &F) {
589 bool MadeChange = removeUnreachableBlocks(F);
590 SmallVector<Instruction *> ToRemove;
591 DenseMap<Value *, Value *> ReplacedValues;
592 for (int Stage = 0; Stage < NumStages; ++Stage) {
593 ToRemove.clear();
594 ReplacedValues.clear();
595 for (BasicBlock *BB : ReversePostOrderTraversal<Function *>(&F))
596 for (Instruction &I : *BB)
597 for (auto &LegalizationFn : LegalizationPipeline[Stage])
598 if (LegalizationFn(I, ToRemove, ReplacedValues)) {
599 MadeChange = true;
600 break;
601 }
602
603 for (auto *Inst : reverse(C&: ToRemove))
604 Inst->eraseFromParent();
605 }
606
607 if (MadeChange)
608 MadeChange |= removeUnreachableBlocks(F);
609 return MadeChange;
610 }
611
612private:
613 enum LegalizationStage { Stage1 = 0, Stage2 = 1, NumStages };
614
615 using LegalizationFnTy =
616 std::function<bool(Instruction &, SmallVectorImpl<Instruction *> &,
617 DenseMap<Value *, Value *> &)>;
618
619 SmallVector<LegalizationFnTy> LegalizationPipeline[NumStages];
620
621 void initializeLegalizationPipeline() {
622 LegalizationPipeline[Stage1].push_back(Elt: upcastI8AllocasAndUses);
623 LegalizationPipeline[Stage1].push_back(Elt: legalizeFreeze);
624 LegalizationPipeline[Stage1].push_back(Elt: legalizeI8Memory);
625 LegalizationPipeline[Stage1].push_back(Elt: legalizeInteger);
626 LegalizationPipeline[Stage1].push_back(Elt: updateFnegToFsub);
627 LegalizationPipeline[Stage1].push_back(Elt: legalizeFloatAtomicExchange);
628 LegalizationPipeline[Stage1].push_back(
629 Elt: downcastI64toI32InsertExtractElements);
630 LegalizationPipeline[Stage2].push_back(Elt: legalizeScalarLoadStoreOnArrays);
631 LegalizationPipeline[Stage2].push_back(Elt: resolveUnreachableSwitchDefault);
632 }
633};
634
635class DXILLegalizeLegacy : public FunctionPass {
636
637public:
638 bool runOnFunction(Function &F) override;
639 DXILLegalizeLegacy() : FunctionPass(ID) {}
640
641 static char ID; // Pass identification.
642};
643} // namespace
644
645PreservedAnalyses DXILLegalizePass::run(Function &F,
646 FunctionAnalysisManager &FAM) {
647 DXILLegalizationPipeline DXLegalize;
648 bool MadeChanges = DXLegalize.runLegalizationPipeline(F);
649 if (!MadeChanges)
650 return PreservedAnalyses::all();
651 PreservedAnalyses PA;
652 return PA;
653}
654
655bool DXILLegalizeLegacy::runOnFunction(Function &F) {
656 DXILLegalizationPipeline DXLegalize;
657 return DXLegalize.runLegalizationPipeline(F);
658}
659
660char DXILLegalizeLegacy::ID = 0;
661
662INITIALIZE_PASS_BEGIN(DXILLegalizeLegacy, DEBUG_TYPE, "DXIL Legalizer", false,
663 false)
664INITIALIZE_PASS_END(DXILLegalizeLegacy, DEBUG_TYPE, "DXIL Legalizer", false,
665 false)
666
667FunctionPass *llvm::createDXILLegalizeLegacyPass() {
668 return new DXILLegalizeLegacy();
669}
670