1//===- DXILMemIntrinsics.cpp - Eliminate Memory Intrinsics ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "DirectX.h"
10#include "llvm/Analysis/DXILResource.h"
11#include "llvm/IR/IRBuilder.h"
12#include "llvm/IR/IntrinsicInst.h"
13#include "llvm/IR/IntrinsicsDirectX.h"
14#include "llvm/IR/Module.h"
15
16#define DEBUG_TYPE "dxil-mem-intrinsics"
17
18using namespace llvm;
19
20void expandMemSet(MemSetInst *MemSet) {
21 IRBuilder<> Builder(MemSet);
22 Value *Dst = MemSet->getDest();
23 Value *Val = MemSet->getValue();
24 ConstantInt *LengthCI = dyn_cast<ConstantInt>(Val: MemSet->getLength());
25 assert(LengthCI && "Expected length to be a ConstantInt");
26
27 [[maybe_unused]] const DataLayout &DL = Builder.getDataLayout();
28 [[maybe_unused]] uint64_t OrigLength = LengthCI->getZExtValue();
29
30 AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: Dst);
31
32 assert(Alloca && "Expected memset on an Alloca");
33 assert(OrigLength == Alloca->getAllocationSize(DL)->getFixedValue() &&
34 "Expected for memset size to match DataLayout size");
35
36 Type *AllocatedTy = Alloca->getAllocatedType();
37 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: AllocatedTy);
38 assert(ArrTy && "Expected Alloca for an Array Type");
39
40 Type *ElemTy = ArrTy->getElementType();
41 uint64_t Size = ArrTy->getArrayNumElements();
42
43 [[maybe_unused]] uint64_t ElemSize = DL.getTypeStoreSize(Ty: ElemTy);
44
45 assert(ElemSize > 0 && "Size must be set");
46 assert(OrigLength == ElemSize * Size && "Size in bytes must match");
47
48 Value *TypedVal = Val;
49
50 if (Val->getType() != ElemTy)
51 TypedVal = Builder.CreateIntCast(V: Val, DestTy: ElemTy, isSigned: false);
52
53 for (uint64_t I = 0; I < Size; ++I) {
54 Value *Zero = Builder.getInt32(C: 0);
55 Value *Offset = Builder.getInt32(C: I);
56 Value *Ptr = GetElementPtrInst::Create(PointeeType: ArrTy, Ptr: Dst, IdxList: {Zero, Offset}, NameStr: "gep",
57 InsertBefore: MemSet->getIterator());
58 Builder.CreateStore(Val: TypedVal, Ptr);
59 }
60
61 MemSet->eraseFromParent();
62}
63
64static Type *getPointeeType(Value *Ptr, const DataLayout &DL) {
65 if (auto *GV = dyn_cast<GlobalVariable>(Val: Ptr))
66 return GV->getValueType();
67 if (auto *AI = dyn_cast<AllocaInst>(Val: Ptr))
68 return AI->getAllocatedType();
69
70 if (auto *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
71 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer) {
72 Type *Ty = cast<dxil::AnyResourceExtType>(Val: II->getArgOperand(i: 0)->getType())
73 ->getResourceType();
74 assert(Ty && "getpointer used on untyped resource");
75 return Ty;
76 }
77 }
78
79 if (auto *GEP = dyn_cast<GEPOperator>(Val: Ptr)) {
80 Type *Ty = GEP->getResultElementType();
81 if (!Ty->isIntegerTy(BitWidth: 8))
82 return Ty;
83
84 // We have ptradd, so we have to hope there's enough information to work out
85 // what we're indexing.
86 Type *IndexedType = getPointeeType(Ptr: GEP->getPointerOperand(), DL);
87 if (auto *AT = dyn_cast<ArrayType>(Val: IndexedType))
88 return AT->getElementType();
89
90 if (auto *ST = dyn_cast<StructType>(Val: IndexedType)) {
91 // Indexing a struct should always be constant
92 APInt ConstantOffset(DL.getIndexTypeSizeInBits(Ty: GEP->getType()), 0);
93 [[maybe_unused]] bool IsConst =
94 GEP->accumulateConstantOffset(DL, Offset&: ConstantOffset);
95 assert(IsConst && "Non-constant GEP into struct?");
96
97 // Now, work out what we'll find at that offset.
98 const StructLayout *Layout = DL.getStructLayout(Ty: ST);
99 unsigned Idx =
100 Layout->getElementContainingOffset(FixedOffset: ConstantOffset.getZExtValue());
101
102 return ST->getTypeAtIndex(N: Idx);
103 }
104
105 llvm_unreachable("Could not infer type from GEP");
106 }
107
108 llvm_unreachable("Could not calculate pointee type");
109}
110
111static size_t flattenTypes(Type *ContainerTy, const DataLayout &DL,
112 SmallVectorImpl<std::pair<Type *, size_t>> &FlatTys,
113 size_t NextOffset = 0) {
114 if (auto *AT = dyn_cast<ArrayType>(Val: ContainerTy)) {
115 for (uint64_t I = 0, E = AT->getNumElements(); I != E; ++I)
116 NextOffset = flattenTypes(ContainerTy: AT->getElementType(), DL, FlatTys, NextOffset);
117 return NextOffset;
118 }
119 if (auto *ST = dyn_cast<StructType>(Val: ContainerTy)) {
120 for (Type *Ty : ST->elements())
121 NextOffset = flattenTypes(ContainerTy: Ty, DL, FlatTys, NextOffset);
122 return NextOffset;
123 }
124
125 FlatTys.emplace_back(Args&: ContainerTy, Args&: NextOffset);
126 return NextOffset + DL.getTypeStoreSize(Ty: ContainerTy);
127}
128
129void expandMemCpy(MemCpyInst *MemCpy) {
130 IRBuilder<> Builder(MemCpy);
131 Value *Dst = MemCpy->getDest();
132 Value *Src = MemCpy->getSource();
133 ConstantInt *LengthCI = dyn_cast<ConstantInt>(Val: MemCpy->getLength());
134 assert(LengthCI && "Expected Length to be a ConstantInt");
135 assert(!MemCpy->isVolatile() && "Handling for volatile not implemented");
136
137 uint64_t ByteLength = LengthCI->getZExtValue();
138 // If length to copy is zero, no memcpy is needed.
139 if (ByteLength == 0)
140 return;
141
142 const DataLayout &DL = Builder.getDataLayout();
143
144 SmallVector<std::pair<Type *, size_t>> FlattenedTypes;
145 [[maybe_unused]] size_t MaxLength =
146 flattenTypes(ContainerTy: getPointeeType(Ptr: Dst, DL), DL, FlatTys&: FlattenedTypes);
147 assert(MaxLength >= ByteLength && "Dst not large enough for memcpy");
148
149 LLVM_DEBUG({
150 // Check if Src is layout compatible with Dst. This should always be true
151 // unless the frontend did something wrong.
152 SmallVector<std::pair<Type *, size_t>> SrcTypes;
153 size_t SrcLength = flattenTypes(getPointeeType(Src, DL), DL, SrcTypes);
154 assert(SrcLength >= ByteLength && "Src not large enough for memcpy");
155 for (const auto &[LHS, RHS] : zip(FlattenedTypes, SrcTypes)) {
156 auto &[DstTy, DstOffset] = LHS;
157 auto &[SrcTy, SrcOffset] = RHS;
158 assert(DstTy == SrcTy && "Mismatched types for memcpy");
159 assert(DstOffset == SrcOffset && "Incompatible layouts for memcpy");
160 if (DstOffset >= ByteLength)
161 break;
162 }
163 });
164
165 for (const auto &[Ty, Offset] : FlattenedTypes) {
166 if (Offset >= ByteLength)
167 break;
168 // TODO: Should we skip padding types here?
169 Value *SrcPtr = Builder.CreatePtrAddWithUnsignedOffset(
170 Ptr: Src, Offset, Name: "", NW: GEPNoWrapFlags::inBounds());
171 Value *SrcVal = Builder.CreateLoad(Ty, Ptr: SrcPtr);
172 Value *DstPtr = Builder.CreatePtrAddWithUnsignedOffset(
173 Ptr: Dst, Offset, Name: "", NW: GEPNoWrapFlags::inBounds());
174 Builder.CreateStore(Val: SrcVal, Ptr: DstPtr);
175 }
176
177 MemCpy->eraseFromParent();
178}
179
180void expandMemMove(MemMoveInst *MemMove) {
181 report_fatal_error(reason: "memmove expansion is not implemented yet.");
182}
183
184static bool eliminateMemIntrinsics(Module &M) {
185 bool HadMemIntrinsicUses = false;
186 for (auto &F : make_early_inc_range(Range: M.functions())) {
187 Intrinsic::ID IID = F.getIntrinsicID();
188 switch (IID) {
189 case Intrinsic::memcpy:
190 case Intrinsic::memcpy_inline:
191 case Intrinsic::memmove:
192 case Intrinsic::memset:
193 case Intrinsic::memset_inline:
194 break;
195 default:
196 continue;
197 }
198 for (User *U : make_early_inc_range(Range: F.users())) {
199 HadMemIntrinsicUses = true;
200 if (auto *MemSet = dyn_cast<MemSetInst>(Val: U))
201 expandMemSet(MemSet);
202 else if (auto *MemCpy = dyn_cast<MemCpyInst>(Val: U))
203 expandMemCpy(MemCpy);
204 else if (auto *MemMove = dyn_cast<MemMoveInst>(Val: U))
205 expandMemMove(MemMove);
206 else
207 llvm_unreachable("Unhandled memory intrinsic");
208 }
209 assert(F.user_empty() && "Mem intrinsic not eliminated?");
210 F.eraseFromParent();
211 }
212 return HadMemIntrinsicUses;
213}
214
215PreservedAnalyses DXILMemIntrinsicsPass::run(Module &M,
216 ModuleAnalysisManager &) {
217 if (eliminateMemIntrinsics(M))
218 return PreservedAnalyses::none();
219 return PreservedAnalyses::all();
220}
221
222class DXILMemIntrinsicsLegacy : public ModulePass {
223public:
224 bool runOnModule(Module &M) override { return eliminateMemIntrinsics(M); }
225 DXILMemIntrinsicsLegacy() : ModulePass(ID) {}
226
227 static char ID; // Pass identification.
228};
229char DXILMemIntrinsicsLegacy::ID = 0;
230
231INITIALIZE_PASS_BEGIN(DXILMemIntrinsicsLegacy, DEBUG_TYPE,
232 "DXIL Memory Intrinsic Elimination", false, false)
233INITIALIZE_PASS_END(DXILMemIntrinsicsLegacy, DEBUG_TYPE,
234 "DXIL Memory Intrinsic Elimination", false, false)
235
236ModulePass *llvm::createDXILMemIntrinsicsLegacyPass() {
237 return new DXILMemIntrinsicsLegacy();
238}
239