1//===AMDGPUAsanInstrumentation.cpp - ASAN related helper functions===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===-------------------------------------------------------------===//
8
9#include "AMDGPUAsanInstrumentation.h"
10#include "AMDGPUInstrInfo.h"
11
12#define DEBUG_TYPE "amdgpu-asan-instrumentation"
13
14using namespace llvm;
15
16namespace llvm {
17namespace AMDGPU {
18
19static uint64_t getRedzoneSizeForScale(int AsanScale) {
20 // Redzone used for stack and globals is at least 32 bytes.
21 // For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.
22 return std::max(a: 32U, b: 1U << AsanScale);
23}
24
25static uint64_t getMinRedzoneSizeForGlobal(int AsanScale) {
26 return getRedzoneSizeForScale(AsanScale);
27}
28
29uint64_t getRedzoneSizeForGlobal(int AsanScale, uint64_t SizeInBytes) {
30 constexpr uint64_t kMaxRZ = 1 << 18;
31 const uint64_t MinRZ = getMinRedzoneSizeForGlobal(AsanScale);
32
33 uint64_t RZ = 0;
34 if (SizeInBytes <= MinRZ / 2) {
35 // Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is
36 // at least 32 bytes, optimize when SizeInBytes is less than or equal to
37 // half of MinRZ.
38 RZ = MinRZ - SizeInBytes;
39 } else {
40 // Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes.
41 RZ = std::clamp(val: (SizeInBytes / MinRZ / 4) * MinRZ, lo: MinRZ, hi: kMaxRZ);
42
43 // Round up to multiple of MinRZ.
44 if (SizeInBytes % MinRZ)
45 RZ += MinRZ - (SizeInBytes % MinRZ);
46 }
47
48 assert((RZ + SizeInBytes) % MinRZ == 0);
49
50 return RZ;
51}
52
53static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) {
54 size_t Res = llvm::countr_zero(Val: TypeSize / 8);
55 return Res;
56}
57
58static Instruction *genAMDGPUReportBlock(Module &M, IRBuilder<> &IRB,
59 Value *Cond, bool Recover) {
60 Value *ReportCond = Cond;
61 if (!Recover) {
62 auto *Ballot =
63 IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_ballot, OverloadTypes: IRB.getInt64Ty(), Args: {Cond});
64 ReportCond = IRB.CreateIsNotNull(Arg: Ballot);
65 }
66
67 auto *Trm = SplitBlockAndInsertIfThen(
68 Cond: ReportCond, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false,
69 BranchWeights: MDBuilder(M.getContext()).createUnlikelyBranchWeights());
70 Trm->getParent()->setName("asan.report");
71
72 if (Recover)
73 return Trm;
74
75 Trm = SplitBlockAndInsertIfThen(Cond, SplitBefore: Trm, Unreachable: false);
76 IRB.SetInsertPoint(Trm);
77 return IRB.CreateIntrinsicWithoutFolding(ID: Intrinsic::amdgcn_unreachable, Args: {});
78}
79
80static Value *createSlowPathCmp(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
81 Value *AddrLong, Value *ShadowValue,
82 uint32_t TypeStoreSize, int AsanScale) {
83 uint64_t Granularity = static_cast<uint64_t>(1) << AsanScale;
84 // Addr & (Granularity - 1)
85 Value *LastAccessedByte =
86 IRB.CreateAnd(LHS: AddrLong, RHS: ConstantInt::get(Ty: IntptrTy, V: Granularity - 1));
87 // (Addr & (Granularity - 1)) + size - 1
88 if (TypeStoreSize / 8 > 1)
89 LastAccessedByte = IRB.CreateAdd(
90 LHS: LastAccessedByte, RHS: ConstantInt::get(Ty: IntptrTy, V: TypeStoreSize / 8 - 1));
91 // (uint8_t) ((Addr & (Granularity-1)) + size - 1)
92 LastAccessedByte =
93 IRB.CreateIntCast(V: LastAccessedByte, DestTy: ShadowValue->getType(), isSigned: false);
94 // ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue
95 return IRB.CreateICmpSGE(LHS: LastAccessedByte, RHS: ShadowValue);
96}
97
98static Instruction *generateCrashCode(Module &M, IRBuilder<> &IRB,
99 Type *IntptrTy, Instruction *InsertBefore,
100 Value *Addr, bool IsWrite,
101 size_t AccessSizeIndex,
102 Value *SizeArgument, bool Recover) {
103 IRB.SetInsertPoint(InsertBefore);
104 CallInst *Call = nullptr;
105 SmallString<128> kAsanReportErrorTemplate{"__asan_report_"};
106 SmallString<64> TypeStr{IsWrite ? "store" : "load"};
107 SmallString<64> EndingStr{Recover ? "_noabort" : ""};
108
109 SmallString<128> AsanErrorCallbackSizedString;
110 raw_svector_ostream AsanErrorCallbackSizedOS(AsanErrorCallbackSizedString);
111 AsanErrorCallbackSizedOS << kAsanReportErrorTemplate << TypeStr << "_n"
112 << EndingStr;
113
114 SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
115 AttributeList AL2;
116 FunctionCallee AsanErrorCallbackSized = M.getOrInsertFunction(
117 Name: AsanErrorCallbackSizedOS.str(),
118 T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args2, isVarArg: false), AttributeList: AL2);
119 SmallVector<Type *, 2> Args1{1, IntptrTy};
120 AttributeList AL1;
121
122 SmallString<128> AsanErrorCallbackString;
123 raw_svector_ostream AsanErrorCallbackOS(AsanErrorCallbackString);
124 AsanErrorCallbackOS << kAsanReportErrorTemplate << TypeStr
125 << (1ULL << AccessSizeIndex) << EndingStr;
126
127 FunctionCallee AsanErrorCallback = M.getOrInsertFunction(
128 Name: AsanErrorCallbackOS.str(),
129 T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false), AttributeList: AL1);
130 if (SizeArgument) {
131 Call = IRB.CreateCall(Callee: AsanErrorCallbackSized, Args: {Addr, SizeArgument});
132 } else {
133 Call = IRB.CreateCall(Callee: AsanErrorCallback, Args: Addr);
134 }
135
136 Call->setCannotMerge();
137 return Call;
138}
139
140static Value *memToShadow(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
141 Value *Shadow, int AsanScale, uint32_t AsanOffset) {
142 // Shadow >> scale
143 Shadow = IRB.CreateLShr(LHS: Shadow, RHS: AsanScale);
144 if (AsanOffset == 0)
145 return Shadow;
146 // (Shadow >> scale) | offset
147 Value *ShadowBase = ConstantInt::get(Ty: IntptrTy, V: AsanOffset);
148 return IRB.CreateAdd(LHS: Shadow, RHS: ShadowBase);
149}
150
151static void instrumentAddressImpl(Module &M, IRBuilder<> &IRB,
152 Instruction *OrigIns,
153 Instruction *InsertBefore, Value *Addr,
154 Align Alignment, uint32_t TypeStoreSize,
155 bool IsWrite, Value *SizeArgument,
156 bool UseCalls, bool Recover, int AsanScale,
157 int AsanOffset) {
158 Type *AddrTy = Addr->getType();
159 Type *IntptrTy = M.getDataLayout().getIntPtrType(
160 C&: M.getContext(), AddressSpace: AddrTy->getPointerAddressSpace());
161 IRB.SetInsertPoint(InsertBefore);
162 size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeSize: TypeStoreSize);
163 Type *ShadowTy = IntegerType::get(C&: M.getContext(),
164 NumBits: std::max(a: 8U, b: TypeStoreSize >> AsanScale));
165 Type *ShadowPtrTy = PointerType::get(C&: M.getContext(), AddressSpace: 0);
166 Value *AddrLong = IRB.CreatePtrToInt(V: Addr, DestTy: IntptrTy);
167 Value *ShadowPtr =
168 memToShadow(M, IRB, IntptrTy, Shadow: AddrLong, AsanScale, AsanOffset);
169 const uint64_t ShadowAlign =
170 std::max<uint64_t>(a: Alignment.value() >> AsanScale, b: 1);
171 Value *ShadowValue = IRB.CreateAlignedLoad(
172 Ty: ShadowTy, Ptr: IRB.CreateIntToPtr(V: ShadowPtr, DestTy: ShadowPtrTy), Align: Align(ShadowAlign));
173 Value *Cmp = IRB.CreateIsNotNull(Arg: ShadowValue);
174 auto *Cmp2 = createSlowPathCmp(M, IRB, IntptrTy, AddrLong, ShadowValue,
175 TypeStoreSize, AsanScale);
176 Cmp = IRB.CreateAnd(LHS: Cmp, RHS: Cmp2);
177 Instruction *CrashTerm = genAMDGPUReportBlock(M, IRB, Cond: Cmp, Recover);
178 Instruction *Crash =
179 generateCrashCode(M, IRB, IntptrTy, InsertBefore: CrashTerm, Addr: AddrLong, IsWrite,
180 AccessSizeIndex, SizeArgument, Recover);
181 Crash->setDebugLoc(OrigIns->getDebugLoc());
182}
183
184void instrumentAddress(Module &M, IRBuilder<> &IRB, Instruction *OrigIns,
185 Instruction *InsertBefore, Value *Addr, Align Alignment,
186 TypeSize TypeStoreSize, bool IsWrite,
187 Value *SizeArgument, bool UseCalls, bool Recover,
188 int AsanScale, int AsanOffset) {
189 if (!TypeStoreSize.isScalable()) {
190 unsigned Granularity = 1 << AsanScale;
191 const auto FixedSize = TypeStoreSize.getFixedValue();
192 switch (FixedSize) {
193 case 8:
194 case 16:
195 case 32:
196 case 64:
197 case 128:
198 if (Alignment.value() >= Granularity ||
199 Alignment.value() >= FixedSize / 8)
200 return instrumentAddressImpl(
201 M, IRB, OrigIns, InsertBefore, Addr, Alignment, TypeStoreSize: FixedSize, IsWrite,
202 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
203 }
204 }
205 // Instrument unusual size or unusual alignment.
206 IRB.SetInsertPoint(InsertBefore);
207 Type *AddrTy = Addr->getType();
208 Type *IntptrTy = M.getDataLayout().getIntPtrType(AddrTy);
209 Value *NumBits = IRB.CreateTypeSize(Ty: IntptrTy, Size: TypeStoreSize);
210 Value *Size = IRB.CreateLShr(LHS: NumBits, RHS: ConstantInt::get(Ty: IntptrTy, V: 3));
211 Value *AddrLong = IRB.CreatePtrToInt(V: Addr, DestTy: IntptrTy);
212 Value *SizeMinusOne =
213 IRB.CreateAdd(LHS: Size, RHS: ConstantInt::getAllOnesValue(Ty: IntptrTy));
214 Value *LastByte =
215 IRB.CreateIntToPtr(V: IRB.CreateAdd(LHS: AddrLong, RHS: SizeMinusOne), DestTy: AddrTy);
216 instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, Addr, Alignment: {}, TypeStoreSize: 8, IsWrite,
217 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
218 instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, Addr: LastByte, Alignment: {}, TypeStoreSize: 8, IsWrite,
219 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
220}
221
222void getInterestingMemoryOperands(
223 Module &M, Instruction *I,
224 SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
225 const DataLayout &DL = M.getDataLayout();
226 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
227 Interesting.emplace_back(Args&: I, Args: LI->getPointerOperandIndex(), Args: false,
228 Args: LI->getType(), Args: LI->getAlign());
229 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
230 Interesting.emplace_back(Args&: I, Args: SI->getPointerOperandIndex(), Args: true,
231 Args: SI->getValueOperand()->getType(), Args: SI->getAlign());
232 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
233 Interesting.emplace_back(Args&: I, Args: RMW->getPointerOperandIndex(), Args: true,
234 Args: RMW->getValOperand()->getType(), Args: std::nullopt);
235 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
236 Interesting.emplace_back(Args&: I, Args: XCHG->getPointerOperandIndex(), Args: true,
237 Args: XCHG->getCompareOperand()->getType(),
238 Args: std::nullopt);
239 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
240 switch (CI->getIntrinsicID()) {
241 case Intrinsic::masked_load:
242 case Intrinsic::masked_store:
243 case Intrinsic::masked_gather:
244 case Intrinsic::masked_scatter: {
245 bool IsWrite = CI->getType()->isVoidTy();
246 // Masked store has an initial operand for the value.
247 unsigned OpOffset = IsWrite ? 1 : 0;
248 Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
249 MaybeAlign Alignment = CI->getParamAlign(ArgNo: OpOffset);
250 Value *Mask = CI->getOperand(i_nocapture: 1 + OpOffset);
251 Interesting.emplace_back(Args&: I, Args&: OpOffset, Args&: IsWrite, Args&: Ty, Args&: Alignment, Args&: Mask);
252 break;
253 }
254 case Intrinsic::masked_expandload:
255 case Intrinsic::masked_compressstore: {
256 bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;
257 unsigned OpOffset = IsWrite ? 1 : 0;
258 auto *BasePtr = CI->getOperand(i_nocapture: OpOffset);
259 MaybeAlign Alignment = BasePtr->getPointerAlignment(DL);
260 Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
261 IRBuilder<> IB(I);
262 Value *Mask = CI->getOperand(i_nocapture: 1 + OpOffset);
263 Type *IntptrTy = M.getDataLayout().getIntPtrType(
264 C&: M.getContext(), AddressSpace: BasePtr->getType()->getPointerAddressSpace());
265 // Use the popcount of Mask as the effective vector length.
266 Type *ExtTy = VectorType::get(ElementType: IntptrTy, Other: cast<VectorType>(Val: Ty));
267 Value *ExtMask = IB.CreateZExt(V: Mask, DestTy: ExtTy);
268 Value *EVL = IB.CreateAddReduce(Src: ExtMask);
269 Value *TrueMask = ConstantInt::get(Ty: Mask->getType(), V: 1);
270 Interesting.emplace_back(Args&: I, Args&: OpOffset, Args&: IsWrite, Args&: Ty, Args&: Alignment, Args&: TrueMask,
271 Args&: EVL);
272 break;
273 }
274 case Intrinsic::vp_load:
275 case Intrinsic::vp_store:
276 case Intrinsic::experimental_vp_strided_load:
277 case Intrinsic::experimental_vp_strided_store: {
278 auto *VPI = cast<VPIntrinsic>(Val: CI);
279 unsigned IID = CI->getIntrinsicID();
280 bool IsWrite = CI->getType()->isVoidTy();
281 unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
282 Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
283 MaybeAlign Alignment = VPI->getOperand(i_nocapture: PtrOpNo)->getPointerAlignment(DL);
284 Value *Stride = nullptr;
285 if (IID == Intrinsic::experimental_vp_strided_store ||
286 IID == Intrinsic::experimental_vp_strided_load) {
287 Stride = VPI->getOperand(i_nocapture: PtrOpNo + 1);
288 // Use the pointer alignment as the element alignment if the stride is a
289 // mutiple of the pointer alignment. Otherwise, the element alignment
290 // should be Align(1).
291 unsigned PointerAlign = Alignment.valueOrOne().value();
292 if (!isa<ConstantInt>(Val: Stride) ||
293 cast<ConstantInt>(Val: Stride)->getZExtValue() % PointerAlign != 0)
294 Alignment = Align(1);
295 }
296 Interesting.emplace_back(Args&: I, Args&: PtrOpNo, Args&: IsWrite, Args&: Ty, Args&: Alignment,
297 Args: VPI->getMaskParam(), Args: VPI->getVectorLengthParam(),
298 Args&: Stride);
299 break;
300 }
301 case Intrinsic::vp_gather:
302 case Intrinsic::vp_scatter: {
303 auto *VPI = cast<VPIntrinsic>(Val: CI);
304 unsigned IID = CI->getIntrinsicID();
305 bool IsWrite = IID == Intrinsic::vp_scatter;
306 unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
307 Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
308 MaybeAlign Alignment = VPI->getPointerAlignment();
309 Interesting.emplace_back(Args&: I, Args&: PtrOpNo, Args&: IsWrite, Args&: Ty, Args&: Alignment,
310 Args: VPI->getMaskParam(),
311 Args: VPI->getVectorLengthParam());
312 break;
313 }
314 case Intrinsic::amdgcn_global_load_tr_b64:
315 case Intrinsic::amdgcn_global_load_tr_b128: {
316 unsigned PtrOpNo = 0;
317 bool IsWrite = false;
318 Type *Ty = CI->getType();
319 Value *Ptr = CI->getArgOperand(i: PtrOpNo);
320 MaybeAlign Alignment = Ptr->getPointerAlignment(DL);
321 Interesting.emplace_back(Args&: I, Args&: PtrOpNo, Args&: IsWrite, Args&: Ty, Args&: Alignment);
322 break;
323 }
324 default:
325 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
326 AMDGPU::lookupBufferIntrinsic(Intr: CI->getIntrinsicID());
327 BufInfo && (BufInfo->Kind == AMDGPU::BufferKindLoad ||
328 BufInfo->Kind == AMDGPU::BufferKindSLoad ||
329 BufInfo->Kind == AMDGPU::BufferKindStore)) {
330 unsigned PtrOpNo = BufInfo->RsrcArg;
331 bool IsWrite = BufInfo->Kind == AMDGPU::BufferKindStore;
332 Value *Ptr = CI->getArgOperand(i: PtrOpNo);
333 Type *Ty = IsWrite ? Ptr->getType() : CI->getType();
334 MaybeAlign Alignment = Ptr->getPointerAlignment(DL);
335 Interesting.emplace_back(Args&: I, Args&: PtrOpNo, Args&: IsWrite, Args&: Ty, Args&: Alignment);
336 break;
337 }
338 for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
339 if (Type *Ty = CI->getParamByRefType(ArgNo)) {
340 Interesting.emplace_back(Args&: I, Args&: ArgNo, Args: false, Args&: Ty, Args: Align(1));
341 } else if (Type *Ty = CI->getParamByValType(ArgNo)) {
342 Interesting.emplace_back(Args&: I, Args&: ArgNo, Args: false, Args&: Ty, Args: Align(1));
343 }
344 }
345 }
346 }
347}
348} // end namespace AMDGPU
349} // end namespace llvm
350