1//===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass abstracted struct/union member accesses in order to support
10// compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program
11// which can run on different kernels. In particular, if bpf program tries to
12// access a particular kernel data structure member, the details of the
13// intermediate member access will be remembered so bpf loader can do
14// necessary adjustment right before program loading.
15//
16// For example,
17//
18// struct s {
19// int a;
20// int b;
21// };
22// struct t {
23// struct s c;
24// int d;
25// };
26// struct t e;
27//
28// For the member access e.c.b, the compiler will generate code
29// &e + 4
30//
31// The compile-once run-everywhere instead generates the following code
32// r = 4
33// &e + r
34// The "4" in "r = 4" can be changed based on a particular kernel version.
35// For example, on a particular kernel version, if struct s is changed to
36//
37// struct s {
38// int new_field;
39// int a;
40// int b;
41// }
42//
43// By repeating the member access on the host, the bpf loader can
44// adjust "r = 4" as "r = 8".
45//
46// This feature relies on the following three intrinsic calls:
47// addr = preserve_array_access_index(base, dimension, index)
48// addr = preserve_union_access_index(base, di_index)
49// !llvm.preserve.access.index <union_ditype>
50// addr = preserve_struct_access_index(base, gep_index, di_index)
51// !llvm.preserve.access.index <struct_ditype>
52//
53// Bitfield member access needs special attention. User cannot take the
54// address of a bitfield acceess. To facilitate kernel verifier
55// for easy bitfield code optimization, a new clang intrinsic is introduced:
56// uint32_t __builtin_preserve_field_info(member_access, info_kind)
57// In IR, a chain with two (or more) intrinsic calls will be generated:
58// ...
59// addr = preserve_struct_access_index(base, 1, 1) !struct s
60// uint32_t result = bpf_preserve_field_info(addr, info_kind)
61//
62// Suppose the info_kind is FIELD_SIGNEDNESS,
63// The above two IR intrinsics will be replaced with
64// a relocatable insn:
65// signness = /* signness of member_access */
66// and signness can be changed by bpf loader based on the
67// types on the host.
68//
69// User can also test whether a field exists or not with
70// uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE)
71// The field will be always available (result = 1) during initial
72// compilation, but bpf loader can patch with the correct value
73// on the target host where the member_access may or may not be available
74//
75//===----------------------------------------------------------------------===//
76
77#include "BPF.h"
78#include "BPFCORE.h"
79#include "BPFTargetMachine.h"
80#include "llvm/BinaryFormat/Dwarf.h"
81#include "llvm/DebugInfo/BTF/BTF.h"
82#include "llvm/IR/DebugInfoMetadata.h"
83#include "llvm/IR/GlobalVariable.h"
84#include "llvm/IR/Instruction.h"
85#include "llvm/IR/Instructions.h"
86#include "llvm/IR/IntrinsicsBPF.h"
87#include "llvm/IR/Module.h"
88#include "llvm/IR/PassManager.h"
89#include "llvm/IR/Type.h"
90#include "llvm/IR/User.h"
91#include "llvm/IR/Value.h"
92#include "llvm/IR/ValueHandle.h"
93#include "llvm/Pass.h"
94#include "llvm/Transforms/Utils/BasicBlockUtils.h"
95#include <stack>
96
97#define DEBUG_TYPE "bpf-abstract-member-access"
98
99namespace llvm {
100constexpr StringRef BPFCoreSharedInfo::AmaAttr;
101uint32_t BPFCoreSharedInfo::SeqNum;
102
103Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB,
104 Instruction *Input,
105 Instruction *Before) {
106 Function *Fn = Intrinsic::getOrInsertDeclaration(
107 M, id: Intrinsic::bpf_passthrough, Tys: {Input->getType(), Input->getType()});
108 Constant *SeqNumVal = ConstantInt::get(Ty: Type::getInt32Ty(C&: BB->getContext()),
109 V: BPFCoreSharedInfo::SeqNum++);
110
111 auto *NewInst = CallInst::Create(Func: Fn, Args: {SeqNumVal, Input});
112 NewInst->insertBefore(InsertPos: Before->getIterator());
113 return NewInst;
114}
115} // namespace llvm
116
117using namespace llvm;
118
119namespace {
120class BPFAbstractMemberAccess final {
121public:
122 BPFAbstractMemberAccess(BPFTargetMachine *TM) : TM(TM) {}
123
124 bool run(Function &F);
125
126 struct CallInfo {
127 uint32_t Kind;
128 uint32_t AccessIndex;
129 MaybeAlign RecordAlignment;
130 MDNode *Metadata;
131 WeakTrackingVH Base;
132 };
133 typedef std::stack<std::pair<CallInst *, CallInfo>> CallInfoStack;
134
135private:
136 enum : uint32_t {
137 BPFPreserveArrayAI = 1,
138 BPFPreserveUnionAI = 2,
139 BPFPreserveStructAI = 3,
140 BPFPreserveFieldInfoAI = 4,
141 };
142
143 TargetMachine *TM;
144 const DataLayout *DL = nullptr;
145 Module *M = nullptr;
146
147 static std::map<std::string, GlobalVariable *> GEPGlobals;
148 // A map to link preserve_*_access_index intrinsic calls.
149 std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain;
150 // A map to hold all the base preserve_*_access_index intrinsic calls.
151 // The base call is not an input of any other preserve_*
152 // intrinsics.
153 std::map<CallInst *, CallInfo> BaseAICalls;
154 // A map to hold <AnonRecord, TypeDef> relationships
155 std::map<DICompositeType *, DIDerivedType *> AnonRecords;
156
157 void CheckAnonRecordType(DIDerivedType *ParentTy, DIType *Ty);
158 void CheckCompositeType(DIDerivedType *ParentTy, DICompositeType *CTy);
159 void CheckDerivedType(DIDerivedType *ParentTy, DIDerivedType *DTy);
160 void ResetMetadata(struct CallInfo &CInfo);
161
162 bool doTransformation(Function &F);
163
164 void traceAICall(CallInst *Call, CallInfo &ParentInfo);
165 void traceBitCast(BitCastInst *BitCast, CallInst *Parent,
166 CallInfo &ParentInfo);
167 void traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
168 CallInfo &ParentInfo);
169 void collectAICallChains(Function &F);
170
171 bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo);
172 bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI,
173 const MDNode *ChildMeta);
174 bool removePreserveAccessIndexIntrinsic(Function &F);
175 bool HasPreserveFieldInfoCall(CallInfoStack &CallStack);
176 void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment,
177 uint32_t &StartBitOffset, uint32_t &EndBitOffset);
178 uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy,
179 uint32_t AccessIndex, uint32_t PatchImm,
180 MaybeAlign RecordAlignment);
181
182 Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo,
183 std::string &AccessKey, MDNode *&BaseMeta);
184 MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo,
185 std::string &AccessKey, bool &IsInt32Ret);
186 bool transformGEPChain(CallInst *Call, CallInfo &CInfo);
187};
188
189std::map<std::string, GlobalVariable *> BPFAbstractMemberAccess::GEPGlobals;
190} // End anonymous namespace
191
192bool BPFAbstractMemberAccess::run(Function &F) {
193 LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n");
194
195 M = F.getParent();
196 if (!M)
197 return false;
198
199 // Bail out if no debug info.
200 if (M->debug_compile_units().empty())
201 return false;
202
203 // For each argument/return/local_variable type, trace the type
204 // pattern like '[derived_type]* [composite_type]' to check
205 // and remember (anon record -> typedef) relations where the
206 // anon record is defined as
207 // typedef [const/volatile/restrict]* [anon record]
208 DISubprogram *SP = F.getSubprogram();
209 if (SP && SP->isDefinition()) {
210 for (DIType *Ty: SP->getType()->getTypeArray())
211 CheckAnonRecordType(ParentTy: nullptr, Ty);
212 for (const DINode *DN : SP->getRetainedNodes()) {
213 if (const auto *DV = dyn_cast<DILocalVariable>(Val: DN))
214 CheckAnonRecordType(ParentTy: nullptr, Ty: DV->getType());
215 }
216 }
217
218 DL = &M->getDataLayout();
219 return doTransformation(F);
220}
221
222void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) {
223 if (auto Ty = dyn_cast<DICompositeType>(Val: CInfo.Metadata)) {
224 auto It = AnonRecords.find(x: Ty);
225 if (It != AnonRecords.end() && It->second != nullptr)
226 CInfo.Metadata = It->second;
227 }
228}
229
230void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy,
231 DICompositeType *CTy) {
232 if (!CTy->getName().empty() || !ParentTy ||
233 ParentTy->getTag() != dwarf::DW_TAG_typedef)
234 return;
235
236 auto [It, Inserted] = AnonRecords.try_emplace(k: CTy, args&: ParentTy);
237 // Two or more typedef's may point to the same anon record.
238 // If this is the case, set the typedef DIType to be nullptr
239 // to indicate the duplication case.
240 if (!Inserted && It->second != ParentTy)
241 It->second = nullptr;
242}
243
244void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy,
245 DIDerivedType *DTy) {
246 DIType *BaseType = DTy->getBaseType();
247 if (!BaseType)
248 return;
249
250 unsigned Tag = DTy->getTag();
251 if (Tag == dwarf::DW_TAG_pointer_type)
252 CheckAnonRecordType(ParentTy: nullptr, Ty: BaseType);
253 else if (Tag == dwarf::DW_TAG_typedef)
254 CheckAnonRecordType(ParentTy: DTy, Ty: BaseType);
255 else
256 CheckAnonRecordType(ParentTy, Ty: BaseType);
257}
258
259void BPFAbstractMemberAccess::CheckAnonRecordType(DIDerivedType *ParentTy,
260 DIType *Ty) {
261 if (!Ty)
262 return;
263
264 if (auto *CTy = dyn_cast<DICompositeType>(Val: Ty))
265 return CheckCompositeType(ParentTy, CTy);
266 else if (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty))
267 return CheckDerivedType(ParentTy, DTy);
268}
269
270static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) {
271 if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type &&
272 Tag != dwarf::DW_TAG_volatile_type &&
273 Tag != dwarf::DW_TAG_restrict_type &&
274 Tag != dwarf::DW_TAG_member)
275 return false;
276 if (Tag == dwarf::DW_TAG_typedef && !skipTypedef)
277 return false;
278 return true;
279}
280
281static DIType * stripQualifiers(DIType *Ty, bool skipTypedef = true) {
282 while (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty)) {
283 if (!SkipDIDerivedTag(Tag: DTy->getTag(), skipTypedef))
284 break;
285 Ty = DTy->getBaseType();
286 }
287 return Ty;
288}
289
290static const DIType * stripQualifiers(const DIType *Ty) {
291 while (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty)) {
292 if (!SkipDIDerivedTag(Tag: DTy->getTag(), skipTypedef: true))
293 break;
294 Ty = DTy->getBaseType();
295 }
296 return Ty;
297}
298
299static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim) {
300 DINodeArray Elements = CTy->getElements();
301 uint32_t DimSize = 1;
302 for (uint32_t I = StartDim; I < Elements.size(); ++I) {
303 if (auto *Element = dyn_cast_or_null<DINode>(Val: Elements[I]))
304 if (Element->getTag() == dwarf::DW_TAG_subrange_type) {
305 const DISubrange *SR = cast<DISubrange>(Val: Element);
306 auto *CI = dyn_cast<ConstantInt *>(Val: SR->getCount());
307 DimSize *= CI->getSExtValue();
308 }
309 }
310
311 return DimSize;
312}
313
314static Type *getBaseElementType(const CallInst *Call) {
315 // Element type is stored in an elementtype() attribute on the first param.
316 return Call->getParamElementType(ArgNo: 0);
317}
318
319static uint64_t getConstant(const Value *IndexValue) {
320 const ConstantInt *CV = dyn_cast<ConstantInt>(Val: IndexValue);
321 assert(CV);
322 return CV->getValue().getZExtValue();
323}
324
325/// Check whether a call is a preserve_*_access_index intrinsic call or not.
326bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,
327 CallInfo &CInfo) {
328 if (!Call)
329 return false;
330
331 const auto *GV = dyn_cast<GlobalValue>(Val: Call->getCalledOperand());
332 if (!GV)
333 return false;
334 if (GV->getName().starts_with(Prefix: "llvm.preserve.array.access.index")) {
335 CInfo.Kind = BPFPreserveArrayAI;
336 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
337 if (!CInfo.Metadata)
338 report_fatal_error(reason: "Missing metadata for llvm.preserve.array.access.index intrinsic");
339 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 2));
340 CInfo.Base = Call->getArgOperand(i: 0);
341 CInfo.RecordAlignment = DL->getABITypeAlign(Ty: getBaseElementType(Call));
342 return true;
343 }
344 if (GV->getName().starts_with(Prefix: "llvm.preserve.union.access.index")) {
345 CInfo.Kind = BPFPreserveUnionAI;
346 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
347 if (!CInfo.Metadata)
348 report_fatal_error(reason: "Missing metadata for llvm.preserve.union.access.index intrinsic");
349 ResetMetadata(CInfo);
350 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 1));
351 CInfo.Base = Call->getArgOperand(i: 0);
352 return true;
353 }
354 if (GV->getName().starts_with(Prefix: "llvm.preserve.struct.access.index")) {
355 CInfo.Kind = BPFPreserveStructAI;
356 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
357 if (!CInfo.Metadata)
358 report_fatal_error(reason: "Missing metadata for llvm.preserve.struct.access.index intrinsic");
359 ResetMetadata(CInfo);
360 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 2));
361 CInfo.Base = Call->getArgOperand(i: 0);
362 CInfo.RecordAlignment = DL->getABITypeAlign(Ty: getBaseElementType(Call));
363 return true;
364 }
365 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.field.info")) {
366 CInfo.Kind = BPFPreserveFieldInfoAI;
367 CInfo.Metadata = nullptr;
368 // Check validity of info_kind as clang did not check this.
369 uint64_t InfoKind = getConstant(IndexValue: Call->getArgOperand(i: 1));
370 if (InfoKind >= BTF::MAX_FIELD_RELOC_KIND)
371 report_fatal_error(reason: "Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic");
372 CInfo.AccessIndex = InfoKind;
373 return true;
374 }
375 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.type.info")) {
376 CInfo.Kind = BPFPreserveFieldInfoAI;
377 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
378 if (!CInfo.Metadata)
379 report_fatal_error(reason: "Missing metadata for llvm.preserve.type.info intrinsic");
380 uint64_t Flag = getConstant(IndexValue: Call->getArgOperand(i: 1));
381 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_TYPE_INFO_FLAG)
382 report_fatal_error(reason: "Incorrect flag for llvm.bpf.preserve.type.info intrinsic");
383 if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_EXISTENCE)
384 CInfo.AccessIndex = BTF::TYPE_EXISTENCE;
385 else if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_MATCH)
386 CInfo.AccessIndex = BTF::TYPE_MATCH;
387 else
388 CInfo.AccessIndex = BTF::TYPE_SIZE;
389 return true;
390 }
391 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.enum.value")) {
392 CInfo.Kind = BPFPreserveFieldInfoAI;
393 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
394 if (!CInfo.Metadata)
395 report_fatal_error(reason: "Missing metadata for llvm.preserve.enum.value intrinsic");
396 uint64_t Flag = getConstant(IndexValue: Call->getArgOperand(i: 2));
397 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_ENUM_VALUE_FLAG)
398 report_fatal_error(reason: "Incorrect flag for llvm.bpf.preserve.enum.value intrinsic");
399 if (Flag == BPFCoreSharedInfo::PRESERVE_ENUM_VALUE_EXISTENCE)
400 CInfo.AccessIndex = BTF::ENUM_VALUE_EXISTENCE;
401 else
402 CInfo.AccessIndex = BTF::ENUM_VALUE;
403 return true;
404 }
405
406 return false;
407}
408
409static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex,
410 uint32_t GEPIndex) {
411 uint32_t Dimension = 1;
412 if (DimensionIndex > 0)
413 Dimension = getConstant(IndexValue: Call->getArgOperand(i: DimensionIndex));
414
415 Constant *Zero =
416 ConstantInt::get(Ty: Type::getInt32Ty(C&: Call->getParent()->getContext()), V: 0);
417 SmallVector<Value *, 4> IdxList(Dimension, Zero);
418 IdxList.push_back(Elt: Call->getArgOperand(i: GEPIndex));
419
420 auto *GEP = GetElementPtrInst::CreateInBounds(PointeeType: getBaseElementType(Call),
421 Ptr: Call->getArgOperand(i: 0), IdxList,
422 NameStr: "", InsertBefore: Call->getIterator());
423 Call->replaceAllUsesWith(V: GEP);
424 Call->eraseFromParent();
425}
426
427void BPFCoreSharedInfo::removeArrayAccessCall(CallInst *Call) {
428 replaceWithGEP(Call, DimensionIndex: 1, GEPIndex: 2);
429}
430
431void BPFCoreSharedInfo::removeStructAccessCall(CallInst *Call) {
432 replaceWithGEP(Call, DimensionIndex: 0, GEPIndex: 1);
433}
434
435void BPFCoreSharedInfo::removeUnionAccessCall(CallInst *Call) {
436 Call->replaceAllUsesWith(V: Call->getArgOperand(i: 0));
437 Call->eraseFromParent();
438}
439
440bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) {
441 std::vector<CallInst *> PreserveArrayIndexCalls;
442 std::vector<CallInst *> PreserveUnionIndexCalls;
443 std::vector<CallInst *> PreserveStructIndexCalls;
444 bool Found = false;
445
446 for (auto &BB : F)
447 for (auto &I : BB) {
448 auto *Call = dyn_cast<CallInst>(Val: &I);
449 CallInfo CInfo;
450 if (!IsPreserveDIAccessIndexCall(Call, CInfo))
451 continue;
452
453 Found = true;
454 if (CInfo.Kind == BPFPreserveArrayAI)
455 PreserveArrayIndexCalls.push_back(x: Call);
456 else if (CInfo.Kind == BPFPreserveUnionAI)
457 PreserveUnionIndexCalls.push_back(x: Call);
458 else
459 PreserveStructIndexCalls.push_back(x: Call);
460 }
461
462 // do the following transformation:
463 // . addr = preserve_array_access_index(base, dimension, index)
464 // is transformed to
465 // addr = GEP(base, dimenion's zero's, index)
466 // . addr = preserve_union_access_index(base, di_index)
467 // is transformed to
468 // addr = base, i.e., all usages of "addr" are replaced by "base".
469 // . addr = preserve_struct_access_index(base, gep_index, di_index)
470 // is transformed to
471 // addr = GEP(base, 0, gep_index)
472 for (CallInst *Call : PreserveArrayIndexCalls)
473 BPFCoreSharedInfo::removeArrayAccessCall(Call);
474 for (CallInst *Call : PreserveStructIndexCalls)
475 BPFCoreSharedInfo::removeStructAccessCall(Call);
476 for (CallInst *Call : PreserveUnionIndexCalls)
477 BPFCoreSharedInfo::removeUnionAccessCall(Call);
478
479 return Found;
480}
481
482/// Check whether the access index chain is valid. We check
483/// here because there may be type casts between two
484/// access indexes. We want to ensure memory access still valid.
485bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType,
486 uint32_t ParentAI,
487 const MDNode *ChildType) {
488 if (!ChildType)
489 return true; // preserve_field_info, no type comparison needed.
490
491 const DIType *PType = stripQualifiers(Ty: cast<DIType>(Val: ParentType));
492 const DIType *CType = stripQualifiers(Ty: cast<DIType>(Val: ChildType));
493
494 // Child is a derived/pointer type, which is due to type casting.
495 // Pointer type cannot be in the middle of chain.
496 if (isa<DIDerivedType>(Val: CType))
497 return false;
498
499 // Parent is a pointer type.
500 if (const auto *PtrTy = dyn_cast<DIDerivedType>(Val: PType)) {
501 if (PtrTy->getTag() != dwarf::DW_TAG_pointer_type)
502 return false;
503 return stripQualifiers(Ty: PtrTy->getBaseType()) == CType;
504 }
505
506 // Otherwise, struct/union/array types
507 const auto *PTy = dyn_cast<DICompositeType>(Val: PType);
508 const auto *CTy = dyn_cast<DICompositeType>(Val: CType);
509 assert(PTy && CTy && "ParentType or ChildType is null or not composite");
510
511 uint32_t PTyTag = PTy->getTag();
512 assert(PTyTag == dwarf::DW_TAG_array_type ||
513 PTyTag == dwarf::DW_TAG_structure_type ||
514 PTyTag == dwarf::DW_TAG_union_type);
515
516 uint32_t CTyTag = CTy->getTag();
517 assert(CTyTag == dwarf::DW_TAG_array_type ||
518 CTyTag == dwarf::DW_TAG_structure_type ||
519 CTyTag == dwarf::DW_TAG_union_type);
520
521 // Multi dimensional arrays, base element should be the same
522 if (PTyTag == dwarf::DW_TAG_array_type && PTyTag == CTyTag)
523 return PTy->getBaseType() == CTy->getBaseType();
524
525 DIType *Ty;
526 if (PTyTag == dwarf::DW_TAG_array_type)
527 Ty = PTy->getBaseType();
528 else
529 Ty = dyn_cast<DIType>(Val: PTy->getElements()[ParentAI]);
530
531 return dyn_cast<DICompositeType>(Val: stripQualifiers(Ty)) == CTy;
532}
533
534void BPFAbstractMemberAccess::traceAICall(CallInst *Call,
535 CallInfo &ParentInfo) {
536 for (User *U : Call->users()) {
537 Instruction *Inst = dyn_cast<Instruction>(Val: U);
538 if (!Inst)
539 continue;
540
541 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
542 traceBitCast(BitCast: BI, Parent: Call, ParentInfo);
543 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
544 CallInfo ChildInfo;
545
546 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
547 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
548 ChildType: ChildInfo.Metadata)) {
549 AIChain[CI] = std::make_pair(x&: Call, y&: ParentInfo);
550 traceAICall(Call: CI, ParentInfo&: ChildInfo);
551 } else {
552 BaseAICalls[Call] = ParentInfo;
553 }
554 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
555 if (GI->hasAllZeroIndices())
556 traceGEP(GEP: GI, Parent: Call, ParentInfo);
557 else
558 BaseAICalls[Call] = ParentInfo;
559 } else {
560 BaseAICalls[Call] = ParentInfo;
561 }
562 }
563}
564
565void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast,
566 CallInst *Parent,
567 CallInfo &ParentInfo) {
568 for (User *U : BitCast->users()) {
569 Instruction *Inst = dyn_cast<Instruction>(Val: U);
570 if (!Inst)
571 continue;
572
573 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
574 traceBitCast(BitCast: BI, Parent, ParentInfo);
575 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
576 CallInfo ChildInfo;
577 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
578 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
579 ChildType: ChildInfo.Metadata)) {
580 AIChain[CI] = std::make_pair(x&: Parent, y&: ParentInfo);
581 traceAICall(Call: CI, ParentInfo&: ChildInfo);
582 } else {
583 BaseAICalls[Parent] = ParentInfo;
584 }
585 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
586 if (GI->hasAllZeroIndices())
587 traceGEP(GEP: GI, Parent, ParentInfo);
588 else
589 BaseAICalls[Parent] = ParentInfo;
590 } else {
591 BaseAICalls[Parent] = ParentInfo;
592 }
593 }
594}
595
596void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
597 CallInfo &ParentInfo) {
598 for (User *U : GEP->users()) {
599 Instruction *Inst = dyn_cast<Instruction>(Val: U);
600 if (!Inst)
601 continue;
602
603 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
604 traceBitCast(BitCast: BI, Parent, ParentInfo);
605 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
606 CallInfo ChildInfo;
607 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
608 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
609 ChildType: ChildInfo.Metadata)) {
610 AIChain[CI] = std::make_pair(x&: Parent, y&: ParentInfo);
611 traceAICall(Call: CI, ParentInfo&: ChildInfo);
612 } else {
613 BaseAICalls[Parent] = ParentInfo;
614 }
615 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
616 if (GI->hasAllZeroIndices())
617 traceGEP(GEP: GI, Parent, ParentInfo);
618 else
619 BaseAICalls[Parent] = ParentInfo;
620 } else {
621 BaseAICalls[Parent] = ParentInfo;
622 }
623 }
624}
625
626void BPFAbstractMemberAccess::collectAICallChains(Function &F) {
627 AIChain.clear();
628 BaseAICalls.clear();
629
630 for (auto &BB : F)
631 for (auto &I : BB) {
632 CallInfo CInfo;
633 auto *Call = dyn_cast<CallInst>(Val: &I);
634 if (!IsPreserveDIAccessIndexCall(Call, CInfo) ||
635 AIChain.find(x: Call) != AIChain.end())
636 continue;
637
638 traceAICall(Call, ParentInfo&: CInfo);
639 }
640}
641
642/// Get the start and the end of storage offset for \p MemberTy.
643void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy,
644 Align RecordAlignment,
645 uint32_t &StartBitOffset,
646 uint32_t &EndBitOffset) {
647 uint32_t MemberBitSize = MemberTy->getSizeInBits();
648 uint32_t MemberBitOffset = MemberTy->getOffsetInBits();
649
650 if (RecordAlignment > 8) {
651 // If the Bits are within an aligned 8-byte, set the RecordAlignment
652 // to 8, other report the fatal error.
653 if (MemberBitOffset / 64 != (MemberBitOffset + MemberBitSize) / 64)
654 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
655 "requiring too big alignment");
656 RecordAlignment = Align(8);
657 }
658
659 uint32_t AlignBits = RecordAlignment.value() * 8;
660 if (MemberBitSize > AlignBits)
661 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
662 "bitfield size greater than record alignment");
663
664 StartBitOffset = MemberBitOffset & ~(AlignBits - 1);
665 if ((StartBitOffset + AlignBits) < (MemberBitOffset + MemberBitSize))
666 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
667 "cross alignment boundary");
668 EndBitOffset = StartBitOffset + AlignBits;
669}
670
671uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind,
672 DICompositeType *CTy,
673 uint32_t AccessIndex,
674 uint32_t PatchImm,
675 MaybeAlign RecordAlignment) {
676 if (InfoKind == BTF::FIELD_EXISTENCE)
677 return 1;
678
679 uint32_t Tag = CTy->getTag();
680 if (InfoKind == BTF::FIELD_BYTE_OFFSET) {
681 if (Tag == dwarf::DW_TAG_array_type) {
682 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
683 PatchImm += AccessIndex * calcArraySize(CTy, StartDim: 1) *
684 (EltTy->getSizeInBits() >> 3);
685 } else if (Tag == dwarf::DW_TAG_structure_type) {
686 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
687 if (!MemberTy->isBitField()) {
688 PatchImm += MemberTy->getOffsetInBits() >> 3;
689 } else {
690 unsigned SBitOffset, NextSBitOffset;
691 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset,
692 EndBitOffset&: NextSBitOffset);
693 PatchImm += SBitOffset >> 3;
694 }
695 }
696 return PatchImm;
697 }
698
699 if (InfoKind == BTF::FIELD_BYTE_SIZE) {
700 if (Tag == dwarf::DW_TAG_array_type) {
701 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
702 return calcArraySize(CTy, StartDim: 1) * (EltTy->getSizeInBits() >> 3);
703 } else {
704 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
705 uint32_t SizeInBits = MemberTy->getSizeInBits();
706 if (!MemberTy->isBitField())
707 return SizeInBits >> 3;
708
709 unsigned SBitOffset, NextSBitOffset;
710 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset,
711 EndBitOffset&: NextSBitOffset);
712 SizeInBits = NextSBitOffset - SBitOffset;
713 if (SizeInBits & (SizeInBits - 1))
714 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info");
715 return SizeInBits >> 3;
716 }
717 }
718
719 if (InfoKind == BTF::FIELD_SIGNEDNESS) {
720 const DIType *BaseTy;
721 if (Tag == dwarf::DW_TAG_array_type) {
722 // Signedness only checked when final array elements are accessed.
723 if (CTy->getElements().size() != 1)
724 report_fatal_error(reason: "Invalid array expression for llvm.bpf.preserve.field.info");
725 BaseTy = stripQualifiers(Ty: CTy->getBaseType());
726 } else {
727 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
728 BaseTy = stripQualifiers(Ty: MemberTy->getBaseType());
729 }
730
731 // Only basic types and enum types have signedness.
732 const auto *BTy = dyn_cast<DIBasicType>(Val: BaseTy);
733 while (!BTy) {
734 const auto *CompTy = dyn_cast<DICompositeType>(Val: BaseTy);
735 // Report an error if the field expression does not have signedness.
736 if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type)
737 report_fatal_error(reason: "Invalid field expression for llvm.bpf.preserve.field.info");
738 BaseTy = stripQualifiers(Ty: CompTy->getBaseType());
739 BTy = dyn_cast<DIBasicType>(Val: BaseTy);
740 }
741 uint32_t Encoding = BTy->getEncoding();
742 return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char);
743 }
744
745 if (InfoKind == BTF::FIELD_LSHIFT_U64) {
746 // The value is loaded into a value with FIELD_BYTE_SIZE size,
747 // and then zero or sign extended to U64.
748 // FIELD_LSHIFT_U64 and FIELD_RSHIFT_U64 are operations
749 // to extract the original value.
750 const Triple &Triple = TM->getTargetTriple();
751 DIDerivedType *MemberTy = nullptr;
752 bool IsBitField = false;
753 uint32_t SizeInBits;
754
755 if (Tag == dwarf::DW_TAG_array_type) {
756 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
757 SizeInBits = calcArraySize(CTy, StartDim: 1) * EltTy->getSizeInBits();
758 } else {
759 MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
760 SizeInBits = MemberTy->getSizeInBits();
761 IsBitField = MemberTy->isBitField();
762 }
763
764 if (!IsBitField) {
765 if (SizeInBits > 64)
766 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
767 return 64 - SizeInBits;
768 }
769
770 unsigned SBitOffset, NextSBitOffset;
771 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset, EndBitOffset&: NextSBitOffset);
772 if (NextSBitOffset - SBitOffset > 64)
773 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
774
775 unsigned OffsetInBits = MemberTy->getOffsetInBits();
776 if (Triple.getArch() == Triple::bpfel)
777 return SBitOffset + 64 - OffsetInBits - SizeInBits;
778 else
779 return OffsetInBits + 64 - NextSBitOffset;
780 }
781
782 if (InfoKind == BTF::FIELD_RSHIFT_U64) {
783 DIDerivedType *MemberTy = nullptr;
784 bool IsBitField = false;
785 uint32_t SizeInBits;
786 if (Tag == dwarf::DW_TAG_array_type) {
787 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
788 SizeInBits = calcArraySize(CTy, StartDim: 1) * EltTy->getSizeInBits();
789 } else {
790 MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
791 SizeInBits = MemberTy->getSizeInBits();
792 IsBitField = MemberTy->isBitField();
793 }
794
795 if (!IsBitField) {
796 if (SizeInBits > 64)
797 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
798 return 64 - SizeInBits;
799 }
800
801 unsigned SBitOffset, NextSBitOffset;
802 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset, EndBitOffset&: NextSBitOffset);
803 if (NextSBitOffset - SBitOffset > 64)
804 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
805
806 return 64 - SizeInBits;
807 }
808
809 llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind");
810}
811
812bool BPFAbstractMemberAccess::HasPreserveFieldInfoCall(CallInfoStack &CallStack) {
813 // This is called in error return path, no need to maintain CallStack.
814 while (CallStack.size()) {
815 auto StackElem = CallStack.top();
816 if (StackElem.second.Kind == BPFPreserveFieldInfoAI)
817 return true;
818 CallStack.pop();
819 }
820 return false;
821}
822
823/// Compute the base of the whole preserve_* intrinsics chains, i.e., the base
824/// pointer of the first preserve_*_access_index call, and construct the access
825/// string, which will be the name of a global variable.
826Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call,
827 CallInfo &CInfo,
828 std::string &AccessKey,
829 MDNode *&TypeMeta) {
830 Value *Base = nullptr;
831 std::string TypeName;
832 CallInfoStack CallStack;
833
834 // Put the access chain into a stack with the top as the head of the chain.
835 while (Call) {
836 CallStack.push(x: std::make_pair(x&: Call, y&: CInfo));
837 auto &Chain = AIChain[Call];
838 CInfo = Chain.second;
839 Call = Chain.first;
840 }
841
842 // The access offset from the base of the head of chain is also
843 // calculated here as all debuginfo types are available.
844
845 // Get type name and calculate the first index.
846 // We only want to get type name from typedef, structure or union.
847 // If user wants a relocation like
848 // int *p; ... __builtin_preserve_access_index(&p[4]) ...
849 // or
850 // int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ...
851 // we will skip them.
852 uint32_t FirstIndex = 0;
853 uint32_t PatchImm = 0; // AccessOffset or the requested field info
854 uint32_t InfoKind = BTF::FIELD_BYTE_OFFSET;
855 while (CallStack.size()) {
856 auto StackElem = CallStack.top();
857 Call = StackElem.first;
858 CInfo = StackElem.second;
859
860 if (!Base)
861 Base = CInfo.Base;
862
863 DIType *PossibleTypeDef = stripQualifiers(Ty: cast<DIType>(Val: CInfo.Metadata),
864 skipTypedef: false);
865 DIType *Ty = stripQualifiers(Ty: PossibleTypeDef);
866 if (CInfo.Kind == BPFPreserveUnionAI ||
867 CInfo.Kind == BPFPreserveStructAI) {
868 // struct or union type. If the typedef is in the metadata, always
869 // use the typedef.
870 TypeName = std::string(PossibleTypeDef->getName());
871 TypeMeta = PossibleTypeDef;
872 PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3);
873 break;
874 }
875
876 assert(CInfo.Kind == BPFPreserveArrayAI);
877
878 // Array entries will always be consumed for accumulative initial index.
879 CallStack.pop();
880
881 // BPFPreserveArrayAI
882 uint64_t AccessIndex = CInfo.AccessIndex;
883
884 DIType *BaseTy = nullptr;
885 bool CheckElemType = false;
886 if (const auto *CTy = dyn_cast<DICompositeType>(Val: Ty)) {
887 // array type
888 assert(CTy->getTag() == dwarf::DW_TAG_array_type);
889
890
891 FirstIndex += AccessIndex * calcArraySize(CTy, StartDim: 1);
892 BaseTy = stripQualifiers(Ty: CTy->getBaseType());
893 CheckElemType = CTy->getElements().size() == 1;
894 } else {
895 // pointer type
896 auto *DTy = cast<DIDerivedType>(Val: Ty);
897 assert(DTy->getTag() == dwarf::DW_TAG_pointer_type);
898
899 BaseTy = stripQualifiers(Ty: DTy->getBaseType());
900 CTy = dyn_cast<DICompositeType>(Val: BaseTy);
901 if (!CTy) {
902 CheckElemType = true;
903 } else if (CTy->getTag() != dwarf::DW_TAG_array_type) {
904 FirstIndex += AccessIndex;
905 CheckElemType = true;
906 } else {
907 FirstIndex += AccessIndex * calcArraySize(CTy, StartDim: 0);
908 }
909 }
910
911 if (CheckElemType) {
912 auto *CTy = dyn_cast<DICompositeType>(Val: BaseTy);
913 if (!CTy) {
914 if (HasPreserveFieldInfoCall(CallStack))
915 report_fatal_error(reason: "Invalid field access for llvm.preserve.field.info intrinsic");
916 return nullptr;
917 }
918
919 unsigned CTag = CTy->getTag();
920 if (CTag == dwarf::DW_TAG_structure_type || CTag == dwarf::DW_TAG_union_type) {
921 TypeName = std::string(CTy->getName());
922 } else {
923 if (HasPreserveFieldInfoCall(CallStack))
924 report_fatal_error(reason: "Invalid field access for llvm.preserve.field.info intrinsic");
925 return nullptr;
926 }
927 TypeMeta = CTy;
928 PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3);
929 break;
930 }
931 }
932 assert(TypeName.size());
933 AccessKey += std::to_string(val: FirstIndex);
934
935 // Traverse the rest of access chain to complete offset calculation
936 // and access key construction.
937 while (CallStack.size()) {
938 auto StackElem = CallStack.top();
939 CInfo = StackElem.second;
940 CallStack.pop();
941
942 if (CInfo.Kind == BPFPreserveFieldInfoAI) {
943 InfoKind = CInfo.AccessIndex;
944 if (InfoKind == BTF::FIELD_EXISTENCE)
945 PatchImm = 1;
946 break;
947 }
948
949 // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI,
950 // the action will be extracting field info.
951 if (CallStack.size()) {
952 auto StackElem2 = CallStack.top();
953 CallInfo CInfo2 = StackElem2.second;
954 if (CInfo2.Kind == BPFPreserveFieldInfoAI) {
955 InfoKind = CInfo2.AccessIndex;
956 assert(CallStack.size() == 1);
957 }
958 }
959
960 // Access Index
961 uint64_t AccessIndex = CInfo.AccessIndex;
962 AccessKey += ":" + std::to_string(val: AccessIndex);
963
964 MDNode *MDN = CInfo.Metadata;
965 // At this stage, it cannot be pointer type.
966 auto *CTy = cast<DICompositeType>(Val: stripQualifiers(Ty: cast<DIType>(Val: MDN)));
967 PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex, PatchImm,
968 RecordAlignment: CInfo.RecordAlignment);
969 }
970
971 // Access key is the
972 // "llvm." + type name + ":" + reloc type + ":" + patched imm + "$" +
973 // access string,
974 // uniquely identifying one relocation.
975 // The prefix "llvm." indicates this is a temporary global, which should
976 // not be emitted to ELF file.
977 AccessKey = "llvm." + TypeName + ":" + std::to_string(val: InfoKind) + ":" +
978 std::to_string(val: PatchImm) + "$" + AccessKey;
979
980 return Base;
981}
982
983MDNode *BPFAbstractMemberAccess::computeAccessKey(CallInst *Call,
984 CallInfo &CInfo,
985 std::string &AccessKey,
986 bool &IsInt32Ret) {
987 DIType *Ty = stripQualifiers(Ty: cast<DIType>(Val: CInfo.Metadata), skipTypedef: false);
988 assert(!Ty->getName().empty());
989
990 int64_t PatchImm;
991 std::string AccessStr("0");
992 if (CInfo.AccessIndex == BTF::TYPE_EXISTENCE ||
993 CInfo.AccessIndex == BTF::TYPE_MATCH) {
994 PatchImm = 1;
995 } else if (CInfo.AccessIndex == BTF::TYPE_SIZE) {
996 // typedef debuginfo type has size 0, get the eventual base type.
997 DIType *BaseTy = stripQualifiers(Ty, skipTypedef: true);
998 PatchImm = BaseTy->getSizeInBits() / 8;
999 } else {
1000 // ENUM_VALUE_EXISTENCE and ENUM_VALUE
1001 IsInt32Ret = false;
1002
1003 // The argument could be a global variable or a getelementptr with base to
1004 // a global variable depending on whether the clang option `opaque-options`
1005 // is set or not.
1006 const GlobalVariable *GV =
1007 cast<GlobalVariable>(Val: Call->getArgOperand(i: 1)->stripPointerCasts());
1008 assert(GV->hasInitializer());
1009 const ConstantDataArray *DA = cast<ConstantDataArray>(Val: GV->getInitializer());
1010 assert(DA->isString());
1011 StringRef ValueStr = DA->getAsString();
1012
1013 // ValueStr format: <EnumeratorStr>:<Value>
1014 size_t Separator = ValueStr.find_first_of(C: ':');
1015 StringRef EnumeratorStr = ValueStr.substr(Start: 0, N: Separator);
1016
1017 // Find enumerator index in the debuginfo
1018 DIType *BaseTy = stripQualifiers(Ty, skipTypedef: true);
1019 const auto *CTy = cast<DICompositeType>(Val: BaseTy);
1020 assert(CTy->getTag() == dwarf::DW_TAG_enumeration_type);
1021 int EnumIndex = 0;
1022 for (const auto Element : CTy->getElements()) {
1023 const auto *Enum = cast<DIEnumerator>(Val: Element);
1024 if (Enum->getName() == EnumeratorStr) {
1025 AccessStr = std::to_string(val: EnumIndex);
1026 break;
1027 }
1028 EnumIndex++;
1029 }
1030
1031 if (CInfo.AccessIndex == BTF::ENUM_VALUE) {
1032 StringRef EValueStr = ValueStr.substr(Start: Separator + 1);
1033 PatchImm = std::stoll(str: std::string(EValueStr));
1034 } else {
1035 PatchImm = 1;
1036 }
1037 }
1038
1039 AccessKey = "llvm." + Ty->getName().str() + ":" +
1040 std::to_string(val: CInfo.AccessIndex) + std::string(":") +
1041 std::to_string(val: PatchImm) + std::string("$") + AccessStr;
1042
1043 return Ty;
1044}
1045
1046/// Call/Kind is the base preserve_*_access_index() call. Attempts to do
1047/// transformation to a chain of relocable GEPs.
1048bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call,
1049 CallInfo &CInfo) {
1050 std::string AccessKey;
1051 MDNode *TypeMeta;
1052 Value *Base = nullptr;
1053 bool IsInt32Ret;
1054
1055 IsInt32Ret = CInfo.Kind == BPFPreserveFieldInfoAI;
1056 if (CInfo.Kind == BPFPreserveFieldInfoAI && CInfo.Metadata) {
1057 TypeMeta = computeAccessKey(Call, CInfo, AccessKey, IsInt32Ret);
1058 } else {
1059 Base = computeBaseAndAccessKey(Call, CInfo, AccessKey, TypeMeta);
1060 if (!Base)
1061 return false;
1062 }
1063
1064 BasicBlock *BB = Call->getParent();
1065 GlobalVariable *GV;
1066
1067 if (GEPGlobals.find(x: AccessKey) == GEPGlobals.end()) {
1068 IntegerType *VarType;
1069 if (IsInt32Ret)
1070 VarType = Type::getInt32Ty(C&: BB->getContext()); // 32bit return value
1071 else
1072 VarType = Type::getInt64Ty(C&: BB->getContext()); // 64bit ptr or enum value
1073
1074 GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage,
1075 nullptr, AccessKey);
1076 GV->addAttribute(Kind: BPFCoreSharedInfo::AmaAttr);
1077 GV->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: TypeMeta);
1078 GEPGlobals[AccessKey] = GV;
1079 } else {
1080 GV = GEPGlobals[AccessKey];
1081 }
1082
1083 if (CInfo.Kind == BPFPreserveFieldInfoAI) {
1084 // Load the global variable which represents the returned field info.
1085 LoadInst *LDInst;
1086 if (IsInt32Ret)
1087 LDInst = new LoadInst(Type::getInt32Ty(C&: BB->getContext()), GV, "",
1088 Call->getIterator());
1089 else
1090 LDInst = new LoadInst(Type::getInt64Ty(C&: BB->getContext()), GV, "",
1091 Call->getIterator());
1092
1093 Instruction *PassThroughInst =
1094 BPFCoreSharedInfo::insertPassThrough(M, BB, Input: LDInst, Before: Call);
1095 Call->replaceAllUsesWith(V: PassThroughInst);
1096 Call->eraseFromParent();
1097 return true;
1098 }
1099
1100 // For any original GEP Call and Base %2 like
1101 // %4 = bitcast %struct.net_device** %dev1 to i64*
1102 // it is transformed to:
1103 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0
1104 // %8 = getelementptr i8, i8* %2, %6
1105 // using %8 instead of %4
1106 // The original Call inst is removed.
1107
1108 // Load the global variable.
1109 auto *LDInst = new LoadInst(Type::getInt64Ty(C&: BB->getContext()), GV, "",
1110 Call->getIterator());
1111
1112 // Generate a GetElementPtr
1113 auto *GEP = GetElementPtrInst::Create(PointeeType: Type::getInt8Ty(C&: BB->getContext()), Ptr: Base,
1114 IdxList: LDInst);
1115 GEP->insertBefore(InsertPos: Call->getIterator());
1116
1117 // For the following code,
1118 // Block0:
1119 // ...
1120 // if (...) goto Block1 else ...
1121 // Block1:
1122 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0
1123 // %8 = getelementptr i8, i8* %2, %6
1124 // ...
1125 // goto CommonExit
1126 // Block2:
1127 // ...
1128 // if (...) goto Block3 else ...
1129 // Block3:
1130 // %6 = load llvm.bpf_map:0:40$0:0:0:2:0
1131 // %8 = getelementptr i8, i8* %2, %6
1132 // ...
1133 // goto CommonExit
1134 // CommonExit
1135 // SimplifyCFG may generate:
1136 // Block0:
1137 // ...
1138 // if (...) goto Block_Common else ...
1139 // Block2:
1140 // ...
1141 // if (...) goto Block_Common else ...
1142 // Block_Common:
1143 // PHI = [llvm.sk_buff:0:50$0:0:0:2:0, llvm.bpf_map:0:40$0:0:0:2:0]
1144 // %6 = load PHI
1145 // %8 = getelementptr i8, i8* %2, %6
1146 // ...
1147 // goto CommonExit
1148 // For the above code, we cannot perform proper relocation since
1149 // "load PHI" has two possible relocations.
1150 //
1151 // To prevent above tail merging, we use __builtin_bpf_passthrough()
1152 // where one of its parameters is a seq_num. Since two
1153 // __builtin_bpf_passthrough() funcs will always have different seq_num,
1154 // tail merging cannot happen. The __builtin_bpf_passthrough() will be
1155 // removed in the beginning of Target IR passes.
1156 //
1157 // This approach is also used in other places when global var
1158 // representing a relocation is used.
1159 Instruction *PassThroughInst =
1160 BPFCoreSharedInfo::insertPassThrough(M, BB, Input: GEP, Before: Call);
1161 Call->replaceAllUsesWith(V: PassThroughInst);
1162 Call->eraseFromParent();
1163
1164 return true;
1165}
1166
1167bool BPFAbstractMemberAccess::doTransformation(Function &F) {
1168 bool Transformed = false;
1169
1170 // Collect PreserveDIAccessIndex Intrinsic call chains.
1171 // The call chains will be used to generate the access
1172 // patterns similar to GEP.
1173 collectAICallChains(F);
1174
1175 for (auto &C : BaseAICalls)
1176 Transformed = transformGEPChain(Call: C.first, CInfo&: C.second) || Transformed;
1177
1178 return removePreserveAccessIndexIntrinsic(F) || Transformed;
1179}
1180
1181PreservedAnalyses
1182BPFAbstractMemberAccessPass::run(Function &F, FunctionAnalysisManager &AM) {
1183 return BPFAbstractMemberAccess(TM).run(F) ? PreservedAnalyses::none()
1184 : PreservedAnalyses::all();
1185}
1186