1//===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass abstracted struct/union member accesses in order to support
10// compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program
11// which can run on different kernels. In particular, if bpf program tries to
12// access a particular kernel data structure member, the details of the
13// intermediate member access will be remembered so bpf loader can do
14// necessary adjustment right before program loading.
15//
16// For example,
17//
18// struct s {
19// int a;
20// int b;
21// };
22// struct t {
23// struct s c;
24// int d;
25// };
26// struct t e;
27//
28// For the member access e.c.b, the compiler will generate code
29// &e + 4
30//
31// The compile-once run-everywhere instead generates the following code
32// r = 4
33// &e + r
34// The "4" in "r = 4" can be changed based on a particular kernel version.
35// For example, on a particular kernel version, if struct s is changed to
36//
37// struct s {
38// int new_field;
39// int a;
40// int b;
41// }
42//
43// By repeating the member access on the host, the bpf loader can
44// adjust "r = 4" as "r = 8".
45//
46// This feature relies on the following three intrinsic calls:
47// addr = preserve_array_access_index(base, dimension, index)
48// addr = preserve_union_access_index(base, di_index)
49// !llvm.preserve.access.index <union_ditype>
50// addr = preserve_struct_access_index(base, gep_index, di_index)
51// !llvm.preserve.access.index <struct_ditype>
52//
53// Bitfield member access needs special attention. User cannot take the
54// address of a bitfield acceess. To facilitate kernel verifier
55// for easy bitfield code optimization, a new clang intrinsic is introduced:
56// uint32_t __builtin_preserve_field_info(member_access, info_kind)
57// In IR, a chain with two (or more) intrinsic calls will be generated:
58// ...
59// addr = preserve_struct_access_index(base, 1, 1) !struct s
60// uint32_t result = bpf_preserve_field_info(addr, info_kind)
61//
62// Suppose the info_kind is FIELD_SIGNEDNESS,
63// The above two IR intrinsics will be replaced with
64// a relocatable insn:
65// signness = /* signness of member_access */
66// and signness can be changed by bpf loader based on the
67// types on the host.
68//
69// User can also test whether a field exists or not with
70// uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE)
71// The field will be always available (result = 1) during initial
72// compilation, but bpf loader can patch with the correct value
73// on the target host where the member_access may or may not be available
74//
75//===----------------------------------------------------------------------===//
76
77#include "BPF.h"
78#include "BPFCORE.h"
79#include "BPFTargetMachine.h"
80#include "llvm/ADT/MapVector.h"
81#include "llvm/ADT/SmallVector.h"
82#include "llvm/BinaryFormat/Dwarf.h"
83#include "llvm/DebugInfo/BTF/BTF.h"
84#include "llvm/IR/DebugInfoMetadata.h"
85#include "llvm/IR/GlobalVariable.h"
86#include "llvm/IR/Instruction.h"
87#include "llvm/IR/Instructions.h"
88#include "llvm/IR/IntrinsicsBPF.h"
89#include "llvm/IR/Module.h"
90#include "llvm/IR/PassManager.h"
91#include "llvm/IR/Type.h"
92#include "llvm/IR/User.h"
93#include "llvm/IR/Value.h"
94#include "llvm/IR/ValueHandle.h"
95#include "llvm/Pass.h"
96#include "llvm/Transforms/Utils/BasicBlockUtils.h"
97#include <stack>
98
99#define DEBUG_TYPE "bpf-abstract-member-access"
100
101namespace llvm {
102uint32_t BPFCoreSharedInfo::SeqNum;
103
104Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB,
105 Instruction *Input,
106 Instruction *Before) {
107 Function *Fn = Intrinsic::getOrInsertDeclaration(
108 M, id: Intrinsic::bpf_passthrough, OverloadTys: {Input->getType(), Input->getType()});
109 Constant *SeqNumVal = ConstantInt::get(Ty: Type::getInt32Ty(C&: BB->getContext()),
110 V: BPFCoreSharedInfo::SeqNum++);
111
112 auto *NewInst = CallInst::Create(Func: Fn, Args: {SeqNumVal, Input});
113 NewInst->insertBefore(InsertPos: Before->getIterator());
114 return NewInst;
115}
116} // namespace llvm
117
118using namespace llvm;
119
120namespace {
121class BPFAbstractMemberAccess final {
122public:
123 BPFAbstractMemberAccess(BPFTargetMachine *TM) : TM(TM) {}
124
125 bool run(Function &F);
126
127 struct CallInfo {
128 uint32_t Kind;
129 uint32_t AccessIndex;
130 MaybeAlign RecordAlignment;
131 MDNode *Metadata;
132 WeakTrackingVH Base;
133 };
134 typedef std::stack<std::pair<CallInst *, CallInfo>> CallInfoStack;
135
136private:
137 enum : uint32_t {
138 BPFPreserveArrayAI = 1,
139 BPFPreserveUnionAI = 2,
140 BPFPreserveStructAI = 3,
141 BPFPreserveFieldInfoAI = 4,
142 };
143
144 TargetMachine *TM;
145 const DataLayout *DL = nullptr;
146 Module *M = nullptr;
147
148 static std::map<std::string, GlobalVariable *> GEPGlobals;
149 // A map to link preserve_*_access_index intrinsic calls.
150 std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain;
151 // A map to hold all the base preserve_*_access_index intrinsic calls.
152 // The base call is not an input of any other preserve_*
153 // intrinsics.
154 // Iterated below, so the order can't come from the addresses.
155 SmallMapVector<CallInst *, CallInfo, 4> BaseAICalls;
156 // A map to hold <AnonRecord, TypeDef> relationships
157 std::map<DICompositeType *, DIDerivedType *> AnonRecords;
158
159 void CheckAnonRecordType(DIDerivedType *ParentTy, DIType *Ty);
160 void CheckCompositeType(DIDerivedType *ParentTy, DICompositeType *CTy);
161 void CheckDerivedType(DIDerivedType *ParentTy, DIDerivedType *DTy);
162 void ResetMetadata(struct CallInfo &CInfo);
163
164 bool doTransformation(Function &F);
165
166 void traceAICall(CallInst *Call, CallInfo &ParentInfo);
167 void traceBitCast(BitCastInst *BitCast, CallInst *Parent,
168 CallInfo &ParentInfo);
169 void traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
170 CallInfo &ParentInfo);
171 void collectAICallChains(Function &F);
172
173 bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo);
174 bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI,
175 const MDNode *ChildMeta);
176 bool removePreserveAccessIndexIntrinsic(Function &F);
177 bool HasPreserveFieldInfoCall(CallInfoStack &CallStack);
178 void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment,
179 uint32_t &StartBitOffset, uint32_t &EndBitOffset);
180 uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy,
181 uint32_t AccessIndex, uint32_t PatchImm,
182 MaybeAlign RecordAlignment);
183
184 Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo,
185 std::string &AccessKey, MDNode *&BaseMeta);
186 MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo,
187 std::string &AccessKey, bool &IsInt32Ret);
188 bool transformGEPChain(CallInst *Call, CallInfo &CInfo);
189};
190
191std::map<std::string, GlobalVariable *> BPFAbstractMemberAccess::GEPGlobals;
192} // End anonymous namespace
193
194bool BPFAbstractMemberAccess::run(Function &F) {
195 LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n");
196
197 M = F.getParent();
198 if (!M)
199 return false;
200
201 // Bail out if no debug info.
202 if (M->debug_compile_units().empty())
203 return false;
204
205 // For each argument/return/local_variable type, trace the type
206 // pattern like '[derived_type]* [composite_type]' to check
207 // and remember (anon record -> typedef) relations where the
208 // anon record is defined as
209 // typedef [const/volatile/restrict]* [anon record]
210 DISubprogram *SP = F.getSubprogram();
211 if (SP && SP->isDefinition()) {
212 for (DIType *Ty: SP->getType()->getTypeArray())
213 CheckAnonRecordType(ParentTy: nullptr, Ty);
214 for (const MDNode *DN : SP->getRetainedNodes()) {
215 if (const auto *DV = dyn_cast<DILocalVariable>(Val: DN))
216 CheckAnonRecordType(ParentTy: nullptr, Ty: DV->getType());
217 }
218 }
219
220 DL = &M->getDataLayout();
221 return doTransformation(F);
222}
223
224void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) {
225 if (auto Ty = dyn_cast<DICompositeType>(Val: CInfo.Metadata)) {
226 auto It = AnonRecords.find(x: Ty);
227 if (It != AnonRecords.end() && It->second != nullptr)
228 CInfo.Metadata = It->second;
229 }
230}
231
232void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy,
233 DICompositeType *CTy) {
234 if (!CTy->getName().empty() || !ParentTy ||
235 ParentTy->getTag() != dwarf::DW_TAG_typedef)
236 return;
237
238 auto [It, Inserted] = AnonRecords.try_emplace(k: CTy, args&: ParentTy);
239 // Two or more typedef's may point to the same anon record.
240 // If this is the case, set the typedef DIType to be nullptr
241 // to indicate the duplication case.
242 if (!Inserted && It->second != ParentTy)
243 It->second = nullptr;
244}
245
246void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy,
247 DIDerivedType *DTy) {
248 DIType *BaseType = DTy->getBaseType();
249 if (!BaseType)
250 return;
251
252 unsigned Tag = DTy->getTag();
253 if (Tag == dwarf::DW_TAG_pointer_type)
254 CheckAnonRecordType(ParentTy: nullptr, Ty: BaseType);
255 else if (Tag == dwarf::DW_TAG_typedef)
256 CheckAnonRecordType(ParentTy: DTy, Ty: BaseType);
257 else
258 CheckAnonRecordType(ParentTy, Ty: BaseType);
259}
260
261void BPFAbstractMemberAccess::CheckAnonRecordType(DIDerivedType *ParentTy,
262 DIType *Ty) {
263 if (!Ty)
264 return;
265
266 if (auto *CTy = dyn_cast<DICompositeType>(Val: Ty))
267 return CheckCompositeType(ParentTy, CTy);
268 else if (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty))
269 return CheckDerivedType(ParentTy, DTy);
270}
271
272static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) {
273 if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type &&
274 Tag != dwarf::DW_TAG_volatile_type &&
275 Tag != dwarf::DW_TAG_restrict_type &&
276 Tag != dwarf::DW_TAG_member)
277 return false;
278 if (Tag == dwarf::DW_TAG_typedef && !skipTypedef)
279 return false;
280 return true;
281}
282
283static DIType * stripQualifiers(DIType *Ty, bool skipTypedef = true) {
284 while (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty)) {
285 if (!SkipDIDerivedTag(Tag: DTy->getTag(), skipTypedef))
286 break;
287 Ty = DTy->getBaseType();
288 }
289 return Ty;
290}
291
292static const DIType * stripQualifiers(const DIType *Ty) {
293 while (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty)) {
294 if (!SkipDIDerivedTag(Tag: DTy->getTag(), skipTypedef: true))
295 break;
296 Ty = DTy->getBaseType();
297 }
298 return Ty;
299}
300
301static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim) {
302 DINodeArray Elements = CTy->getElements();
303 uint32_t DimSize = 1;
304 for (uint32_t I = StartDim; I < Elements.size(); ++I) {
305 if (auto *Element = dyn_cast_or_null<DINode>(Val: Elements[I]))
306 if (Element->getTag() == dwarf::DW_TAG_subrange_type) {
307 const DISubrange *SR = cast<DISubrange>(Val: Element);
308 auto *CI = dyn_cast<ConstantInt *>(Val: SR->getCount());
309 DimSize *= CI->getSExtValue();
310 }
311 }
312
313 return DimSize;
314}
315
316static Type *getBaseElementType(const CallInst *Call) {
317 // Element type is stored in an elementtype() attribute on the first param.
318 return Call->getParamElementType(ArgNo: 0);
319}
320
321static uint64_t getConstant(const Value *IndexValue) {
322 const ConstantInt *CV = dyn_cast<ConstantInt>(Val: IndexValue);
323 assert(CV);
324 return CV->getValue().getZExtValue();
325}
326
327/// Check whether a call is a preserve_*_access_index intrinsic call or not.
328bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,
329 CallInfo &CInfo) {
330 if (!Call)
331 return false;
332
333 const auto *GV = dyn_cast<GlobalValue>(Val: Call->getCalledOperand());
334 if (!GV)
335 return false;
336 if (GV->getName().starts_with(Prefix: "llvm.preserve.array.access.index")) {
337 CInfo.Kind = BPFPreserveArrayAI;
338 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
339 if (!CInfo.Metadata)
340 report_fatal_error(reason: "Missing metadata for llvm.preserve.array.access.index intrinsic");
341 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 2));
342 CInfo.Base = Call->getArgOperand(i: 0);
343 CInfo.RecordAlignment = DL->getABITypeAlign(Ty: getBaseElementType(Call));
344 return true;
345 }
346 if (GV->getName().starts_with(Prefix: "llvm.preserve.union.access.index")) {
347 CInfo.Kind = BPFPreserveUnionAI;
348 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
349 if (!CInfo.Metadata)
350 report_fatal_error(reason: "Missing metadata for llvm.preserve.union.access.index intrinsic");
351 ResetMetadata(CInfo);
352 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 1));
353 CInfo.Base = Call->getArgOperand(i: 0);
354 return true;
355 }
356 if (GV->getName().starts_with(Prefix: "llvm.preserve.struct.access.index")) {
357 CInfo.Kind = BPFPreserveStructAI;
358 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
359 if (!CInfo.Metadata)
360 report_fatal_error(reason: "Missing metadata for llvm.preserve.struct.access.index intrinsic");
361 ResetMetadata(CInfo);
362 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 2));
363 CInfo.Base = Call->getArgOperand(i: 0);
364 CInfo.RecordAlignment = DL->getABITypeAlign(Ty: getBaseElementType(Call));
365 return true;
366 }
367 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.field.info")) {
368 CInfo.Kind = BPFPreserveFieldInfoAI;
369 CInfo.Metadata = nullptr;
370 // Check validity of info_kind as clang did not check this.
371 uint64_t InfoKind = getConstant(IndexValue: Call->getArgOperand(i: 1));
372 if (InfoKind >= BTF::MAX_FIELD_RELOC_KIND)
373 report_fatal_error(reason: "Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic");
374 CInfo.AccessIndex = InfoKind;
375 return true;
376 }
377 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.type.info")) {
378 CInfo.Kind = BPFPreserveFieldInfoAI;
379 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
380 if (!CInfo.Metadata)
381 report_fatal_error(reason: "Missing metadata for llvm.preserve.type.info intrinsic");
382 uint64_t Flag = getConstant(IndexValue: Call->getArgOperand(i: 1));
383 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_TYPE_INFO_FLAG)
384 report_fatal_error(reason: "Incorrect flag for llvm.bpf.preserve.type.info intrinsic");
385 if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_EXISTENCE)
386 CInfo.AccessIndex = BTF::TYPE_EXISTENCE;
387 else if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_MATCH)
388 CInfo.AccessIndex = BTF::TYPE_MATCH;
389 else
390 CInfo.AccessIndex = BTF::TYPE_SIZE;
391 return true;
392 }
393 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.enum.value")) {
394 CInfo.Kind = BPFPreserveFieldInfoAI;
395 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
396 if (!CInfo.Metadata)
397 report_fatal_error(reason: "Missing metadata for llvm.preserve.enum.value intrinsic");
398 uint64_t Flag = getConstant(IndexValue: Call->getArgOperand(i: 2));
399 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_ENUM_VALUE_FLAG)
400 report_fatal_error(reason: "Incorrect flag for llvm.bpf.preserve.enum.value intrinsic");
401 if (Flag == BPFCoreSharedInfo::PRESERVE_ENUM_VALUE_EXISTENCE)
402 CInfo.AccessIndex = BTF::ENUM_VALUE_EXISTENCE;
403 else
404 CInfo.AccessIndex = BTF::ENUM_VALUE;
405 return true;
406 }
407
408 return false;
409}
410
411static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex,
412 uint32_t GEPIndex) {
413 uint32_t Dimension = 1;
414 if (DimensionIndex > 0)
415 Dimension = getConstant(IndexValue: Call->getArgOperand(i: DimensionIndex));
416
417 Constant *Zero =
418 ConstantInt::get(Ty: Type::getInt32Ty(C&: Call->getParent()->getContext()), V: 0);
419 SmallVector<Value *, 4> IdxList(Dimension, Zero);
420 IdxList.push_back(Elt: Call->getArgOperand(i: GEPIndex));
421
422 auto *GEP = GetElementPtrInst::CreateInBounds(PointeeType: getBaseElementType(Call),
423 Ptr: Call->getArgOperand(i: 0), IdxList,
424 NameStr: "", InsertBefore: Call->getIterator());
425 Call->replaceAllUsesWith(V: GEP);
426 Call->eraseFromParent();
427}
428
429void BPFCoreSharedInfo::removeArrayAccessCall(CallInst *Call) {
430 replaceWithGEP(Call, DimensionIndex: 1, GEPIndex: 2);
431}
432
433void BPFCoreSharedInfo::removeStructAccessCall(CallInst *Call) {
434 replaceWithGEP(Call, DimensionIndex: 0, GEPIndex: 1);
435}
436
437void BPFCoreSharedInfo::removeUnionAccessCall(CallInst *Call) {
438 Call->replaceAllUsesWith(V: Call->getArgOperand(i: 0));
439 Call->eraseFromParent();
440}
441
442bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) {
443 std::vector<CallInst *> PreserveArrayIndexCalls;
444 std::vector<CallInst *> PreserveUnionIndexCalls;
445 std::vector<CallInst *> PreserveStructIndexCalls;
446 bool Found = false;
447
448 for (auto &BB : F)
449 for (auto &I : BB) {
450 auto *Call = dyn_cast<CallInst>(Val: &I);
451 CallInfo CInfo;
452 if (!IsPreserveDIAccessIndexCall(Call, CInfo))
453 continue;
454
455 Found = true;
456 if (CInfo.Kind == BPFPreserveArrayAI)
457 PreserveArrayIndexCalls.push_back(x: Call);
458 else if (CInfo.Kind == BPFPreserveUnionAI)
459 PreserveUnionIndexCalls.push_back(x: Call);
460 else
461 PreserveStructIndexCalls.push_back(x: Call);
462 }
463
464 // do the following transformation:
465 // . addr = preserve_array_access_index(base, dimension, index)
466 // is transformed to
467 // addr = GEP(base, dimenion's zero's, index)
468 // . addr = preserve_union_access_index(base, di_index)
469 // is transformed to
470 // addr = base, i.e., all usages of "addr" are replaced by "base".
471 // . addr = preserve_struct_access_index(base, gep_index, di_index)
472 // is transformed to
473 // addr = GEP(base, 0, gep_index)
474 for (CallInst *Call : PreserveArrayIndexCalls)
475 BPFCoreSharedInfo::removeArrayAccessCall(Call);
476 for (CallInst *Call : PreserveStructIndexCalls)
477 BPFCoreSharedInfo::removeStructAccessCall(Call);
478 for (CallInst *Call : PreserveUnionIndexCalls)
479 BPFCoreSharedInfo::removeUnionAccessCall(Call);
480
481 return Found;
482}
483
484/// Check whether the access index chain is valid. We check
485/// here because there may be type casts between two
486/// access indexes. We want to ensure memory access still valid.
487bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType,
488 uint32_t ParentAI,
489 const MDNode *ChildType) {
490 if (!ChildType)
491 return true; // preserve_field_info, no type comparison needed.
492
493 const DIType *PType = stripQualifiers(Ty: cast<DIType>(Val: ParentType));
494 const DIType *CType = stripQualifiers(Ty: cast<DIType>(Val: ChildType));
495
496 // Child is a derived/pointer type, which is due to type casting.
497 // Pointer type cannot be in the middle of chain.
498 if (isa<DIDerivedType>(Val: CType))
499 return false;
500
501 // Parent is a pointer type.
502 if (const auto *PtrTy = dyn_cast<DIDerivedType>(Val: PType)) {
503 if (PtrTy->getTag() != dwarf::DW_TAG_pointer_type)
504 return false;
505 return stripQualifiers(Ty: PtrTy->getBaseType()) == CType;
506 }
507
508 // Otherwise, struct/union/array types
509 const auto *PTy = dyn_cast<DICompositeType>(Val: PType);
510 const auto *CTy = dyn_cast<DICompositeType>(Val: CType);
511 assert(PTy && CTy && "ParentType or ChildType is null or not composite");
512
513 uint32_t PTyTag = PTy->getTag();
514 assert(PTyTag == dwarf::DW_TAG_array_type ||
515 PTyTag == dwarf::DW_TAG_structure_type ||
516 PTyTag == dwarf::DW_TAG_union_type);
517
518 uint32_t CTyTag = CTy->getTag();
519 assert(CTyTag == dwarf::DW_TAG_array_type ||
520 CTyTag == dwarf::DW_TAG_structure_type ||
521 CTyTag == dwarf::DW_TAG_union_type);
522
523 // Multi dimensional arrays, base element should be the same
524 if (PTyTag == dwarf::DW_TAG_array_type && PTyTag == CTyTag)
525 return PTy->getBaseType() == CTy->getBaseType();
526
527 DIType *Ty;
528 if (PTyTag == dwarf::DW_TAG_array_type)
529 Ty = PTy->getBaseType();
530 else
531 Ty = dyn_cast<DIType>(Val: PTy->getElements()[ParentAI]);
532
533 return dyn_cast<DICompositeType>(Val: stripQualifiers(Ty)) == CTy;
534}
535
536void BPFAbstractMemberAccess::traceAICall(CallInst *Call,
537 CallInfo &ParentInfo) {
538 for (User *U : Call->users()) {
539 Instruction *Inst = dyn_cast<Instruction>(Val: U);
540 if (!Inst)
541 continue;
542
543 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
544 traceBitCast(BitCast: BI, Parent: Call, ParentInfo);
545 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
546 CallInfo ChildInfo;
547
548 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
549 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
550 ChildType: ChildInfo.Metadata)) {
551 AIChain[CI] = std::make_pair(x&: Call, y&: ParentInfo);
552 traceAICall(Call: CI, ParentInfo&: ChildInfo);
553 } else {
554 BaseAICalls[Call] = ParentInfo;
555 }
556 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
557 if (GI->hasAllZeroIndices())
558 traceGEP(GEP: GI, Parent: Call, ParentInfo);
559 else
560 BaseAICalls[Call] = ParentInfo;
561 } else {
562 BaseAICalls[Call] = ParentInfo;
563 }
564 }
565}
566
567void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast,
568 CallInst *Parent,
569 CallInfo &ParentInfo) {
570 for (User *U : BitCast->users()) {
571 Instruction *Inst = dyn_cast<Instruction>(Val: U);
572 if (!Inst)
573 continue;
574
575 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
576 traceBitCast(BitCast: BI, Parent, ParentInfo);
577 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
578 CallInfo ChildInfo;
579 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
580 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
581 ChildType: ChildInfo.Metadata)) {
582 AIChain[CI] = std::make_pair(x&: Parent, y&: ParentInfo);
583 traceAICall(Call: CI, ParentInfo&: ChildInfo);
584 } else {
585 BaseAICalls[Parent] = ParentInfo;
586 }
587 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
588 if (GI->hasAllZeroIndices())
589 traceGEP(GEP: GI, Parent, ParentInfo);
590 else
591 BaseAICalls[Parent] = ParentInfo;
592 } else {
593 BaseAICalls[Parent] = ParentInfo;
594 }
595 }
596}
597
598void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
599 CallInfo &ParentInfo) {
600 for (User *U : GEP->users()) {
601 Instruction *Inst = dyn_cast<Instruction>(Val: U);
602 if (!Inst)
603 continue;
604
605 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
606 traceBitCast(BitCast: BI, Parent, ParentInfo);
607 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
608 CallInfo ChildInfo;
609 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
610 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
611 ChildType: ChildInfo.Metadata)) {
612 AIChain[CI] = std::make_pair(x&: Parent, y&: ParentInfo);
613 traceAICall(Call: CI, ParentInfo&: ChildInfo);
614 } else {
615 BaseAICalls[Parent] = ParentInfo;
616 }
617 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
618 if (GI->hasAllZeroIndices())
619 traceGEP(GEP: GI, Parent, ParentInfo);
620 else
621 BaseAICalls[Parent] = ParentInfo;
622 } else {
623 BaseAICalls[Parent] = ParentInfo;
624 }
625 }
626}
627
628void BPFAbstractMemberAccess::collectAICallChains(Function &F) {
629 AIChain.clear();
630 BaseAICalls.clear();
631
632 for (auto &BB : F)
633 for (auto &I : BB) {
634 CallInfo CInfo;
635 auto *Call = dyn_cast<CallInst>(Val: &I);
636 if (!IsPreserveDIAccessIndexCall(Call, CInfo) ||
637 AIChain.find(x: Call) != AIChain.end())
638 continue;
639
640 traceAICall(Call, ParentInfo&: CInfo);
641 }
642}
643
644/// Get the start and the end of storage offset for \p MemberTy.
645void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy,
646 Align RecordAlignment,
647 uint32_t &StartBitOffset,
648 uint32_t &EndBitOffset) {
649 uint32_t MemberBitSize = MemberTy->getSizeInBits();
650 uint32_t MemberBitOffset = MemberTy->getOffsetInBits();
651
652 if (RecordAlignment > 8) {
653 // If the Bits are within an aligned 8-byte, set the RecordAlignment
654 // to 8, other report the fatal error.
655 if (MemberBitOffset / 64 != (MemberBitOffset + MemberBitSize) / 64)
656 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
657 "requiring too big alignment");
658 RecordAlignment = Align(8);
659 }
660
661 uint32_t AlignBits = RecordAlignment.value() * 8;
662 if (MemberBitSize > AlignBits)
663 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
664 "bitfield size greater than record alignment");
665
666 StartBitOffset = MemberBitOffset & ~(AlignBits - 1);
667 if ((StartBitOffset + AlignBits) < (MemberBitOffset + MemberBitSize))
668 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
669 "cross alignment boundary");
670 EndBitOffset = StartBitOffset + AlignBits;
671}
672
673uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind,
674 DICompositeType *CTy,
675 uint32_t AccessIndex,
676 uint32_t PatchImm,
677 MaybeAlign RecordAlignment) {
678 if (InfoKind == BTF::FIELD_EXISTENCE)
679 return 1;
680
681 uint32_t Tag = CTy->getTag();
682 if (InfoKind == BTF::FIELD_BYTE_OFFSET) {
683 if (Tag == dwarf::DW_TAG_array_type) {
684 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
685 PatchImm += AccessIndex * calcArraySize(CTy, StartDim: 1) *
686 (EltTy->getSizeInBits() >> 3);
687 } else if (Tag == dwarf::DW_TAG_structure_type) {
688 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
689 if (!MemberTy->isBitField()) {
690 PatchImm += MemberTy->getOffsetInBits() >> 3;
691 } else {
692 unsigned SBitOffset, NextSBitOffset;
693 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset,
694 EndBitOffset&: NextSBitOffset);
695 PatchImm += SBitOffset >> 3;
696 }
697 }
698 return PatchImm;
699 }
700
701 if (InfoKind == BTF::FIELD_BYTE_SIZE) {
702 if (Tag == dwarf::DW_TAG_array_type) {
703 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
704 return calcArraySize(CTy, StartDim: 1) * (EltTy->getSizeInBits() >> 3);
705 } else {
706 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
707 uint32_t SizeInBits = MemberTy->getSizeInBits();
708 if (!MemberTy->isBitField())
709 return SizeInBits >> 3;
710
711 unsigned SBitOffset, NextSBitOffset;
712 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset,
713 EndBitOffset&: NextSBitOffset);
714 SizeInBits = NextSBitOffset - SBitOffset;
715 if (SizeInBits & (SizeInBits - 1))
716 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info");
717 return SizeInBits >> 3;
718 }
719 }
720
721 if (InfoKind == BTF::FIELD_SIGNEDNESS) {
722 const DIType *BaseTy;
723 if (Tag == dwarf::DW_TAG_array_type) {
724 // Signedness only checked when final array elements are accessed.
725 if (CTy->getElements().size() != 1)
726 report_fatal_error(reason: "Invalid array expression for llvm.bpf.preserve.field.info");
727 BaseTy = stripQualifiers(Ty: CTy->getBaseType());
728 } else {
729 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
730 BaseTy = stripQualifiers(Ty: MemberTy->getBaseType());
731 }
732
733 // Only basic types and enum types have signedness.
734 const auto *BTy = dyn_cast<DIBasicType>(Val: BaseTy);
735 while (!BTy) {
736 const auto *CompTy = dyn_cast<DICompositeType>(Val: BaseTy);
737 // Report an error if the field expression does not have signedness.
738 if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type)
739 report_fatal_error(reason: "Invalid field expression for llvm.bpf.preserve.field.info");
740 BaseTy = stripQualifiers(Ty: CompTy->getBaseType());
741 BTy = dyn_cast<DIBasicType>(Val: BaseTy);
742 }
743 uint32_t Encoding = BTy->getEncoding();
744 return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char);
745 }
746
747 if (InfoKind == BTF::FIELD_LSHIFT_U64) {
748 // The value is loaded into a value with FIELD_BYTE_SIZE size,
749 // and then zero or sign extended to U64.
750 // FIELD_LSHIFT_U64 and FIELD_RSHIFT_U64 are operations
751 // to extract the original value.
752 const Triple &Triple = TM->getTargetTriple();
753 DIDerivedType *MemberTy = nullptr;
754 bool IsBitField = false;
755 uint32_t SizeInBits;
756
757 if (Tag == dwarf::DW_TAG_array_type) {
758 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
759 SizeInBits = calcArraySize(CTy, StartDim: 1) * EltTy->getSizeInBits();
760 } else {
761 MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
762 SizeInBits = MemberTy->getSizeInBits();
763 IsBitField = MemberTy->isBitField();
764 }
765
766 if (!IsBitField) {
767 if (SizeInBits > 64)
768 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
769 return 64 - SizeInBits;
770 }
771
772 unsigned SBitOffset, NextSBitOffset;
773 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset, EndBitOffset&: NextSBitOffset);
774 if (NextSBitOffset - SBitOffset > 64)
775 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
776
777 unsigned OffsetInBits = MemberTy->getOffsetInBits();
778 if (Triple.getArch() == Triple::bpfel)
779 return SBitOffset + 64 - OffsetInBits - SizeInBits;
780 else
781 return OffsetInBits + 64 - NextSBitOffset;
782 }
783
784 if (InfoKind == BTF::FIELD_RSHIFT_U64) {
785 DIDerivedType *MemberTy = nullptr;
786 bool IsBitField = false;
787 uint32_t SizeInBits;
788 if (Tag == dwarf::DW_TAG_array_type) {
789 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
790 SizeInBits = calcArraySize(CTy, StartDim: 1) * EltTy->getSizeInBits();
791 } else {
792 MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
793 SizeInBits = MemberTy->getSizeInBits();
794 IsBitField = MemberTy->isBitField();
795 }
796
797 if (!IsBitField) {
798 if (SizeInBits > 64)
799 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
800 return 64 - SizeInBits;
801 }
802
803 unsigned SBitOffset, NextSBitOffset;
804 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset, EndBitOffset&: NextSBitOffset);
805 if (NextSBitOffset - SBitOffset > 64)
806 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
807
808 return 64 - SizeInBits;
809 }
810
811 llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind");
812}
813
814bool BPFAbstractMemberAccess::HasPreserveFieldInfoCall(CallInfoStack &CallStack) {
815 // This is called in error return path, no need to maintain CallStack.
816 while (CallStack.size()) {
817 auto StackElem = CallStack.top();
818 if (StackElem.second.Kind == BPFPreserveFieldInfoAI)
819 return true;
820 CallStack.pop();
821 }
822 return false;
823}
824
825/// Compute the base of the whole preserve_* intrinsics chains, i.e., the base
826/// pointer of the first preserve_*_access_index call, and construct the access
827/// string, which will be the name of a global variable.
828Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call,
829 CallInfo &CInfo,
830 std::string &AccessKey,
831 MDNode *&TypeMeta) {
832 Value *Base = nullptr;
833 std::string TypeName;
834 CallInfoStack CallStack;
835
836 // Put the access chain into a stack with the top as the head of the chain.
837 while (Call) {
838 CallStack.push(x: std::make_pair(x&: Call, y&: CInfo));
839 auto &Chain = AIChain[Call];
840 CInfo = Chain.second;
841 Call = Chain.first;
842 }
843
844 // The access offset from the base of the head of chain is also
845 // calculated here as all debuginfo types are available.
846
847 // Get type name and calculate the first index.
848 // We only want to get type name from typedef, structure or union.
849 // If user wants a relocation like
850 // int *p; ... __builtin_preserve_access_index(&p[4]) ...
851 // or
852 // int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ...
853 // we will skip them.
854 uint32_t FirstIndex = 0;
855 uint32_t PatchImm = 0; // AccessOffset or the requested field info
856 uint32_t InfoKind = BTF::FIELD_BYTE_OFFSET;
857 while (CallStack.size()) {
858 auto StackElem = CallStack.top();
859 Call = StackElem.first;
860 CInfo = StackElem.second;
861
862 if (!Base)
863 Base = CInfo.Base;
864
865 DIType *PossibleTypeDef = stripQualifiers(Ty: cast<DIType>(Val: CInfo.Metadata),
866 skipTypedef: false);
867 DIType *Ty = stripQualifiers(Ty: PossibleTypeDef);
868 if (CInfo.Kind == BPFPreserveUnionAI ||
869 CInfo.Kind == BPFPreserveStructAI) {
870 // struct or union type. If the typedef is in the metadata, always
871 // use the typedef.
872 TypeName = std::string(PossibleTypeDef->getName());
873 TypeMeta = PossibleTypeDef;
874 PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3);
875 break;
876 }
877
878 assert(CInfo.Kind == BPFPreserveArrayAI);
879
880 // Array entries will always be consumed for accumulative initial index.
881 CallStack.pop();
882
883 // BPFPreserveArrayAI
884 uint64_t AccessIndex = CInfo.AccessIndex;
885
886 DIType *BaseTy = nullptr;
887 bool CheckElemType = false;
888 if (const auto *CTy = dyn_cast<DICompositeType>(Val: Ty)) {
889 // array type
890 assert(CTy->getTag() == dwarf::DW_TAG_array_type);
891
892
893 FirstIndex += AccessIndex * calcArraySize(CTy, StartDim: 1);
894 BaseTy = stripQualifiers(Ty: CTy->getBaseType());
895 CheckElemType = CTy->getElements().size() == 1;
896 } else {
897 // pointer type
898 auto *DTy = cast<DIDerivedType>(Val: Ty);
899 assert(DTy->getTag() == dwarf::DW_TAG_pointer_type);
900
901 BaseTy = stripQualifiers(Ty: DTy->getBaseType());
902 CTy = dyn_cast<DICompositeType>(Val: BaseTy);
903 if (!CTy) {
904 CheckElemType = true;
905 } else if (CTy->getTag() != dwarf::DW_TAG_array_type) {
906 FirstIndex += AccessIndex;
907 CheckElemType = true;
908 } else {
909 FirstIndex += AccessIndex * calcArraySize(CTy, StartDim: 0);
910 }
911 }
912
913 if (CheckElemType) {
914 auto *CTy = dyn_cast<DICompositeType>(Val: BaseTy);
915 if (!CTy) {
916 if (HasPreserveFieldInfoCall(CallStack))
917 report_fatal_error(reason: "Invalid field access for llvm.preserve.field.info intrinsic");
918 return nullptr;
919 }
920
921 unsigned CTag = CTy->getTag();
922 if (CTag == dwarf::DW_TAG_structure_type || CTag == dwarf::DW_TAG_union_type) {
923 TypeName = std::string(CTy->getName());
924 } else {
925 if (HasPreserveFieldInfoCall(CallStack))
926 report_fatal_error(reason: "Invalid field access for llvm.preserve.field.info intrinsic");
927 return nullptr;
928 }
929 TypeMeta = CTy;
930 PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3);
931 break;
932 }
933 }
934 assert(TypeName.size());
935 AccessKey += std::to_string(val: FirstIndex);
936
937 // Traverse the rest of access chain to complete offset calculation
938 // and access key construction.
939 while (CallStack.size()) {
940 auto StackElem = CallStack.top();
941 CInfo = StackElem.second;
942 CallStack.pop();
943
944 if (CInfo.Kind == BPFPreserveFieldInfoAI) {
945 InfoKind = CInfo.AccessIndex;
946 if (InfoKind == BTF::FIELD_EXISTENCE)
947 PatchImm = 1;
948 break;
949 }
950
951 // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI,
952 // the action will be extracting field info.
953 if (CallStack.size()) {
954 auto StackElem2 = CallStack.top();
955 CallInfo CInfo2 = StackElem2.second;
956 if (CInfo2.Kind == BPFPreserveFieldInfoAI) {
957 InfoKind = CInfo2.AccessIndex;
958 assert(CallStack.size() == 1);
959 }
960 }
961
962 // Access Index
963 uint64_t AccessIndex = CInfo.AccessIndex;
964 MDNode *MDN = CInfo.Metadata;
965 // At this stage, it cannot be pointer type.
966 auto *CTy = cast<DICompositeType>(Val: stripQualifiers(Ty: cast<DIType>(Val: MDN)));
967
968 uint64_t BTFIndex = AccessIndex;
969 if (CTy->getTag() == dwarf::DW_TAG_structure_type) {
970 DINodeArray Elements = CTy->getElements();
971 uint64_t Offset = getBTFRecordElementOffset(Element: Elements[AccessIndex]);
972 // Find this element's position in the stable offset order without
973 // sorting the whole record for every CO-RE access.
974 BTFIndex = 0;
975 for (unsigned I = 0; I < Elements.size(); ++I) {
976 uint64_t ElementOffset = getBTFRecordElementOffset(Element: Elements[I]);
977 if (ElementOffset < Offset ||
978 (ElementOffset == Offset && I < AccessIndex))
979 ++BTFIndex;
980 }
981 }
982 AccessKey += ":" + std::to_string(val: BTFIndex);
983
984 PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex, PatchImm,
985 RecordAlignment: CInfo.RecordAlignment);
986 }
987
988 // Access key is the
989 // "llvm." + type name + ":" + reloc type + ":" + patched imm + "$" +
990 // access string,
991 // uniquely identifying one relocation.
992 // The prefix "llvm." indicates this is a temporary global, which should
993 // not be emitted to ELF file.
994 AccessKey = "llvm." + TypeName + ":" + std::to_string(val: InfoKind) + ":" +
995 std::to_string(val: PatchImm) + "$" + AccessKey;
996
997 return Base;
998}
999
1000MDNode *BPFAbstractMemberAccess::computeAccessKey(CallInst *Call,
1001 CallInfo &CInfo,
1002 std::string &AccessKey,
1003 bool &IsInt32Ret) {
1004 DIType *Ty = stripQualifiers(Ty: cast<DIType>(Val: CInfo.Metadata), skipTypedef: false);
1005 assert(!Ty->getName().empty());
1006
1007 int64_t PatchImm;
1008 std::string AccessStr("0");
1009 if (CInfo.AccessIndex == BTF::TYPE_EXISTENCE ||
1010 CInfo.AccessIndex == BTF::TYPE_MATCH) {
1011 PatchImm = 1;
1012 } else if (CInfo.AccessIndex == BTF::TYPE_SIZE) {
1013 // typedef debuginfo type has size 0, get the eventual base type.
1014 DIType *BaseTy = stripQualifiers(Ty, skipTypedef: true);
1015 PatchImm = BaseTy->getSizeInBits() / 8;
1016 } else {
1017 // ENUM_VALUE_EXISTENCE and ENUM_VALUE
1018 IsInt32Ret = false;
1019
1020 // The argument could be a global variable or a getelementptr with base to
1021 // a global variable depending on whether the clang option `opaque-options`
1022 // is set or not.
1023 const GlobalVariable *GV =
1024 cast<GlobalVariable>(Val: Call->getArgOperand(i: 1)->stripPointerCasts());
1025 assert(GV->hasInitializer());
1026 const ConstantDataArray *DA = cast<ConstantDataArray>(Val: GV->getInitializer());
1027 assert(DA->isString());
1028 StringRef ValueStr = DA->getAsString();
1029
1030 // ValueStr format: <EnumeratorStr>:<Value>
1031 size_t Separator = ValueStr.find_first_of(C: ':');
1032 StringRef EnumeratorStr = ValueStr.substr(Start: 0, N: Separator);
1033
1034 // Find enumerator index in the debuginfo
1035 DIType *BaseTy = stripQualifiers(Ty, skipTypedef: true);
1036 const auto *CTy = cast<DICompositeType>(Val: BaseTy);
1037 assert(CTy->getTag() == dwarf::DW_TAG_enumeration_type);
1038 int EnumIndex = 0;
1039 for (const auto Element : CTy->getElements()) {
1040 const auto *Enum = cast<DIEnumerator>(Val: Element);
1041 if (Enum->getName() == EnumeratorStr) {
1042 AccessStr = std::to_string(val: EnumIndex);
1043 break;
1044 }
1045 EnumIndex++;
1046 }
1047
1048 if (CInfo.AccessIndex == BTF::ENUM_VALUE) {
1049 StringRef EValueStr = ValueStr.substr(Start: Separator + 1);
1050 PatchImm = std::stoll(str: std::string(EValueStr));
1051 } else {
1052 PatchImm = 1;
1053 }
1054 }
1055
1056 AccessKey = "llvm." + Ty->getName().str() + ":" +
1057 std::to_string(val: CInfo.AccessIndex) + std::string(":") +
1058 std::to_string(val: PatchImm) + std::string("$") + AccessStr;
1059
1060 return Ty;
1061}
1062
1063/// Call/Kind is the base preserve_*_access_index() call. Attempts to do
1064/// transformation to a chain of relocable GEPs.
1065bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call,
1066 CallInfo &CInfo) {
1067 std::string AccessKey;
1068 MDNode *TypeMeta;
1069 Value *Base = nullptr;
1070 bool IsInt32Ret;
1071
1072 IsInt32Ret = CInfo.Kind == BPFPreserveFieldInfoAI;
1073 if (CInfo.Kind == BPFPreserveFieldInfoAI && CInfo.Metadata) {
1074 TypeMeta = computeAccessKey(Call, CInfo, AccessKey, IsInt32Ret);
1075 } else {
1076 Base = computeBaseAndAccessKey(Call, CInfo, AccessKey, TypeMeta);
1077 if (!Base)
1078 return false;
1079 }
1080
1081 BasicBlock *BB = Call->getParent();
1082 GlobalVariable *GV;
1083
1084 if (GEPGlobals.find(x: AccessKey) == GEPGlobals.end()) {
1085 IntegerType *VarType;
1086 if (IsInt32Ret)
1087 VarType = Type::getInt32Ty(C&: BB->getContext()); // 32bit return value
1088 else
1089 VarType = Type::getInt64Ty(C&: BB->getContext()); // 64bit ptr or enum value
1090
1091 GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage,
1092 nullptr, AccessKey);
1093 GV->addAttribute(Kind: BPFCoreSharedInfo::AmaAttr);
1094 GV->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: TypeMeta);
1095 GEPGlobals[AccessKey] = GV;
1096 } else {
1097 GV = GEPGlobals[AccessKey];
1098 }
1099
1100 if (CInfo.Kind == BPFPreserveFieldInfoAI) {
1101 // Load the global variable which represents the returned field info.
1102 LoadInst *LDInst;
1103 if (IsInt32Ret)
1104 LDInst = new LoadInst(Type::getInt32Ty(C&: BB->getContext()), GV, "",
1105 Call->getIterator());
1106 else
1107 LDInst = new LoadInst(Type::getInt64Ty(C&: BB->getContext()), GV, "",
1108 Call->getIterator());
1109
1110 Instruction *PassThroughInst =
1111 BPFCoreSharedInfo::insertPassThrough(M, BB, Input: LDInst, Before: Call);
1112 Call->replaceAllUsesWith(V: PassThroughInst);
1113 Call->eraseFromParent();
1114 return true;
1115 }
1116
1117 // For any original GEP Call and Base %2 like
1118 // %4 = bitcast %struct.net_device** %dev1 to i64*
1119 // it is transformed to:
1120 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0
1121 // %8 = getelementptr i8, i8* %2, %6
1122 // using %8 instead of %4
1123 // The original Call inst is removed.
1124
1125 // Load the global variable.
1126 auto *LDInst = new LoadInst(Type::getInt64Ty(C&: BB->getContext()), GV, "",
1127 Call->getIterator());
1128
1129 // Generate a GetElementPtr
1130 auto *GEP = GetElementPtrInst::Create(PointeeType: Type::getInt8Ty(C&: BB->getContext()), Ptr: Base,
1131 IdxList: LDInst);
1132 GEP->insertBefore(InsertPos: Call->getIterator());
1133
1134 // For the following code,
1135 // Block0:
1136 // ...
1137 // if (...) goto Block1 else ...
1138 // Block1:
1139 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0
1140 // %8 = getelementptr i8, i8* %2, %6
1141 // ...
1142 // goto CommonExit
1143 // Block2:
1144 // ...
1145 // if (...) goto Block3 else ...
1146 // Block3:
1147 // %6 = load llvm.bpf_map:0:40$0:0:0:2:0
1148 // %8 = getelementptr i8, i8* %2, %6
1149 // ...
1150 // goto CommonExit
1151 // CommonExit
1152 // SimplifyCFG may generate:
1153 // Block0:
1154 // ...
1155 // if (...) goto Block_Common else ...
1156 // Block2:
1157 // ...
1158 // if (...) goto Block_Common else ...
1159 // Block_Common:
1160 // PHI = [llvm.sk_buff:0:50$0:0:0:2:0, llvm.bpf_map:0:40$0:0:0:2:0]
1161 // %6 = load PHI
1162 // %8 = getelementptr i8, i8* %2, %6
1163 // ...
1164 // goto CommonExit
1165 // For the above code, we cannot perform proper relocation since
1166 // "load PHI" has two possible relocations.
1167 //
1168 // To prevent above tail merging, we use __builtin_bpf_passthrough()
1169 // where one of its parameters is a seq_num. Since two
1170 // __builtin_bpf_passthrough() funcs will always have different seq_num,
1171 // tail merging cannot happen. The __builtin_bpf_passthrough() will be
1172 // removed in the beginning of Target IR passes.
1173 //
1174 // This approach is also used in other places when global var
1175 // representing a relocation is used.
1176 Instruction *PassThroughInst =
1177 BPFCoreSharedInfo::insertPassThrough(M, BB, Input: GEP, Before: Call);
1178 Call->replaceAllUsesWith(V: PassThroughInst);
1179 Call->eraseFromParent();
1180
1181 return true;
1182}
1183
1184bool BPFAbstractMemberAccess::doTransformation(Function &F) {
1185 bool Transformed = false;
1186
1187 // Collect PreserveDIAccessIndex Intrinsic call chains.
1188 // The call chains will be used to generate the access
1189 // patterns similar to GEP.
1190 collectAICallChains(F);
1191
1192 for (auto &C : BaseAICalls)
1193 Transformed = transformGEPChain(Call: C.first, CInfo&: C.second) || Transformed;
1194
1195 return removePreserveAccessIndexIntrinsic(F) || Transformed;
1196}
1197
1198PreservedAnalyses
1199BPFAbstractMemberAccessPass::run(Function &F, FunctionAnalysisManager &AM) {
1200 return BPFAbstractMemberAccess(TM).run(F) ? PreservedAnalyses::none()
1201 : PreservedAnalyses::all();
1202}
1203