1//===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass abstracted struct/union member accesses in order to support
10// compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program
11// which can run on different kernels. In particular, if bpf program tries to
12// access a particular kernel data structure member, the details of the
13// intermediate member access will be remembered so bpf loader can do
14// necessary adjustment right before program loading.
15//
16// For example,
17//
18// struct s {
19// int a;
20// int b;
21// };
22// struct t {
23// struct s c;
24// int d;
25// };
26// struct t e;
27//
28// For the member access e.c.b, the compiler will generate code
29// &e + 4
30//
31// The compile-once run-everywhere instead generates the following code
32// r = 4
33// &e + r
34// The "4" in "r = 4" can be changed based on a particular kernel version.
35// For example, on a particular kernel version, if struct s is changed to
36//
37// struct s {
38// int new_field;
39// int a;
40// int b;
41// }
42//
43// By repeating the member access on the host, the bpf loader can
44// adjust "r = 4" as "r = 8".
45//
46// This feature relies on the following three intrinsic calls:
47// addr = preserve_array_access_index(base, dimension, index)
48// addr = preserve_union_access_index(base, di_index)
49// !llvm.preserve.access.index <union_ditype>
50// addr = preserve_struct_access_index(base, gep_index, di_index)
51// !llvm.preserve.access.index <struct_ditype>
52//
53// Bitfield member access needs special attention. User cannot take the
54// address of a bitfield acceess. To facilitate kernel verifier
55// for easy bitfield code optimization, a new clang intrinsic is introduced:
56// uint32_t __builtin_preserve_field_info(member_access, info_kind)
57// In IR, a chain with two (or more) intrinsic calls will be generated:
58// ...
59// addr = preserve_struct_access_index(base, 1, 1) !struct s
60// uint32_t result = bpf_preserve_field_info(addr, info_kind)
61//
62// Suppose the info_kind is FIELD_SIGNEDNESS,
63// The above two IR intrinsics will be replaced with
64// a relocatable insn:
65// signness = /* signness of member_access */
66// and signness can be changed by bpf loader based on the
67// types on the host.
68//
69// User can also test whether a field exists or not with
70// uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE)
71// The field will be always available (result = 1) during initial
72// compilation, but bpf loader can patch with the correct value
73// on the target host where the member_access may or may not be available
74//
75//===----------------------------------------------------------------------===//
76
77#include "BPF.h"
78#include "BPFCORE.h"
79#include "BPFTargetMachine.h"
80#include "llvm/ADT/ArrayRef.h"
81#include "llvm/ADT/MapVector.h"
82#include "llvm/ADT/STLExtras.h"
83#include "llvm/ADT/SmallVector.h"
84#include "llvm/ADT/Twine.h"
85#include "llvm/BinaryFormat/Dwarf.h"
86#include "llvm/DebugInfo/BTF/BTF.h"
87#include "llvm/IR/DebugInfoMetadata.h"
88#include "llvm/IR/GlobalVariable.h"
89#include "llvm/IR/Instruction.h"
90#include "llvm/IR/Instructions.h"
91#include "llvm/IR/IntrinsicsBPF.h"
92#include "llvm/IR/Module.h"
93#include "llvm/IR/PassManager.h"
94#include "llvm/IR/Type.h"
95#include "llvm/IR/User.h"
96#include "llvm/IR/Value.h"
97#include "llvm/IR/ValueHandle.h"
98#include "llvm/Pass.h"
99#include "llvm/Transforms/Utils/BasicBlockUtils.h"
100#include <optional>
101
102#define DEBUG_TYPE "bpf-abstract-member-access"
103
104namespace llvm {
105uint32_t BPFCoreSharedInfo::SeqNum;
106
107BPFCoreSharedInfo::AmaRelocation
108BPFCoreSharedInfo::AmaRelocation::decode(StringRef Name) {
109 // Type names may contain ':' and '$'. Only the suffix has a fixed format.
110 auto [Prefix, AccessString] = Name.rsplit(Separator: '$');
111 auto [NameAndKind, PatchString] = Prefix.rsplit(Separator: ':');
112 StringRef KindString = NameAndKind.rsplit(Separator: ':').second;
113 uint32_t Kind;
114 int64_t PatchImm;
115 if (AccessString.empty() || KindString.getAsInteger(Radix: 10, Result&: Kind) ||
116 PatchString.getAsInteger(Radix: 10, Result&: PatchImm))
117 reportFatalInternalError(reason: Twine("Invalid BPF CO-RE relocation global ") +
118 Name);
119 return {.Kind: static_cast<BTF::PatchableRelocKind>(Kind), .PatchImm: PatchImm, .AccessString: AccessString};
120}
121
122Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB,
123 Instruction *Input,
124 Instruction *Before) {
125 Function *Fn = Intrinsic::getOrInsertDeclaration(
126 M, id: Intrinsic::bpf_passthrough, OverloadTys: {Input->getType(), Input->getType()});
127 Constant *SeqNumVal = ConstantInt::get(Ty: Type::getInt32Ty(C&: BB->getContext()),
128 V: BPFCoreSharedInfo::SeqNum++);
129
130 auto *NewInst = CallInst::Create(Func: Fn, Args: {SeqNumVal, Input});
131 NewInst->insertBefore(InsertPos: Before->getIterator());
132 return NewInst;
133}
134} // namespace llvm
135
136using namespace llvm;
137
138namespace {
139class BPFAbstractMemberAccess final {
140public:
141 BPFAbstractMemberAccess(BPFTargetMachine *TM) : TM(TM) {}
142
143 bool run(Function &F);
144
145 struct CallInfo {
146 uint32_t Kind;
147 uint32_t AccessIndex;
148 MaybeAlign RecordAlignment;
149 MDNode *Metadata;
150 WeakTrackingVH Base;
151 };
152
153private:
154 enum : uint32_t {
155 BPFPreserveArrayAI = 1,
156 BPFPreserveUnionAI = 2,
157 BPFPreserveStructAI = 3,
158 BPFPreserveFieldInfoAI = 4,
159 };
160
161 TargetMachine *TM;
162 const DataLayout *DL = nullptr;
163 Module *M = nullptr;
164
165 // A map to link preserve_*_access_index intrinsic calls.
166 std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain;
167 // A map to hold all the base preserve_*_access_index intrinsic calls.
168 // The base call is not an input of any other preserve_*
169 // intrinsics.
170 // Iterated below, so the order can't come from the addresses.
171 SmallMapVector<CallInst *, CallInfo, 4> BaseAICalls;
172 // A map to hold <AnonRecord, TypeDef> relationships
173 std::map<DICompositeType *, DIDerivedType *> AnonRecords;
174
175 void CheckAnonRecordType(DIDerivedType *ParentTy, DIType *Ty);
176 void CheckCompositeType(DIDerivedType *ParentTy, DICompositeType *CTy);
177 void CheckDerivedType(DIDerivedType *ParentTy, DIDerivedType *DTy);
178 void ResetMetadata(struct CallInfo &CInfo);
179
180 bool doTransformation(Function &F);
181
182 void traceAICall(CallInst *Call, CallInfo &ParentInfo);
183 void traceBitCast(BitCastInst *BitCast, CallInst *Parent,
184 CallInfo &ParentInfo);
185 void traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
186 CallInfo &ParentInfo);
187 void collectAICallChains(Function &F);
188
189 bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo);
190 bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI,
191 const MDNode *ChildMeta);
192 bool removePreserveAccessIndexIntrinsic(Function &F);
193 void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment,
194 uint32_t &StartBitOffset, uint32_t &EndBitOffset);
195 uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy,
196 uint32_t AccessIndex, uint32_t PatchImm,
197 MaybeAlign RecordAlignment);
198
199 struct AccessRoot {
200 DIType *Type;
201 uint32_t Index;
202 };
203 static std::optional<AccessRoot>
204 consumeAccessRoot(SmallVectorImpl<CallInfo> &CallStack);
205 Value *computeFieldAccess(CallInst *Call, const CallInfo &Terminal,
206 std::string &RelocSuffix, DIType *&RootType);
207 struct EnumAccess {
208 size_t Index;
209 StringRef Value;
210 };
211 static EnumAccess computeEnumAccess(const CallInst *Call, const DIType *Ty);
212 DIType *computeTypeAccess(CallInst *Call, const CallInfo &CInfo,
213 std::string &RelocSuffix);
214 bool transformGEPChain(CallInst *Call, const CallInfo &CInfo);
215};
216
217} // End anonymous namespace
218
219bool BPFAbstractMemberAccess::run(Function &F) {
220 LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n");
221
222 M = F.getParent();
223 if (!M)
224 return false;
225
226 // Bail out if no debug info.
227 if (M->debug_compile_units().empty())
228 return false;
229
230 // For each argument/return/local_variable type, trace the type
231 // pattern like '[derived_type]* [composite_type]' to check
232 // and remember (anon record -> typedef) relations where the
233 // anon record is defined as
234 // typedef [const/volatile/restrict]* [anon record]
235 DISubprogram *SP = F.getSubprogram();
236 if (SP && SP->isDefinition()) {
237 for (DIType *Ty: SP->getType()->getTypeArray())
238 CheckAnonRecordType(ParentTy: nullptr, Ty);
239 for (const MDNode *DN : SP->getRetainedNodes()) {
240 if (const auto *DV = dyn_cast<DILocalVariable>(Val: DN))
241 CheckAnonRecordType(ParentTy: nullptr, Ty: DV->getType());
242 }
243 }
244
245 DL = &M->getDataLayout();
246 return doTransformation(F);
247}
248
249void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) {
250 if (auto Ty = dyn_cast<DICompositeType>(Val: CInfo.Metadata)) {
251 auto It = AnonRecords.find(x: Ty);
252 if (It != AnonRecords.end() && It->second != nullptr)
253 CInfo.Metadata = It->second;
254 }
255}
256
257void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy,
258 DICompositeType *CTy) {
259 if (!CTy->getName().empty() || !ParentTy ||
260 ParentTy->getTag() != dwarf::DW_TAG_typedef)
261 return;
262
263 auto [It, Inserted] = AnonRecords.try_emplace(k: CTy, args&: ParentTy);
264 // Two or more typedef's may point to the same anon record.
265 // If this is the case, set the typedef DIType to be nullptr
266 // to indicate the duplication case.
267 if (!Inserted && It->second != ParentTy)
268 It->second = nullptr;
269}
270
271void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy,
272 DIDerivedType *DTy) {
273 DIType *BaseType = DTy->getBaseType();
274 if (!BaseType)
275 return;
276
277 unsigned Tag = DTy->getTag();
278 if (Tag == dwarf::DW_TAG_pointer_type)
279 CheckAnonRecordType(ParentTy: nullptr, Ty: BaseType);
280 else if (Tag == dwarf::DW_TAG_typedef)
281 CheckAnonRecordType(ParentTy: DTy, Ty: BaseType);
282 else
283 CheckAnonRecordType(ParentTy, Ty: BaseType);
284}
285
286void BPFAbstractMemberAccess::CheckAnonRecordType(DIDerivedType *ParentTy,
287 DIType *Ty) {
288 if (!Ty)
289 return;
290
291 if (auto *CTy = dyn_cast<DICompositeType>(Val: Ty))
292 return CheckCompositeType(ParentTy, CTy);
293 else if (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty))
294 return CheckDerivedType(ParentTy, DTy);
295}
296
297static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) {
298 if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type &&
299 Tag != dwarf::DW_TAG_volatile_type &&
300 Tag != dwarf::DW_TAG_restrict_type &&
301 Tag != dwarf::DW_TAG_member)
302 return false;
303 if (Tag == dwarf::DW_TAG_typedef && !skipTypedef)
304 return false;
305 return true;
306}
307
308static DIType * stripQualifiers(DIType *Ty, bool skipTypedef = true) {
309 while (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty)) {
310 if (!SkipDIDerivedTag(Tag: DTy->getTag(), skipTypedef))
311 break;
312 Ty = DTy->getBaseType();
313 }
314 return Ty;
315}
316
317static const DIType * stripQualifiers(const DIType *Ty) {
318 while (auto *DTy = dyn_cast<DIDerivedType>(Val: Ty)) {
319 if (!SkipDIDerivedTag(Tag: DTy->getTag(), skipTypedef: true))
320 break;
321 Ty = DTy->getBaseType();
322 }
323 return Ty;
324}
325
326static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim) {
327 DINodeArray Elements = CTy->getElements();
328 uint32_t DimSize = 1;
329 for (uint32_t I = StartDim; I < Elements.size(); ++I) {
330 if (auto *Element = dyn_cast_or_null<DINode>(Val: Elements[I]))
331 if (Element->getTag() == dwarf::DW_TAG_subrange_type) {
332 const DISubrange *SR = cast<DISubrange>(Val: Element);
333 auto *CI = dyn_cast<ConstantInt *>(Val: SR->getCount());
334 DimSize *= CI->getSExtValue();
335 }
336 }
337
338 return DimSize;
339}
340
341/// Return the number of innermost elements in \p Ty, looking through array
342/// element types such as Row in typedef struct S Row[3]. \p Ty is updated to
343/// the innermost element type.
344static uint32_t calcElementCount(DIType *&Ty) {
345 uint32_t Count = 1;
346 for (;;) {
347 auto *Array = dyn_cast<DICompositeType>(Val: stripQualifiers(Ty));
348 if (!Array || Array->getTag() != dwarf::DW_TAG_array_type)
349 return Count;
350 Count *= calcArraySize(CTy: Array, StartDim: 0);
351 Ty = Array->getBaseType();
352 }
353}
354
355static Type *getBaseElementType(const CallInst *Call) {
356 // Element type is stored in an elementtype() attribute on the first param.
357 return Call->getParamElementType(ArgNo: 0);
358}
359
360static uint64_t getConstant(const Value *IndexValue) {
361 const ConstantInt *CV = dyn_cast<ConstantInt>(Val: IndexValue);
362 assert(CV);
363 return CV->getValue().getZExtValue();
364}
365
366/// Check whether a call is a preserve_*_access_index intrinsic call or not.
367bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,
368 CallInfo &CInfo) {
369 if (!Call)
370 return false;
371
372 const auto *GV = dyn_cast<GlobalValue>(Val: Call->getCalledOperand());
373 if (!GV)
374 return false;
375 if (GV->getName().starts_with(Prefix: "llvm.preserve.array.access.index")) {
376 CInfo.Kind = BPFPreserveArrayAI;
377 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
378 if (!CInfo.Metadata)
379 report_fatal_error(reason: "Missing metadata for llvm.preserve.array.access.index intrinsic");
380 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 2));
381 CInfo.Base = Call->getArgOperand(i: 0);
382 CInfo.RecordAlignment = DL->getABITypeAlign(Ty: getBaseElementType(Call));
383 return true;
384 }
385 if (GV->getName().starts_with(Prefix: "llvm.preserve.union.access.index")) {
386 CInfo.Kind = BPFPreserveUnionAI;
387 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
388 if (!CInfo.Metadata)
389 report_fatal_error(reason: "Missing metadata for llvm.preserve.union.access.index intrinsic");
390 ResetMetadata(CInfo);
391 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 1));
392 CInfo.Base = Call->getArgOperand(i: 0);
393 return true;
394 }
395 if (GV->getName().starts_with(Prefix: "llvm.preserve.struct.access.index")) {
396 CInfo.Kind = BPFPreserveStructAI;
397 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
398 if (!CInfo.Metadata)
399 report_fatal_error(reason: "Missing metadata for llvm.preserve.struct.access.index intrinsic");
400 ResetMetadata(CInfo);
401 CInfo.AccessIndex = getConstant(IndexValue: Call->getArgOperand(i: 2));
402 CInfo.Base = Call->getArgOperand(i: 0);
403 CInfo.RecordAlignment = DL->getABITypeAlign(Ty: getBaseElementType(Call));
404 return true;
405 }
406 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.field.info")) {
407 CInfo.Kind = BPFPreserveFieldInfoAI;
408 CInfo.Metadata = nullptr;
409 // Check validity of info_kind as clang did not check this.
410 uint64_t InfoKind = getConstant(IndexValue: Call->getArgOperand(i: 1));
411 if (InfoKind >= BTF::MAX_FIELD_RELOC_KIND)
412 report_fatal_error(reason: "Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic");
413 CInfo.AccessIndex = InfoKind;
414 return true;
415 }
416 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.type.info")) {
417 CInfo.Kind = BPFPreserveFieldInfoAI;
418 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
419 if (!CInfo.Metadata)
420 report_fatal_error(reason: "Missing metadata for llvm.preserve.type.info intrinsic");
421 uint64_t Flag = getConstant(IndexValue: Call->getArgOperand(i: 1));
422 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_TYPE_INFO_FLAG)
423 report_fatal_error(reason: "Incorrect flag for llvm.bpf.preserve.type.info intrinsic");
424 if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_EXISTENCE)
425 CInfo.AccessIndex = BTF::TYPE_EXISTENCE;
426 else if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_MATCH)
427 CInfo.AccessIndex = BTF::TYPE_MATCH;
428 else
429 CInfo.AccessIndex = BTF::TYPE_SIZE;
430 return true;
431 }
432 if (GV->getName().starts_with(Prefix: "llvm.bpf.preserve.enum.value")) {
433 CInfo.Kind = BPFPreserveFieldInfoAI;
434 CInfo.Metadata = Call->getMetadata(KindID: LLVMContext::MD_preserve_access_index);
435 if (!CInfo.Metadata)
436 report_fatal_error(reason: "Missing metadata for llvm.preserve.enum.value intrinsic");
437 uint64_t Flag = getConstant(IndexValue: Call->getArgOperand(i: 2));
438 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_ENUM_VALUE_FLAG)
439 report_fatal_error(reason: "Incorrect flag for llvm.bpf.preserve.enum.value intrinsic");
440 if (Flag == BPFCoreSharedInfo::PRESERVE_ENUM_VALUE_EXISTENCE)
441 CInfo.AccessIndex = BTF::ENUM_VALUE_EXISTENCE;
442 else
443 CInfo.AccessIndex = BTF::ENUM_VALUE;
444 return true;
445 }
446
447 return false;
448}
449
450static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex,
451 uint32_t GEPIndex) {
452 uint32_t Dimension = 1;
453 if (DimensionIndex > 0)
454 Dimension = getConstant(IndexValue: Call->getArgOperand(i: DimensionIndex));
455
456 Constant *Zero =
457 ConstantInt::get(Ty: Type::getInt32Ty(C&: Call->getParent()->getContext()), V: 0);
458 SmallVector<Value *, 4> IdxList(Dimension, Zero);
459 IdxList.push_back(Elt: Call->getArgOperand(i: GEPIndex));
460
461 auto *GEP = GetElementPtrInst::CreateInBounds(PointeeType: getBaseElementType(Call),
462 Ptr: Call->getArgOperand(i: 0), IdxList,
463 NameStr: "", InsertBefore: Call->getIterator());
464 Call->replaceAllUsesWith(V: GEP);
465 Call->eraseFromParent();
466}
467
468void BPFCoreSharedInfo::removeArrayAccessCall(CallInst *Call) {
469 replaceWithGEP(Call, DimensionIndex: 1, GEPIndex: 2);
470}
471
472void BPFCoreSharedInfo::removeStructAccessCall(CallInst *Call) {
473 replaceWithGEP(Call, DimensionIndex: 0, GEPIndex: 1);
474}
475
476void BPFCoreSharedInfo::removeUnionAccessCall(CallInst *Call) {
477 Call->replaceAllUsesWith(V: Call->getArgOperand(i: 0));
478 Call->eraseFromParent();
479}
480
481bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) {
482 std::vector<CallInst *> PreserveArrayIndexCalls;
483 std::vector<CallInst *> PreserveUnionIndexCalls;
484 std::vector<CallInst *> PreserveStructIndexCalls;
485 std::vector<CallInst *> PreserveFieldInfoCalls;
486 bool Found = false;
487
488 for (auto &BB : F)
489 for (auto &I : BB) {
490 auto *Call = dyn_cast<CallInst>(Val: &I);
491 CallInfo CInfo;
492 if (!IsPreserveDIAccessIndexCall(Call, CInfo))
493 continue;
494
495 Found = true;
496 if (CInfo.Kind == BPFPreserveArrayAI)
497 PreserveArrayIndexCalls.push_back(x: Call);
498 else if (CInfo.Kind == BPFPreserveUnionAI)
499 PreserveUnionIndexCalls.push_back(x: Call);
500 else if (CInfo.Kind == BPFPreserveFieldInfoAI)
501 PreserveFieldInfoCalls.push_back(x: Call);
502 else
503 PreserveStructIndexCalls.push_back(x: Call);
504 }
505
506 // A preserve_field_info(), preserve_type_info() or preserve_enum_value()
507 // call left here has an unused result, so it is not relocated. It has no
508 // side effects, so just remove it.
509 for (CallInst *Call : PreserveFieldInfoCalls)
510 if (Call->use_empty())
511 Call->eraseFromParent();
512
513 // do the following transformation:
514 // . addr = preserve_array_access_index(base, dimension, index)
515 // is transformed to
516 // addr = GEP(base, dimenion's zero's, index)
517 // . addr = preserve_union_access_index(base, di_index)
518 // is transformed to
519 // addr = base, i.e., all usages of "addr" are replaced by "base".
520 // . addr = preserve_struct_access_index(base, gep_index, di_index)
521 // is transformed to
522 // addr = GEP(base, 0, gep_index)
523 for (CallInst *Call : PreserveArrayIndexCalls)
524 BPFCoreSharedInfo::removeArrayAccessCall(Call);
525 for (CallInst *Call : PreserveStructIndexCalls)
526 BPFCoreSharedInfo::removeStructAccessCall(Call);
527 for (CallInst *Call : PreserveUnionIndexCalls)
528 BPFCoreSharedInfo::removeUnionAccessCall(Call);
529
530 return Found;
531}
532
533/// Check whether the access index chain is valid. We check
534/// here because there may be type casts between two
535/// access indexes. We want to ensure memory access still valid.
536bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType,
537 uint32_t ParentAI,
538 const MDNode *ChildType) {
539 if (!ChildType)
540 return true; // preserve_field_info, no type comparison needed.
541
542 const DIType *PType = stripQualifiers(Ty: cast<DIType>(Val: ParentType));
543 const DIType *CType = stripQualifiers(Ty: cast<DIType>(Val: ChildType));
544
545 // Child is a derived/pointer type, which is due to type casting.
546 // Pointer type cannot be in the middle of chain.
547 if (isa<DIDerivedType>(Val: CType))
548 return false;
549
550 // Parent is a pointer type.
551 if (const auto *PtrTy = dyn_cast<DIDerivedType>(Val: PType)) {
552 if (PtrTy->getTag() != dwarf::DW_TAG_pointer_type)
553 return false;
554 return stripQualifiers(Ty: PtrTy->getBaseType()) == CType;
555 }
556
557 // Otherwise, struct/union/array types
558 const auto *PTy = dyn_cast<DICompositeType>(Val: PType);
559 const auto *CTy = dyn_cast<DICompositeType>(Val: CType);
560 assert(PTy && CTy && "ParentType or ChildType is null or not composite");
561
562 uint32_t PTyTag = PTy->getTag();
563 assert(PTyTag == dwarf::DW_TAG_array_type ||
564 PTyTag == dwarf::DW_TAG_structure_type ||
565 PTyTag == dwarf::DW_TAG_union_type);
566
567 uint32_t CTyTag = CTy->getTag();
568 assert(CTyTag == dwarf::DW_TAG_array_type ||
569 CTyTag == dwarf::DW_TAG_structure_type ||
570 CTyTag == dwarf::DW_TAG_union_type);
571
572 // Multi dimensional arrays, base element should be the same. Otherwise the
573 // child may still be the element type, e.g. Row in Row a[2] with
574 // typedef struct S Row[3].
575 if (PTyTag == dwarf::DW_TAG_array_type && PTyTag == CTyTag &&
576 PTy->getBaseType() == CTy->getBaseType())
577 return true;
578
579 DIType *Ty;
580 if (PTyTag == dwarf::DW_TAG_array_type)
581 Ty = PTy->getBaseType();
582 else
583 Ty = dyn_cast<DIType>(Val: PTy->getElements()[ParentAI]);
584
585 return dyn_cast<DICompositeType>(Val: stripQualifiers(Ty)) == CTy;
586}
587
588void BPFAbstractMemberAccess::traceAICall(CallInst *Call,
589 CallInfo &ParentInfo) {
590 for (User *U : Call->users()) {
591 Instruction *Inst = dyn_cast<Instruction>(Val: U);
592 if (!Inst)
593 continue;
594
595 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
596 traceBitCast(BitCast: BI, Parent: Call, ParentInfo);
597 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
598 CallInfo ChildInfo;
599
600 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
601 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
602 ChildType: ChildInfo.Metadata)) {
603 AIChain[CI] = std::make_pair(x&: Call, y&: ParentInfo);
604 traceAICall(Call: CI, ParentInfo&: ChildInfo);
605 } else {
606 BaseAICalls[Call] = ParentInfo;
607 }
608 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
609 if (GI->hasAllZeroIndices())
610 traceGEP(GEP: GI, Parent: Call, ParentInfo);
611 else
612 BaseAICalls[Call] = ParentInfo;
613 } else {
614 BaseAICalls[Call] = ParentInfo;
615 }
616 }
617}
618
619void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast,
620 CallInst *Parent,
621 CallInfo &ParentInfo) {
622 for (User *U : BitCast->users()) {
623 Instruction *Inst = dyn_cast<Instruction>(Val: U);
624 if (!Inst)
625 continue;
626
627 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
628 traceBitCast(BitCast: BI, Parent, ParentInfo);
629 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
630 CallInfo ChildInfo;
631 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
632 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
633 ChildType: ChildInfo.Metadata)) {
634 AIChain[CI] = std::make_pair(x&: Parent, y&: ParentInfo);
635 traceAICall(Call: CI, ParentInfo&: ChildInfo);
636 } else {
637 BaseAICalls[Parent] = ParentInfo;
638 }
639 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
640 if (GI->hasAllZeroIndices())
641 traceGEP(GEP: GI, Parent, ParentInfo);
642 else
643 BaseAICalls[Parent] = ParentInfo;
644 } else {
645 BaseAICalls[Parent] = ParentInfo;
646 }
647 }
648}
649
650void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
651 CallInfo &ParentInfo) {
652 for (User *U : GEP->users()) {
653 Instruction *Inst = dyn_cast<Instruction>(Val: U);
654 if (!Inst)
655 continue;
656
657 if (auto *BI = dyn_cast<BitCastInst>(Val: Inst)) {
658 traceBitCast(BitCast: BI, Parent, ParentInfo);
659 } else if (auto *CI = dyn_cast<CallInst>(Val: Inst)) {
660 CallInfo ChildInfo;
661 if (IsPreserveDIAccessIndexCall(Call: CI, CInfo&: ChildInfo) &&
662 IsValidAIChain(ParentType: ParentInfo.Metadata, ParentAI: ParentInfo.AccessIndex,
663 ChildType: ChildInfo.Metadata)) {
664 AIChain[CI] = std::make_pair(x&: Parent, y&: ParentInfo);
665 traceAICall(Call: CI, ParentInfo&: ChildInfo);
666 } else {
667 BaseAICalls[Parent] = ParentInfo;
668 }
669 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Val: Inst)) {
670 if (GI->hasAllZeroIndices())
671 traceGEP(GEP: GI, Parent, ParentInfo);
672 else
673 BaseAICalls[Parent] = ParentInfo;
674 } else {
675 BaseAICalls[Parent] = ParentInfo;
676 }
677 }
678}
679
680void BPFAbstractMemberAccess::collectAICallChains(Function &F) {
681 AIChain.clear();
682 BaseAICalls.clear();
683
684 for (auto &BB : F)
685 for (auto &I : BB) {
686 CallInfo CInfo;
687 auto *Call = dyn_cast<CallInst>(Val: &I);
688 if (!IsPreserveDIAccessIndexCall(Call, CInfo) ||
689 AIChain.find(x: Call) != AIChain.end())
690 continue;
691
692 traceAICall(Call, ParentInfo&: CInfo);
693 }
694}
695
696/// Get the start and the end of storage offset for \p MemberTy.
697void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy,
698 Align RecordAlignment,
699 uint32_t &StartBitOffset,
700 uint32_t &EndBitOffset) {
701 uint32_t MemberBitSize = MemberTy->getSizeInBits();
702 uint32_t MemberBitOffset = MemberTy->getOffsetInBits();
703
704 if (RecordAlignment > 8) {
705 // If the Bits are within an aligned 8-byte, set the RecordAlignment
706 // to 8, other report the fatal error.
707 if (MemberBitOffset / 64 != (MemberBitOffset + MemberBitSize) / 64)
708 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
709 "requiring too big alignment");
710 RecordAlignment = Align(8);
711 }
712
713 uint32_t AlignBits = RecordAlignment.value() * 8;
714 if (MemberBitSize > AlignBits)
715 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
716 "bitfield size greater than record alignment");
717
718 StartBitOffset = MemberBitOffset & ~(AlignBits - 1);
719 if ((StartBitOffset + AlignBits) < (MemberBitOffset + MemberBitSize))
720 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info, "
721 "cross alignment boundary");
722 EndBitOffset = StartBitOffset + AlignBits;
723}
724
725uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind,
726 DICompositeType *CTy,
727 uint32_t AccessIndex,
728 uint32_t PatchImm,
729 MaybeAlign RecordAlignment) {
730 if (InfoKind == BTF::FIELD_EXISTENCE)
731 return 1;
732
733 uint32_t Tag = CTy->getTag();
734 if (InfoKind == BTF::FIELD_BYTE_OFFSET) {
735 if (Tag == dwarf::DW_TAG_array_type) {
736 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
737 PatchImm += AccessIndex * calcArraySize(CTy, StartDim: 1) *
738 (EltTy->getSizeInBits() >> 3);
739 } else if (Tag == dwarf::DW_TAG_structure_type) {
740 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
741 if (!MemberTy->isBitField()) {
742 PatchImm += MemberTy->getOffsetInBits() >> 3;
743 } else {
744 unsigned SBitOffset, NextSBitOffset;
745 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset,
746 EndBitOffset&: NextSBitOffset);
747 PatchImm += SBitOffset >> 3;
748 }
749 }
750 return PatchImm;
751 }
752
753 if (InfoKind == BTF::FIELD_BYTE_SIZE) {
754 if (Tag == dwarf::DW_TAG_array_type) {
755 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
756 return calcArraySize(CTy, StartDim: 1) * (EltTy->getSizeInBits() >> 3);
757 } else {
758 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
759 uint32_t SizeInBits = MemberTy->getSizeInBits();
760 if (!MemberTy->isBitField())
761 return SizeInBits >> 3;
762
763 unsigned SBitOffset, NextSBitOffset;
764 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset,
765 EndBitOffset&: NextSBitOffset);
766 SizeInBits = NextSBitOffset - SBitOffset;
767 if (SizeInBits & (SizeInBits - 1))
768 report_fatal_error(reason: "Unsupported field expression for llvm.bpf.preserve.field.info");
769 return SizeInBits >> 3;
770 }
771 }
772
773 if (InfoKind == BTF::FIELD_SIGNEDNESS) {
774 const DIType *BaseTy;
775 if (Tag == dwarf::DW_TAG_array_type) {
776 // Signedness only checked when final array elements are accessed.
777 if (CTy->getElements().size() != 1)
778 report_fatal_error(reason: "Invalid array expression for llvm.bpf.preserve.field.info");
779 BaseTy = stripQualifiers(Ty: CTy->getBaseType());
780 } else {
781 auto *MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
782 BaseTy = stripQualifiers(Ty: MemberTy->getBaseType());
783 }
784
785 // Only basic types and enum types have signedness.
786 const auto *BTy = dyn_cast<DIBasicType>(Val: BaseTy);
787 while (!BTy) {
788 const auto *CompTy = dyn_cast<DICompositeType>(Val: BaseTy);
789 // Report an error if the field expression does not have signedness.
790 if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type)
791 report_fatal_error(reason: "Invalid field expression for llvm.bpf.preserve.field.info");
792 BaseTy = stripQualifiers(Ty: CompTy->getBaseType());
793 BTy = dyn_cast<DIBasicType>(Val: BaseTy);
794 }
795 uint32_t Encoding = BTy->getEncoding();
796 return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char);
797 }
798
799 if (InfoKind == BTF::FIELD_LSHIFT_U64) {
800 // The value is loaded into a value with FIELD_BYTE_SIZE size,
801 // and then zero or sign extended to U64.
802 // FIELD_LSHIFT_U64 and FIELD_RSHIFT_U64 are operations
803 // to extract the original value.
804 const Triple &Triple = TM->getTargetTriple();
805 DIDerivedType *MemberTy = nullptr;
806 bool IsBitField = false;
807 uint32_t SizeInBits;
808
809 if (Tag == dwarf::DW_TAG_array_type) {
810 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
811 SizeInBits = calcArraySize(CTy, StartDim: 1) * EltTy->getSizeInBits();
812 } else {
813 MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
814 SizeInBits = MemberTy->getSizeInBits();
815 IsBitField = MemberTy->isBitField();
816 }
817
818 if (!IsBitField) {
819 if (SizeInBits > 64)
820 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
821 return 64 - SizeInBits;
822 }
823
824 unsigned SBitOffset, NextSBitOffset;
825 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset, EndBitOffset&: NextSBitOffset);
826 if (NextSBitOffset - SBitOffset > 64)
827 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
828
829 unsigned OffsetInBits = MemberTy->getOffsetInBits();
830 if (Triple.getArch() == Triple::bpfel)
831 return SBitOffset + 64 - OffsetInBits - SizeInBits;
832 else
833 return OffsetInBits + 64 - NextSBitOffset;
834 }
835
836 if (InfoKind == BTF::FIELD_RSHIFT_U64) {
837 DIDerivedType *MemberTy = nullptr;
838 bool IsBitField = false;
839 uint32_t SizeInBits;
840 if (Tag == dwarf::DW_TAG_array_type) {
841 auto *EltTy = stripQualifiers(Ty: CTy->getBaseType());
842 SizeInBits = calcArraySize(CTy, StartDim: 1) * EltTy->getSizeInBits();
843 } else {
844 MemberTy = cast<DIDerivedType>(Val: CTy->getElements()[AccessIndex]);
845 SizeInBits = MemberTy->getSizeInBits();
846 IsBitField = MemberTy->isBitField();
847 }
848
849 if (!IsBitField) {
850 if (SizeInBits > 64)
851 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
852 return 64 - SizeInBits;
853 }
854
855 unsigned SBitOffset, NextSBitOffset;
856 GetStorageBitRange(MemberTy, RecordAlignment: *RecordAlignment, StartBitOffset&: SBitOffset, EndBitOffset&: NextSBitOffset);
857 if (NextSBitOffset - SBitOffset > 64)
858 report_fatal_error(reason: "too big field size for llvm.bpf.preserve.field.info");
859
860 return 64 - SizeInBits;
861 }
862
863 llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind");
864}
865
866/// Consume leading array accesses, leaving the first record access on the
867/// stack.
868std::optional<BPFAbstractMemberAccess::AccessRoot>
869BPFAbstractMemberAccess::consumeAccessRoot(
870 SmallVectorImpl<CallInfo> &CallStack) {
871 // Fold the leading array accesses into one index, counted in units of the
872 // innermost element, such as struct S in Row a[2] with
873 // typedef struct S Row[3].
874 uint32_t FirstIndex = 0;
875 DIType *ElementTy = nullptr;
876 while (!CallStack.empty() && CallStack.back().Kind == BPFPreserveArrayAI) {
877 const CallInfo &CInfo = CallStack.back();
878 auto *Ty = stripQualifiers(Ty: cast<DIType>(Val: CInfo.Metadata));
879 uint32_t Stride;
880 switch (Ty->getTag()) {
881 case dwarf::DW_TAG_array_type: {
882 auto *Array = cast<DICompositeType>(Val: Ty);
883 ElementTy = Array->getBaseType();
884 Stride = calcArraySize(CTy: Array, StartDim: 1) * calcElementCount(Ty&: ElementTy);
885 break;
886 }
887 case dwarf::DW_TAG_pointer_type:
888 ElementTy = cast<DIDerivedType>(Val: Ty)->getBaseType();
889 Stride = calcElementCount(Ty&: ElementTy);
890 break;
891 default:
892 llvm_unreachable("Expected an array or pointer type");
893 }
894 FirstIndex += CInfo.AccessIndex * Stride;
895 CallStack.pop_back();
896 }
897
898 if (!ElementTy) {
899 // A typedef may be the only name the loader can match for this record.
900 const CallInfo &CInfo = CallStack.back();
901 assert(CInfo.Kind == BPFPreserveStructAI ||
902 CInfo.Kind == BPFPreserveUnionAI);
903 return AccessRoot{
904 .Type: stripQualifiers(Ty: cast<DIType>(Val: CInfo.Metadata), /*skipTypedef=*/false),
905 .Index: 0};
906 }
907
908 // The innermost element supplies the relocation root.
909 auto *Record = stripQualifiers(Ty: ElementTy);
910 switch (Record->getTag()) {
911 case dwarf::DW_TAG_structure_type:
912 case dwarf::DW_TAG_union_type:
913 // An anonymous record can only be matched by its typedef name.
914 return AccessRoot{.Type: Record->getName().empty()
915 ? stripQualifiers(Ty: ElementTy, /*skipTypedef=*/false)
916 : Record,
917 .Index: FirstIndex};
918 default:
919 return std::nullopt;
920 }
921}
922
923static std::string formatAmaSuffix(uint32_t Kind, int64_t PatchImm,
924 const Twine &AccessString) {
925 return (":" + Twine(Kind) + ":" + Twine(PatchImm) + "$" + AccessString).str();
926}
927
928/// Compute the base pointer, root type and relocation suffix of a field access.
929Value *BPFAbstractMemberAccess::computeFieldAccess(CallInst *Call,
930 const CallInfo &Terminal,
931 std::string &RelocSuffix,
932 DIType *&RootType) {
933 // Put the access chain into a stack with the top as the head of the chain.
934 // A terminal preserve_field_info call is not part of the chain.
935 const bool IsFieldInfo = Terminal.Kind == BPFPreserveFieldInfoAI;
936 SmallVector<CallInfo, 4> CallStack;
937 if (!IsFieldInfo)
938 CallStack.push_back(Elt: Terminal);
939 for (auto It = AIChain.find(x: Call); It != AIChain.end();
940 It = AIChain.find(x: It->second.first))
941 CallStack.push_back(Elt: It->second.second);
942
943 Value *const Base = CallStack.back().Base;
944 const auto Root = consumeAccessRoot(CallStack);
945 if (!Root) {
946 if (IsFieldInfo)
947 report_fatal_error(
948 reason: "Invalid field access for llvm.preserve.field.info intrinsic");
949 // With no record root, accesses such as &p[4] (int *p) or &a[2][3]
950 // (int a[10][20]) lower to ordinary GEPs.
951 return nullptr;
952 }
953 assert(!Root->Type->getName().empty());
954 const uint32_t InfoKind =
955 IsFieldInfo ? Terminal.AccessIndex : BTF::FIELD_BYTE_OFFSET;
956 uint32_t PatchImm =
957 Root->Index * (stripQualifiers(Ty: Root->Type)->getSizeInBits() >> 3);
958 std::string AccessString = std::to_string(val: Root->Index);
959
960 // Build the access string and compute the local byte offset or requested
961 // field property while each access still has its debug type.
962 ArrayRef<CallInfo> Fields = CallStack;
963 while (!Fields.empty()) {
964 const CallInfo &CInfo = Fields.back();
965 Fields = Fields.drop_back();
966
967 // Access Index
968 uint64_t AccessIndex = CInfo.AccessIndex;
969 MDNode *MDN = CInfo.Metadata;
970 // At this stage, it cannot be pointer type.
971 auto *CTy = cast<DICompositeType>(Val: stripQualifiers(Ty: cast<DIType>(Val: MDN)));
972
973 uint64_t BTFIndex = AccessIndex;
974 if (CTy->getTag() == dwarf::DW_TAG_structure_type) {
975 DINodeArray Elements = CTy->getElements();
976 uint64_t Offset = getBTFRecordElementOffset(Element: Elements[AccessIndex]);
977 // Find this element's position in the stable offset order without
978 // sorting the whole record for every CO-RE access.
979 BTFIndex = 0;
980 for (unsigned I = 0; I < Elements.size(); ++I) {
981 uint64_t ElementOffset = getBTFRecordElementOffset(Element: Elements[I]);
982 if (ElementOffset < Offset ||
983 (ElementOffset == Offset && I < AccessIndex))
984 ++BTFIndex;
985 }
986 }
987 AccessString += ":" + std::to_string(val: BTFIndex);
988
989 // Earlier accesses contribute byte offsets. Only the final field supplies
990 // the property requested by preserve_field_info.
991 const uint32_t FieldKind =
992 Fields.empty() ? InfoKind : BTF::FIELD_BYTE_OFFSET;
993 PatchImm = GetFieldInfo(InfoKind: FieldKind, CTy, AccessIndex, PatchImm,
994 RecordAlignment: CInfo.RecordAlignment);
995 }
996
997 if (InfoKind == BTF::FIELD_EXISTENCE)
998 PatchImm = 1;
999
1000 RootType = Root->Type;
1001 RelocSuffix = formatAmaSuffix(Kind: InfoKind, PatchImm, AccessString);
1002
1003 return Base;
1004}
1005
1006BPFAbstractMemberAccess::EnumAccess
1007BPFAbstractMemberAccess::computeEnumAccess(const CallInst *Call,
1008 const DIType *Ty) {
1009 // The argument can be a global or a GEP into a global.
1010 const auto *GV =
1011 cast<GlobalVariable>(Val: Call->getArgOperand(i: 1)->stripPointerCasts());
1012 assert(GV->hasInitializer());
1013 const auto *DA = cast<ConstantDataArray>(Val: GV->getInitializer());
1014 assert(DA->isString());
1015 StringRef ValueStr = DA->getAsString();
1016 size_t Separator = ValueStr.find(C: ':');
1017 StringRef Enumerator = ValueStr.substr(Start: 0, N: Separator);
1018 StringRef Value = ValueStr.substr(Start: Separator + 1);
1019
1020 const auto *CTy = cast<DICompositeType>(Val: stripQualifiers(Ty));
1021 assert(CTy->getTag() == dwarf::DW_TAG_enumeration_type);
1022 for (auto [Index, Element] : enumerate(First: CTy->getElements()))
1023 if (cast<DIEnumerator>(Val: Element)->getName() == Enumerator)
1024 return {.Index: Index, .Value: Value};
1025 return {.Index: 0, .Value: Value};
1026}
1027
1028DIType *BPFAbstractMemberAccess::computeTypeAccess(CallInst *Call,
1029 const CallInfo &CInfo,
1030 std::string &RelocSuffix) {
1031 DIType *Ty = stripQualifiers(Ty: cast<DIType>(Val: CInfo.Metadata), skipTypedef: false);
1032 assert(!Ty->getName().empty());
1033
1034 int64_t PatchImm = 1;
1035 size_t AccessIndex = 0;
1036 switch (CInfo.AccessIndex) {
1037 case BTF::TYPE_EXISTENCE:
1038 case BTF::TYPE_MATCH:
1039 break;
1040 case BTF::TYPE_SIZE:
1041 // typedef debuginfo type has size 0, get the eventual base type.
1042 PatchImm = stripQualifiers(Ty)->getSizeInBits() / 8;
1043 break;
1044 case BTF::ENUM_VALUE_EXISTENCE:
1045 AccessIndex = computeEnumAccess(Call, Ty).Index;
1046 break;
1047 case BTF::ENUM_VALUE: {
1048 auto [Index, Value] = computeEnumAccess(Call, Ty);
1049 PatchImm = std::stoll(str: Value.str());
1050 AccessIndex = Index;
1051 break;
1052 }
1053 default:
1054 llvm_unreachable("Unexpected BPF type relocation kind");
1055 }
1056
1057 RelocSuffix =
1058 formatAmaSuffix(Kind: CInfo.AccessIndex, PatchImm, AccessString: Twine(AccessIndex));
1059
1060 return Ty;
1061}
1062
1063static GlobalVariable *getOrCreateAmaGlobal(Module &M, DIType *Root,
1064 IntegerType *ValueType,
1065 StringRef RelocSuffix) {
1066 const std::string Prefix = ("llvm." + Root->getName()).str();
1067 std::string Name = (Twine(Prefix) + RelocSuffix).str();
1068 for (unsigned N = 1; GlobalValue *Existing = M.getNamedValue(Name); ++N) {
1069 auto *GV = dyn_cast<GlobalVariable>(Val: Existing);
1070 if (GV && GV->isDeclaration() && GV->hasExternalLinkage() &&
1071 GV->hasAttribute(Kind: BPFCoreSharedInfo::AmaAttr) &&
1072 GV->getValueType() == ValueType &&
1073 GV->getMetadata(KindID: LLVMContext::MD_preserve_access_index) == Root)
1074 return GV;
1075 // Same-named roots and different load widths need distinct globals. Keep
1076 // the discriminator before the suffix that the backend will decode.
1077 Name = (Twine(Prefix) + "." + Twine(N) + RelocSuffix).str();
1078 }
1079
1080 auto *GV = new GlobalVariable(M, ValueType, false,
1081 GlobalValue::ExternalLinkage, nullptr, Name);
1082 GV->addAttribute(Kind: BPFCoreSharedInfo::AmaAttr);
1083 GV->setMetadata(KindID: LLVMContext::MD_preserve_access_index, Node: Root);
1084 return GV;
1085}
1086
1087/// Call/Kind is the base preserve_*_access_index() call. Attempts to do
1088/// transformation to a chain of relocable GEPs.
1089bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call,
1090 const CallInfo &CInfo) {
1091 std::string RelocSuffix;
1092 DIType *Root;
1093 Value *Base = nullptr;
1094
1095 const bool IsFieldInfo = CInfo.Kind == BPFPreserveFieldInfoAI;
1096 if (IsFieldInfo && CInfo.Metadata) {
1097 Root = computeTypeAccess(Call, CInfo, RelocSuffix);
1098 } else {
1099 Base = computeFieldAccess(Call, Terminal: CInfo, RelocSuffix, RootType&: Root);
1100 if (!Base)
1101 return false;
1102 }
1103
1104 BasicBlock *BB = Call->getParent();
1105 auto *ValueType = IsFieldInfo ? cast<IntegerType>(Val: Call->getType())
1106 : Type::getInt64Ty(C&: Call->getContext());
1107 auto *GV = getOrCreateAmaGlobal(M&: *M, Root, ValueType, RelocSuffix);
1108 Instruction *Input = new LoadInst(ValueType, GV, "", Call->getIterator());
1109 if (!IsFieldInfo) {
1110 auto *GEP = GetElementPtrInst::Create(PointeeType: Type::getInt8Ty(C&: Call->getContext()),
1111 Ptr: Base, IdxList: Input);
1112 GEP->insertBefore(InsertPos: Call->getIterator());
1113 Input = GEP;
1114 }
1115
1116 // For the following code,
1117 // Block0:
1118 // ...
1119 // if (...) goto Block1 else ...
1120 // Block1:
1121 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0
1122 // %8 = getelementptr i8, i8* %2, %6
1123 // ...
1124 // goto CommonExit
1125 // Block2:
1126 // ...
1127 // if (...) goto Block3 else ...
1128 // Block3:
1129 // %6 = load llvm.bpf_map:0:40$0:0:0:2:0
1130 // %8 = getelementptr i8, i8* %2, %6
1131 // ...
1132 // goto CommonExit
1133 // CommonExit
1134 // SimplifyCFG may generate:
1135 // Block0:
1136 // ...
1137 // if (...) goto Block_Common else ...
1138 // Block2:
1139 // ...
1140 // if (...) goto Block_Common else ...
1141 // Block_Common:
1142 // PHI = [llvm.sk_buff:0:50$0:0:0:2:0, llvm.bpf_map:0:40$0:0:0:2:0]
1143 // %6 = load PHI
1144 // %8 = getelementptr i8, i8* %2, %6
1145 // ...
1146 // goto CommonExit
1147 // For the above code, we cannot perform proper relocation since
1148 // "load PHI" has two possible relocations.
1149 //
1150 // To prevent above tail merging, we use __builtin_bpf_passthrough()
1151 // where one of its parameters is a seq_num. Since two
1152 // __builtin_bpf_passthrough() funcs will always have different seq_num,
1153 // tail merging cannot happen. The __builtin_bpf_passthrough() will be
1154 // removed in the beginning of Target IR passes.
1155 //
1156 // This approach is also used in other places when global var
1157 // representing a relocation is used.
1158 Instruction *PassThroughInst =
1159 BPFCoreSharedInfo::insertPassThrough(M, BB, Input, Before: Call);
1160 Call->replaceAllUsesWith(V: PassThroughInst);
1161 Call->eraseFromParent();
1162
1163 return true;
1164}
1165
1166bool BPFAbstractMemberAccess::doTransformation(Function &F) {
1167 bool Transformed = false;
1168
1169 // Collect PreserveDIAccessIndex Intrinsic call chains.
1170 // The call chains will be used to generate the access
1171 // patterns similar to GEP.
1172 collectAICallChains(F);
1173
1174 for (auto &C : BaseAICalls)
1175 Transformed = transformGEPChain(Call: C.first, CInfo: C.second) || Transformed;
1176
1177 return removePreserveAccessIndexIntrinsic(F) || Transformed;
1178}
1179
1180PreservedAnalyses
1181BPFAbstractMemberAccessPass::run(Function &F, FunctionAnalysisManager &AM) {
1182 return BPFAbstractMemberAccess(TM).run(F) ? PreservedAnalyses::none()
1183 : PreservedAnalyses::all();
1184}
1185