1//===------------ BPFCheckAndAdjustIR.cpp - Check and Adjust IR -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Check IR and adjust IR for verifier friendly codes.
10// The following are done for IR checking:
11// - no relocation globals in PHI node.
12// The following are done for IR adjustment:
13// - remove __builtin_bpf_passthrough builtins. Target independent IR
14// optimizations are done and those builtins can be removed.
15// - remove llvm.bpf.getelementptr.and.load builtins.
16// - remove llvm.bpf.getelementptr.and.store builtins.
17// - for loads and stores with base addresses from non-zero address space
18// cast base address to zero address space (support for BPF address spaces).
19//
20//===----------------------------------------------------------------------===//
21
22#include "BPF.h"
23#include "BPFCORE.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/IR/Analysis.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Instruction.h"
29#include "llvm/IR/Instructions.h"
30#include "llvm/IR/IntrinsicInst.h"
31#include "llvm/IR/IntrinsicsBPF.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/PassManager.h"
34#include "llvm/IR/ProfDataUtils.h"
35#include "llvm/IR/Type.h"
36#include "llvm/IR/Value.h"
37#include "llvm/Pass.h"
38#include "llvm/Transforms/Utils/BasicBlockUtils.h"
39
40#define DEBUG_TYPE "bpf-check-and-opt-ir"
41
42using namespace llvm;
43
44namespace {
45
46class BPFCheckAndAdjustIRLegacy final : public ModulePass {
47 bool runOnModule(Module &F) override;
48
49public:
50 static char ID;
51 BPFCheckAndAdjustIRLegacy() : ModulePass(ID) {}
52 void getAnalysisUsage(AnalysisUsage &AU) const override;
53};
54} // End anonymous namespace
55
56char BPFCheckAndAdjustIRLegacy::ID = 0;
57INITIALIZE_PASS(BPFCheckAndAdjustIRLegacy, DEBUG_TYPE,
58 "BPF Check And Adjust IR", false, false)
59
60ModulePass *llvm::createBPFCheckAndAdjustIRLegacyPass() {
61 return new BPFCheckAndAdjustIRLegacy();
62}
63
64static void checkIR(Module &M) {
65 // Ensure relocation global won't appear in PHI node
66 // This may happen if the compiler generated the following code:
67 // B1:
68 // g1 = @llvm.skb_buff:0:1...
69 // ...
70 // goto B_COMMON
71 // B2:
72 // g2 = @llvm.skb_buff:0:2...
73 // ...
74 // goto B_COMMON
75 // B_COMMON:
76 // g = PHI(g1, g2)
77 // x = load g
78 // ...
79 // If anything likes the above "g = PHI(g1, g2)", issue a fatal error.
80 for (Function &F : M)
81 for (auto &BB : F)
82 for (auto &I : BB) {
83 PHINode *PN = dyn_cast<PHINode>(Val: &I);
84 if (!PN || PN->use_empty())
85 continue;
86 for (int i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
87 auto *GV = dyn_cast<GlobalVariable>(Val: PN->getIncomingValue(i));
88 if (!GV)
89 continue;
90 if (GV->hasAttribute(Kind: BPFCoreSharedInfo::AmaAttr) ||
91 GV->hasAttribute(Kind: BPFCoreSharedInfo::TypeIdAttr))
92 report_fatal_error(reason: "relocation global in PHI node");
93 }
94 }
95}
96
97static bool removePassThroughBuiltin(Module &M) {
98 // Remove __builtin_bpf_passthrough()'s which are used to prevent
99 // certain IR optimizations. Now major IR optimizations are done,
100 // remove them.
101 bool Changed = false;
102 CallInst *ToBeDeleted = nullptr;
103 for (Function &F : M)
104 for (auto &BB : F)
105 for (auto &I : BB) {
106 if (ToBeDeleted) {
107 ToBeDeleted->eraseFromParent();
108 ToBeDeleted = nullptr;
109 }
110
111 auto *Call = dyn_cast<CallInst>(Val: &I);
112 if (!Call)
113 continue;
114 auto *GV = dyn_cast<GlobalValue>(Val: Call->getCalledOperand());
115 if (!GV)
116 continue;
117 if (!GV->getName().starts_with(Prefix: "llvm.bpf.passthrough"))
118 continue;
119 Changed = true;
120 Value *Arg = Call->getArgOperand(i: 1);
121 Call->replaceAllUsesWith(V: Arg);
122 ToBeDeleted = Call;
123 }
124 return Changed;
125}
126
127static bool removeCompareBuiltin(Module &M) {
128 // Remove __builtin_bpf_compare()'s which are used to prevent
129 // certain IR optimizations. Now major IR optimizations are done,
130 // remove them.
131 bool Changed = false;
132 CallInst *ToBeDeleted = nullptr;
133 for (Function &F : M)
134 for (auto &BB : F)
135 for (auto &I : BB) {
136 if (ToBeDeleted) {
137 ToBeDeleted->eraseFromParent();
138 ToBeDeleted = nullptr;
139 }
140
141 auto *Call = dyn_cast<CallInst>(Val: &I);
142 if (!Call)
143 continue;
144 auto *GV = dyn_cast<GlobalValue>(Val: Call->getCalledOperand());
145 if (!GV)
146 continue;
147 if (!GV->getName().starts_with(Prefix: "llvm.bpf.compare"))
148 continue;
149
150 Changed = true;
151 Value *Arg0 = Call->getArgOperand(i: 0);
152 Value *Arg1 = Call->getArgOperand(i: 1);
153 Value *Arg2 = Call->getArgOperand(i: 2);
154
155 auto OpVal = cast<ConstantInt>(Val: Arg0)->getValue().getZExtValue();
156 CmpInst::Predicate Opcode = (CmpInst::Predicate)OpVal;
157
158 auto *ICmp = new ICmpInst(Opcode, Arg1, Arg2);
159 ICmp->insertBefore(InsertPos: Call->getIterator());
160
161 Call->replaceAllUsesWith(V: ICmp);
162 ToBeDeleted = Call;
163 }
164 return Changed;
165}
166
167struct MinMaxSinkInfo {
168 ICmpInst *ICmp;
169 Value *Other;
170 ICmpInst::Predicate Predicate;
171 CallInst *MinMax;
172 ZExtInst *ZExt;
173 SExtInst *SExt;
174
175 MinMaxSinkInfo(ICmpInst *ICmp, Value *Other, ICmpInst::Predicate Predicate)
176 : ICmp(ICmp), Other(Other), Predicate(Predicate), MinMax(nullptr),
177 ZExt(nullptr), SExt(nullptr) {}
178};
179
180static bool sinkMinMaxInBB(BasicBlock &BB,
181 const std::function<bool(Instruction *)> &Filter) {
182 // Check if V is:
183 // (fn %a %b) or (ext (fn %a %b))
184 // Where:
185 // ext := sext | zext
186 // fn := smin | umin | smax | umax
187 auto IsMinMaxCall = [=](Value *V, MinMaxSinkInfo &Info) {
188 if (auto *ZExt = dyn_cast<ZExtInst>(Val: V)) {
189 V = ZExt->getOperand(i_nocapture: 0);
190 Info.ZExt = ZExt;
191 } else if (auto *SExt = dyn_cast<SExtInst>(Val: V)) {
192 V = SExt->getOperand(i_nocapture: 0);
193 Info.SExt = SExt;
194 }
195
196 auto *Call = dyn_cast<CallInst>(Val: V);
197 if (!Call)
198 return false;
199
200 auto *Called = dyn_cast<Function>(Val: Call->getCalledOperand());
201 if (!Called)
202 return false;
203
204 switch (Called->getIntrinsicID()) {
205 case Intrinsic::smin:
206 case Intrinsic::umin:
207 case Intrinsic::smax:
208 case Intrinsic::umax:
209 break;
210 default:
211 return false;
212 }
213
214 if (!Filter(Call))
215 return false;
216
217 Info.MinMax = Call;
218
219 return true;
220 };
221
222 auto ZeroOrSignExtend = [](IRBuilder<> &Builder, Value *V,
223 MinMaxSinkInfo &Info) {
224 if (Info.SExt) {
225 if (Info.SExt->getType() == V->getType())
226 return V;
227 return Builder.CreateSExt(V, DestTy: Info.SExt->getType());
228 }
229 if (Info.ZExt) {
230 if (Info.ZExt->getType() == V->getType())
231 return V;
232 return Builder.CreateZExt(V, DestTy: Info.ZExt->getType());
233 }
234 return V;
235 };
236
237 bool Changed = false;
238 SmallVector<MinMaxSinkInfo, 2> SinkList;
239
240 // Check BB for instructions like:
241 // insn := (icmp %a (fn ...)) | (icmp (fn ...) %a)
242 //
243 // Where:
244 // fn := min | max | (sext (min ...)) | (sext (max ...))
245 //
246 // Put such instructions to SinkList.
247 for (Instruction &I : BB) {
248 ICmpInst *ICmp = dyn_cast<ICmpInst>(Val: &I);
249 if (!ICmp)
250 continue;
251 if (!ICmp->isRelational())
252 continue;
253 MinMaxSinkInfo First(ICmp, ICmp->getOperand(i_nocapture: 1),
254 ICmpInst::getSwappedPredicate(pred: ICmp->getPredicate()));
255 MinMaxSinkInfo Second(ICmp, ICmp->getOperand(i_nocapture: 0), ICmp->getPredicate());
256 bool FirstMinMax = IsMinMaxCall(ICmp->getOperand(i_nocapture: 0), First);
257 bool SecondMinMax = IsMinMaxCall(ICmp->getOperand(i_nocapture: 1), Second);
258 if (!(FirstMinMax ^ SecondMinMax))
259 continue;
260 SinkList.push_back(Elt: FirstMinMax ? First : Second);
261 }
262
263 // Iterate SinkList and replace each (icmp ...) with corresponding
264 // `x < a && x < b` or similar expression.
265 for (auto &Info : SinkList) {
266 ICmpInst *ICmp = Info.ICmp;
267 CallInst *MinMax = Info.MinMax;
268 Intrinsic::ID IID = MinMax->getCalledFunction()->getIntrinsicID();
269 ICmpInst::Predicate P = Info.Predicate;
270 if (ICmpInst::isSigned(Pred: P) && IID != Intrinsic::smin &&
271 IID != Intrinsic::smax)
272 continue;
273
274 IRBuilder<> Builder(ICmp);
275 Value *X = Info.Other;
276 Value *A = ZeroOrSignExtend(Builder, MinMax->getArgOperand(i: 0), Info);
277 Value *B = ZeroOrSignExtend(Builder, MinMax->getArgOperand(i: 1), Info);
278 bool IsMin = IID == Intrinsic::smin || IID == Intrinsic::umin;
279 bool IsMax = IID == Intrinsic::smax || IID == Intrinsic::umax;
280 bool IsLess = ICmpInst::isLE(P) || ICmpInst::isLT(P);
281 bool IsGreater = ICmpInst::isGE(P) || ICmpInst::isGT(P);
282 assert(IsMin ^ IsMax);
283 assert(IsLess ^ IsGreater);
284
285 Value *Replacement;
286 Value *LHS = Builder.CreateICmp(P, LHS: X, RHS: A);
287 Value *RHS = Builder.CreateICmp(P, LHS: X, RHS: B);
288 if ((IsLess && IsMin) || (IsGreater && IsMax))
289 // x < min(a, b) -> x < a && x < b
290 // x > max(a, b) -> x > a && x > b
291 Replacement = Builder.CreateLogicalAnd(Cond1: LHS, Cond2: RHS);
292 else
293 // x > min(a, b) -> x > a || x > b
294 // x < max(a, b) -> x < a || x < b
295 Replacement = Builder.CreateLogicalOr(Cond1: LHS, Cond2: RHS);
296
297 if (SelectInst *ReplacementInst = dyn_cast<SelectInst>(Val: Replacement))
298 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *ReplacementInst, DEBUG_TYPE);
299
300 ICmp->replaceAllUsesWith(V: Replacement);
301
302 Instruction *ToRemove[] = {ICmp, Info.ZExt, Info.SExt, MinMax};
303 for (Instruction *I : ToRemove)
304 if (I && I->use_empty())
305 I->eraseFromParent();
306
307 Changed = true;
308 }
309
310 return Changed;
311}
312
313// Do the following transformation:
314//
315// x < min(a, b) -> x < a && x < b
316// x > min(a, b) -> x > a || x > b
317// x < max(a, b) -> x < a || x < b
318// x > max(a, b) -> x > a && x > b
319//
320// Such patterns are introduced by LICM.cpp:hoistMinMax()
321// transformation and might lead to BPF verification failures for
322// older kernels.
323//
324// To minimize "collateral" changes only do it for icmp + min/max
325// calls when icmp is inside a loop and min/max is outside of that
326// loop.
327//
328// Verification failure happens when:
329// - RHS operand of some `icmp LHS, RHS` is replaced by some RHS1;
330// - verifier can recognize RHS as a constant scalar in some context;
331// - verifier can't recognize RHS1 as a constant scalar in the same
332// context;
333//
334// The "constant scalar" is not a compile time constant, but a register
335// that holds a scalar value known to verifier at some point in time
336// during abstract interpretation.
337//
338// See also:
339// https://lore.kernel.org/bpf/20230406164505.1046801-1-yhs@fb.com/
340static bool sinkMinMax(Module &M,
341 function_ref<LoopInfo &(Function &)> GetLoopInfo) {
342 bool Changed = false;
343
344 for (Function &F : M) {
345 if (F.isDeclaration())
346 continue;
347
348 LoopInfo &LI = GetLoopInfo(F);
349 for (Loop *L : LI)
350 for (BasicBlock *BB : L->blocks()) {
351 // Filter out instructions coming from the same loop
352 Loop *BBLoop = LI.getLoopFor(BB);
353 auto OtherLoopFilter = [&](Instruction *I) {
354 return LI.getLoopFor(BB: I->getParent()) != BBLoop;
355 };
356 Changed |= sinkMinMaxInBB(BB&: *BB, Filter: OtherLoopFilter);
357 }
358 }
359
360 return Changed;
361}
362
363void BPFCheckAndAdjustIRLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
364 AU.addRequired<LoopInfoWrapperPass>();
365}
366
367static void unrollGEPLoad(CallInst *Call) {
368 auto [GEP, Load] = BPFPreserveStaticOffsetPass::reconstructLoad(Call);
369 GEP->insertBefore(InsertPos: Call->getIterator());
370 Load->insertBefore(InsertPos: Call->getIterator());
371 Call->replaceAllUsesWith(V: Load);
372 Call->eraseFromParent();
373}
374
375static void unrollGEPStore(CallInst *Call) {
376 auto [GEP, Store] = BPFPreserveStaticOffsetPass::reconstructStore(Call);
377 GEP->insertBefore(InsertPos: Call->getIterator());
378 Store->insertBefore(InsertPos: Call->getIterator());
379 Call->eraseFromParent();
380}
381
382static bool removeGEPBuiltinsInFunc(Function &F) {
383 SmallVector<CallInst *> GEPLoads;
384 SmallVector<CallInst *> GEPStores;
385 for (auto &BB : F)
386 for (auto &Insn : BB)
387 if (auto *Call = dyn_cast<CallInst>(Val: &Insn))
388 if (auto *Called = Call->getCalledFunction())
389 switch (Called->getIntrinsicID()) {
390 case Intrinsic::bpf_getelementptr_and_load:
391 GEPLoads.push_back(Elt: Call);
392 break;
393 case Intrinsic::bpf_getelementptr_and_store:
394 GEPStores.push_back(Elt: Call);
395 break;
396 }
397
398 if (GEPLoads.empty() && GEPStores.empty())
399 return false;
400
401 for_each(Range&: GEPLoads, F: unrollGEPLoad);
402 for_each(Range&: GEPStores, F: unrollGEPStore);
403
404 return true;
405}
406
407// Rewrites the following builtins:
408// - llvm.bpf.getelementptr.and.load
409// - llvm.bpf.getelementptr.and.store
410// As (load (getelementptr ...)) or (store (getelementptr ...)).
411static bool removeGEPBuiltins(Module &M) {
412 bool Changed = false;
413 for (auto &F : M)
414 Changed = removeGEPBuiltinsInFunc(F) || Changed;
415 return Changed;
416}
417
418// Wrap ToWrap with cast to address space zero:
419// - if ToWrap is a getelementptr,
420// wrap it's base pointer instead and return a copy;
421// - if ToWrap is Instruction, insert address space cast
422// immediately after ToWrap;
423// - if ToWrap is not an Instruction (function parameter
424// or a global value), insert address space cast at the
425// beginning of the Function F;
426// - use Cache to avoid inserting too many casts;
427static Value *aspaceWrapValue(DenseMap<Value *, Value *> &Cache, Function *F,
428 Value *ToWrap) {
429 auto It = Cache.find(Val: ToWrap);
430 if (It != Cache.end())
431 return It->getSecond();
432
433 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: ToWrap)) {
434 Value *Ptr = GEP->getPointerOperand();
435 Value *WrappedPtr = aspaceWrapValue(Cache, F, ToWrap: Ptr);
436 auto *GEPTy = cast<PointerType>(Val: GEP->getType());
437 auto *NewGEP = GEP->clone();
438 NewGEP->insertAfter(InsertPos: GEP->getIterator());
439 NewGEP->mutateType(Ty: PointerType::getUnqual(C&: GEPTy->getContext()));
440 NewGEP->setOperand(i: GEP->getPointerOperandIndex(), Val: WrappedPtr);
441 NewGEP->setName(GEP->getName());
442 Cache[ToWrap] = NewGEP;
443 return NewGEP;
444 }
445
446 IRBuilder IB(F->getContext());
447 if (Instruction *InsnPtr = dyn_cast<Instruction>(Val: ToWrap))
448 IB.SetInsertPoint(*InsnPtr->getInsertionPointAfterDef());
449 else
450 IB.SetInsertPoint(F->getEntryBlock().getFirstInsertionPt());
451 auto *ASZeroPtrTy = IB.getPtrTy(AddrSpace: 0);
452 auto *ACast = IB.CreateAddrSpaceCast(V: ToWrap, DestTy: ASZeroPtrTy, Name: ToWrap->getName());
453 Cache[ToWrap] = ACast;
454 return ACast;
455}
456
457// Wrap a pointer operand OpNum of instruction I
458// with cast to address space zero
459static void aspaceWrapOperand(DenseMap<Value *, Value *> &Cache, Instruction *I,
460 unsigned OpNum) {
461 Value *OldOp = I->getOperand(i: OpNum);
462 if (OldOp->getType()->getPointerAddressSpace() == 0)
463 return;
464
465 Value *NewOp = aspaceWrapValue(Cache, F: I->getFunction(), ToWrap: OldOp);
466 I->setOperand(i: OpNum, Val: NewOp);
467 // Check if there are any remaining users of old GEP,
468 // delete those w/o users
469 for (;;) {
470 auto *OldGEP = dyn_cast<GetElementPtrInst>(Val: OldOp);
471 if (!OldGEP)
472 break;
473 if (!OldGEP->use_empty())
474 break;
475 OldOp = OldGEP->getPointerOperand();
476 OldGEP->eraseFromParent();
477 }
478}
479
480static Value *wrapPtrIfASNotZero(DenseMap<Value *, Value *> &Cache,
481 CallInst *CI, Value *P) {
482 if (auto *PTy = dyn_cast<PointerType>(Val: P->getType())) {
483 if (PTy->getAddressSpace() == 0)
484 return P;
485 }
486 return aspaceWrapValue(Cache, F: CI->getFunction(), ToWrap: P);
487}
488
489static Instruction *aspaceMemSet(Intrinsic::ID ID,
490 DenseMap<Value *, Value *> &Cache,
491 CallInst *CI) {
492 auto *MI = cast<MemIntrinsic>(Val: CI);
493 IRBuilder<> B(CI);
494
495 Value *OldDst = CI->getArgOperand(i: 0);
496 Value *NewDst = wrapPtrIfASNotZero(Cache, CI, P: OldDst);
497 if (OldDst == NewDst)
498 return nullptr;
499
500 // memset(new_dst, val, len, align, isvolatile, md)
501 Value *Val = CI->getArgOperand(i: 1);
502 Value *Len = CI->getArgOperand(i: 2);
503
504 auto *MS = cast<MemSetInst>(Val: CI);
505 MaybeAlign Align = MS->getDestAlign();
506 bool IsVolatile = MS->isVolatile();
507
508 if (ID == Intrinsic::memset)
509 return B.CreateMemSet(Ptr: NewDst, Val, Size: Len, Align, isVolatile: IsVolatile,
510 AAInfo: MI->getAAMetadata());
511 else
512 return B.CreateMemSetInline(Dst: NewDst, DstAlign: Align, Val, Size: Len, IsVolatile,
513 AAInfo: MI->getAAMetadata());
514}
515
516static Instruction *aspaceMemCpy(Intrinsic::ID ID,
517 DenseMap<Value *, Value *> &Cache,
518 CallInst *CI) {
519 auto *MI = cast<MemIntrinsic>(Val: CI);
520 IRBuilder<> B(CI);
521
522 Value *OldDst = CI->getArgOperand(i: 0);
523 Value *OldSrc = CI->getArgOperand(i: 1);
524 Value *NewDst = wrapPtrIfASNotZero(Cache, CI, P: OldDst);
525 Value *NewSrc = wrapPtrIfASNotZero(Cache, CI, P: OldSrc);
526 if (OldDst == NewDst && OldSrc == NewSrc)
527 return nullptr;
528
529 // memcpy(new_dst, dst_align, new_src, src_align, len, isvolatile, md)
530 Value *Len = CI->getArgOperand(i: 2);
531
532 auto *MT = cast<MemTransferInst>(Val: CI);
533 MaybeAlign DstAlign = MT->getDestAlign();
534 MaybeAlign SrcAlign = MT->getSourceAlign();
535 bool IsVolatile = MT->isVolatile();
536
537 return B.CreateMemTransferInst(IntrID: ID, Dst: NewDst, DstAlign, Src: NewSrc, SrcAlign, Size: Len,
538 isVolatile: IsVolatile, AAInfo: MI->getAAMetadata());
539}
540
541static Instruction *aspaceMemMove(DenseMap<Value *, Value *> &Cache,
542 CallInst *CI) {
543 auto *MI = cast<MemIntrinsic>(Val: CI);
544 IRBuilder<> B(CI);
545
546 Value *OldDst = CI->getArgOperand(i: 0);
547 Value *OldSrc = CI->getArgOperand(i: 1);
548 Value *NewDst = wrapPtrIfASNotZero(Cache, CI, P: OldDst);
549 Value *NewSrc = wrapPtrIfASNotZero(Cache, CI, P: OldSrc);
550 if (OldDst == NewDst && OldSrc == NewSrc)
551 return nullptr;
552
553 // memmove(new_dst, dst_align, new_src, src_align, len, isvolatile, md)
554 Value *Len = CI->getArgOperand(i: 2);
555
556 auto *MT = cast<MemTransferInst>(Val: CI);
557 MaybeAlign DstAlign = MT->getDestAlign();
558 MaybeAlign SrcAlign = MT->getSourceAlign();
559 bool IsVolatile = MT->isVolatile();
560
561 return B.CreateMemMove(Dst: NewDst, DstAlign, Src: NewSrc, SrcAlign, Size: Len, isVolatile: IsVolatile,
562 AAInfo: MI->getAAMetadata());
563}
564
565// Support for BPF address spaces:
566// - for each function in the module M, update pointer operand of
567// each memory access instruction (load/store/cmpxchg/atomicrmw)
568// or intrinsic call insns (memset/memcpy/memmove)
569// by casting it from non-zero address space to zero address space, e.g:
570//
571// (load (ptr addrspace (N) %p) ...)
572// -> (load (addrspacecast ptr addrspace (N) %p to ptr))
573//
574// - assign section with name .addr_space.N for globals defined in
575// non-zero address space N
576static bool insertASpaceCasts(Module &M) {
577 bool Changed = false;
578 for (Function &F : M) {
579 DenseMap<Value *, Value *> CastsCache;
580 for (BasicBlock &BB : F) {
581 for (Instruction &I : llvm::make_early_inc_range(Range&: BB)) {
582 unsigned PtrOpNum;
583
584 if (auto *LD = dyn_cast<LoadInst>(Val: &I)) {
585 PtrOpNum = LD->getPointerOperandIndex();
586 aspaceWrapOperand(Cache&: CastsCache, I: &I, OpNum: PtrOpNum);
587 continue;
588 }
589 if (auto *ST = dyn_cast<StoreInst>(Val: &I)) {
590 PtrOpNum = ST->getPointerOperandIndex();
591 aspaceWrapOperand(Cache&: CastsCache, I: &I, OpNum: PtrOpNum);
592 continue;
593 }
594 if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(Val: &I)) {
595 PtrOpNum = CmpXchg->getPointerOperandIndex();
596 aspaceWrapOperand(Cache&: CastsCache, I: &I, OpNum: PtrOpNum);
597 continue;
598 }
599 if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: &I)) {
600 PtrOpNum = RMW->getPointerOperandIndex();
601 aspaceWrapOperand(Cache&: CastsCache, I: &I, OpNum: PtrOpNum);
602 continue;
603 }
604
605 auto *CI = dyn_cast<CallInst>(Val: &I);
606 if (!CI)
607 continue;
608
609 Function *Callee = CI->getCalledFunction();
610 if (!Callee || !Callee->isIntrinsic())
611 continue;
612
613 // Check memset/memcpy/memmove
614 Intrinsic::ID ID = Callee->getIntrinsicID();
615 bool IsSet = ID == Intrinsic::memset || ID == Intrinsic::memset_inline;
616 bool IsCpy = ID == Intrinsic::memcpy || ID == Intrinsic::memcpy_inline;
617 bool IsMove = ID == Intrinsic::memmove;
618 if (!IsSet && !IsCpy && !IsMove)
619 continue;
620
621 Instruction *New;
622 if (IsSet)
623 New = aspaceMemSet(ID, Cache&: CastsCache, CI);
624 else if (IsCpy)
625 New = aspaceMemCpy(ID, Cache&: CastsCache, CI);
626 else
627 New = aspaceMemMove(Cache&: CastsCache, CI);
628
629 if (!New)
630 continue;
631
632 I.replaceAllUsesWith(V: New);
633 New->takeName(V: &I);
634 I.eraseFromParent();
635 }
636 }
637 Changed |= !CastsCache.empty();
638 }
639 // Merge all globals within same address space into single
640 // .addr_space.<addr space no> section
641 for (GlobalVariable &G : M.globals()) {
642 if (G.getAddressSpace() == 0 || G.hasSection())
643 continue;
644 SmallString<16> SecName;
645 raw_svector_ostream OS(SecName);
646 OS << ".addr_space." << G.getAddressSpace();
647 G.setSection(SecName);
648 // Prevent having separate section for constants
649 G.setConstant(false);
650 }
651 return Changed;
652}
653
654static bool adjustIR(Module &M,
655 function_ref<LoopInfo &(Function &)> GetLoopInfo) {
656 bool Changed = removePassThroughBuiltin(M);
657 Changed = removeCompareBuiltin(M) || Changed;
658 Changed = sinkMinMax(M, GetLoopInfo) || Changed;
659 Changed = removeGEPBuiltins(M) || Changed;
660 Changed = insertASpaceCasts(M) || Changed;
661 return Changed;
662}
663
664bool BPFCheckAndAdjustIRLegacy::runOnModule(Module &M) {
665 checkIR(M);
666 return adjustIR(M, GetLoopInfo: [&](Function &F) -> LoopInfo & {
667 return getAnalysis<LoopInfoWrapperPass>(F).getLoopInfo();
668 });
669}
670
671PreservedAnalyses BPFCheckAndAdjustIRPass::run(Module &M,
672 ModuleAnalysisManager &MAM) {
673 checkIR(M);
674 bool Changed = adjustIR(M, GetLoopInfo: [&](Function &F) -> LoopInfo & {
675 return MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M)
676 .getManager()
677 .getResult<LoopAnalysis>(IR&: F);
678 });
679 return Changed ? PreservedAnalyses::none().preserveSet<CFGAnalyses>()
680 : PreservedAnalyses::all();
681}
682