1//===--- ExpandMemCmp.cpp - Expand memcmp() to load/stores ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass tries to expand memcmp() calls into optimally-sized loads and
10// compares for the target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Scalar/ExpandMemCmp.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/Analysis/ConstantFolding.h"
17#include "llvm/Analysis/DomTreeUpdater.h"
18#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
19#include "llvm/Analysis/ProfileSummaryInfo.h"
20#include "llvm/Analysis/TargetLibraryInfo.h"
21#include "llvm/Analysis/TargetTransformInfo.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/IR/Dominators.h"
24#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/InstIterator.h"
26#include "llvm/IR/PatternMatch.h"
27#include "llvm/IR/ProfDataUtils.h"
28#include "llvm/Transforms/Utils/BasicBlockUtils.h"
29#include "llvm/Transforms/Utils/Local.h"
30#include "llvm/Transforms/Utils/SizeOpts.h"
31#include <optional>
32
33using namespace llvm;
34using namespace llvm::PatternMatch;
35
36#define DEBUG_TYPE "expand-memcmp"
37
38STATISTIC(NumMemCmpCalls, "Number of memcmp calls");
39STATISTIC(NumMemCmpNotConstant, "Number of memcmp calls without constant size");
40STATISTIC(NumMemCmpGreaterThanMax,
41 "Number of memcmp calls with size greater than max size");
42STATISTIC(NumMemCmpInlined, "Number of inlined memcmp calls");
43
44static cl::opt<unsigned> MemCmpEqZeroNumLoadsPerBlock(
45 "memcmp-num-loads-per-block", cl::Hidden, cl::init(Val: 1),
46 cl::desc("The number of loads per basic block for inline expansion of "
47 "memcmp that is only being compared against zero."));
48
49static cl::opt<unsigned> MaxLoadsPerMemcmp(
50 "max-loads-per-memcmp", cl::Hidden,
51 cl::desc("Set maximum number of loads used in expanded memcmp"));
52
53static cl::opt<unsigned> MaxLoadsPerMemcmpOptSize(
54 "max-loads-per-memcmp-opt-size", cl::Hidden,
55 cl::desc("Set maximum number of loads used in expanded memcmp for -Os/Oz"));
56
57namespace {
58
59// Return the known alignment of the pointer argument \p ArgNo of \p CI,
60// combining the alignment of the underlying pointer value with any align
61// attribute on the call site itself.
62static Align getMemCmpArgAlignment(const CallInst *CI, unsigned ArgNo,
63 const DataLayout &DL) {
64 Align A = CI->getArgOperand(i: ArgNo)->getPointerAlignment(DL);
65 if (MaybeAlign ParamAlign = CI->getParamAlign(ArgNo))
66 A = std::max(a: A, b: *ParamAlign);
67 return A;
68}
69
70// This class provides helper functions to expand a memcmp library call into an
71// inline expansion.
72class MemCmpExpansion {
73 struct ResultBlock {
74 BasicBlock *BB = nullptr;
75 PHINode *PhiSrc1 = nullptr;
76 PHINode *PhiSrc2 = nullptr;
77
78 ResultBlock() = default;
79 };
80
81 CallInst *const CI = nullptr;
82 ResultBlock ResBlock;
83 const uint64_t Size;
84 unsigned MaxLoadSize = 0;
85 uint64_t NumLoadsNonOneByte = 0;
86 const uint64_t NumLoadsPerBlockForZeroCmp;
87 std::vector<BasicBlock *> LoadCmpBlocks;
88 BasicBlock *EndBlock = nullptr;
89 PHINode *PhiRes = nullptr;
90 const bool IsUsedForZeroCmp;
91 const DataLayout &DL;
92 DomTreeUpdater *DTU = nullptr;
93 IRBuilder<> Builder;
94 // Represents the decomposition in blocks of the expansion. For example,
95 // comparing 33 bytes on X86+sse can be done with 2x16-byte loads and
96 // 1x1-byte load, which would be represented as [{16, 0}, {16, 16}, {1, 32}.
97 struct LoadEntry {
98 LoadEntry(unsigned LoadSize, uint64_t Offset)
99 : LoadSize(LoadSize), Offset(Offset) {
100 }
101
102 // The size of the load for this block, in bytes.
103 unsigned LoadSize;
104 // The offset of this load from the base pointer, in bytes.
105 uint64_t Offset;
106 };
107 using LoadEntryVector = SmallVector<LoadEntry, 8>;
108 LoadEntryVector LoadSequence;
109
110 void createLoadCmpBlocks();
111 void createResultBlock();
112 void setupResultBlockPHINodes();
113 void setupEndBlockPHINodes();
114 Value *getCompareLoadPairs(unsigned BlockIndex, unsigned &LoadIndex);
115 void emitLoadCompareBlock(unsigned BlockIndex);
116 void emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
117 unsigned &LoadIndex);
118 void emitLoadCompareByteBlock(unsigned BlockIndex, unsigned OffsetBytes);
119 void emitMemCmpResultBlock();
120 Value *getMemCmpExpansionZeroCase();
121 Value *getMemCmpEqZeroOneBlock();
122 Value *getMemCmpOneBlock();
123 struct LoadPair {
124 Value *Lhs = nullptr;
125 Value *Rhs = nullptr;
126 };
127 LoadPair getLoadPair(Type *LoadSizeType, Type *BSwapSizeType,
128 Type *CmpSizeType, unsigned OffsetBytes);
129
130 static LoadEntryVector
131 computeGreedyLoadSequence(uint64_t Size, llvm::ArrayRef<unsigned> LoadSizes,
132 unsigned MaxNumLoads, unsigned &NumLoadsNonOneByte);
133 static LoadEntryVector
134 computeOverlappingLoadSequence(uint64_t Size, unsigned MaxLoadSize,
135 unsigned MaxNumLoads,
136 unsigned &NumLoadsNonOneByte);
137
138 static void optimiseLoadSequence(
139 LoadEntryVector &LoadSequence,
140 const TargetTransformInfo::MemCmpExpansionOptions &Options,
141 bool IsUsedForZeroCmp);
142
143public:
144 MemCmpExpansion(CallInst *CI, uint64_t Size,
145 const TargetTransformInfo::MemCmpExpansionOptions &Options,
146 const bool IsUsedForZeroCmp, const DataLayout &TheDataLayout,
147 DomTreeUpdater *DTU);
148
149 unsigned getNumBlocks();
150 uint64_t getNumLoads() const { return LoadSequence.size(); }
151
152 Value *getMemCmpExpansion();
153};
154
155MemCmpExpansion::LoadEntryVector MemCmpExpansion::computeGreedyLoadSequence(
156 uint64_t Size, llvm::ArrayRef<unsigned> LoadSizes,
157 const unsigned MaxNumLoads, unsigned &NumLoadsNonOneByte) {
158 NumLoadsNonOneByte = 0;
159 LoadEntryVector LoadSequence;
160 uint64_t Offset = 0;
161 while (Size && !LoadSizes.empty()) {
162 const unsigned LoadSize = LoadSizes.front();
163 const uint64_t NumLoadsForThisSize = Size / LoadSize;
164 if (LoadSequence.size() + NumLoadsForThisSize > MaxNumLoads) {
165 // Do not expand if the total number of loads is larger than what the
166 // target allows. Note that it's important that we exit before completing
167 // the expansion to avoid using a ton of memory to store the expansion for
168 // large sizes.
169 return {};
170 }
171 if (NumLoadsForThisSize > 0) {
172 for (uint64_t I = 0; I < NumLoadsForThisSize; ++I) {
173 LoadSequence.push_back(Elt: {LoadSize, Offset});
174 Offset += LoadSize;
175 }
176 if (LoadSize > 1)
177 ++NumLoadsNonOneByte;
178 Size = Size % LoadSize;
179 }
180 LoadSizes = LoadSizes.drop_front();
181 }
182 return LoadSequence;
183}
184
185MemCmpExpansion::LoadEntryVector
186MemCmpExpansion::computeOverlappingLoadSequence(uint64_t Size,
187 const unsigned MaxLoadSize,
188 const unsigned MaxNumLoads,
189 unsigned &NumLoadsNonOneByte) {
190 // These are already handled by the greedy approach.
191 if (Size < 2 || MaxLoadSize < 2)
192 return {};
193
194 // We try to do as many non-overlapping loads as possible starting from the
195 // beginning.
196 const uint64_t NumNonOverlappingLoads = Size / MaxLoadSize;
197 assert(NumNonOverlappingLoads && "there must be at least one load");
198 // There remain 0 to (MaxLoadSize - 1) bytes to load, this will be done with
199 // an overlapping load.
200 Size = Size - NumNonOverlappingLoads * MaxLoadSize;
201 // Bail if we do not need an overloapping store, this is already handled by
202 // the greedy approach.
203 if (Size == 0)
204 return {};
205 // Bail if the number of loads (non-overlapping + potential overlapping one)
206 // is larger than the max allowed.
207 if ((NumNonOverlappingLoads + 1) > MaxNumLoads)
208 return {};
209
210 // Add non-overlapping loads.
211 LoadEntryVector LoadSequence;
212 uint64_t Offset = 0;
213 for (uint64_t I = 0; I < NumNonOverlappingLoads; ++I) {
214 LoadSequence.push_back(Elt: {MaxLoadSize, Offset});
215 Offset += MaxLoadSize;
216 }
217
218 // Add the last overlapping load.
219 assert(Size > 0 && Size < MaxLoadSize && "broken invariant");
220 LoadSequence.push_back(Elt: {MaxLoadSize, Offset - (MaxLoadSize - Size)});
221 NumLoadsNonOneByte = 1;
222 return LoadSequence;
223}
224
225void MemCmpExpansion::optimiseLoadSequence(
226 LoadEntryVector &LoadSequence,
227 const TargetTransformInfo::MemCmpExpansionOptions &Options,
228 bool IsUsedForZeroCmp) {
229 // This part of code attempts to optimize the LoadSequence by merging allowed
230 // subsequences into single loads of allowed sizes from
231 // `MemCmpExpansionOptions::AllowedTailExpansions`. If it is for zero
232 // comparison or if no allowed tail expansions are specified, we exit early.
233 if (IsUsedForZeroCmp || Options.AllowedTailExpansions.empty())
234 return;
235
236 while (LoadSequence.size() >= 2) {
237 auto Last = LoadSequence[LoadSequence.size() - 1];
238 auto PreLast = LoadSequence[LoadSequence.size() - 2];
239
240 // Exit the loop if the two sequences are not contiguous
241 if (PreLast.Offset + PreLast.LoadSize != Last.Offset)
242 break;
243
244 auto LoadSize = Last.LoadSize + PreLast.LoadSize;
245 if (find(Range: Options.AllowedTailExpansions, Val: LoadSize) ==
246 Options.AllowedTailExpansions.end())
247 break;
248
249 // Remove the last two sequences and replace with the combined sequence
250 LoadSequence.pop_back();
251 LoadSequence.pop_back();
252 LoadSequence.emplace_back(Args&: PreLast.Offset, Args&: LoadSize);
253 }
254}
255
256// Initialize the basic block structure required for expansion of memcmp call
257// with given maximum load size and memcmp size parameter.
258// This structure includes:
259// 1. A list of load compare blocks - LoadCmpBlocks.
260// 2. An EndBlock, split from original instruction point, which is the block to
261// return from.
262// 3. ResultBlock, block to branch to for early exit when a
263// LoadCmpBlock finds a difference.
264MemCmpExpansion::MemCmpExpansion(
265 CallInst *const CI, uint64_t Size,
266 const TargetTransformInfo::MemCmpExpansionOptions &Options,
267 const bool IsUsedForZeroCmp, const DataLayout &TheDataLayout,
268 DomTreeUpdater *DTU)
269 : CI(CI), Size(Size), NumLoadsPerBlockForZeroCmp(Options.NumLoadsPerBlock),
270 IsUsedForZeroCmp(IsUsedForZeroCmp), DL(TheDataLayout), DTU(DTU),
271 Builder(CI) {
272 assert(Size > 0 && "zero blocks");
273 // Scale the max size down if the target can load more bytes than we need.
274 llvm::ArrayRef<unsigned> LoadSizes(Options.LoadSizes);
275 while (!LoadSizes.empty() && LoadSizes.front() > Size) {
276 LoadSizes = LoadSizes.drop_front();
277 }
278 assert(!LoadSizes.empty() && "cannot load Size bytes");
279 MaxLoadSize = LoadSizes.front();
280 // Compute the decomposition.
281 unsigned GreedyNumLoadsNonOneByte = 0;
282 LoadSequence = computeGreedyLoadSequence(Size, LoadSizes, MaxNumLoads: Options.MaxNumLoads,
283 NumLoadsNonOneByte&: GreedyNumLoadsNonOneByte);
284 NumLoadsNonOneByte = GreedyNumLoadsNonOneByte;
285 assert(LoadSequence.size() <= Options.MaxNumLoads && "broken invariant");
286 // If we allow overlapping loads and the load sequence is not already optimal,
287 // use overlapping loads.
288 if (Options.AllowOverlappingLoads &&
289 (LoadSequence.empty() || LoadSequence.size() > 2)) {
290 unsigned OverlappingNumLoadsNonOneByte = 0;
291 auto OverlappingLoads = computeOverlappingLoadSequence(
292 Size, MaxLoadSize, MaxNumLoads: Options.MaxNumLoads, NumLoadsNonOneByte&: OverlappingNumLoadsNonOneByte);
293 if (!OverlappingLoads.empty() &&
294 (LoadSequence.empty() ||
295 OverlappingLoads.size() < LoadSequence.size())) {
296 LoadSequence = OverlappingLoads;
297 NumLoadsNonOneByte = OverlappingNumLoadsNonOneByte;
298 }
299 }
300 assert(LoadSequence.size() <= Options.MaxNumLoads && "broken invariant");
301 optimiseLoadSequence(LoadSequence, Options, IsUsedForZeroCmp);
302}
303
304unsigned MemCmpExpansion::getNumBlocks() {
305 if (IsUsedForZeroCmp)
306 return getNumLoads() / NumLoadsPerBlockForZeroCmp +
307 (getNumLoads() % NumLoadsPerBlockForZeroCmp != 0 ? 1 : 0);
308 return getNumLoads();
309}
310
311void MemCmpExpansion::createLoadCmpBlocks() {
312 for (unsigned i = 0; i < getNumBlocks(); i++) {
313 BasicBlock *BB = BasicBlock::Create(Context&: CI->getContext(), Name: "loadbb",
314 Parent: EndBlock->getParent(), InsertBefore: EndBlock);
315 LoadCmpBlocks.push_back(x: BB);
316 }
317}
318
319void MemCmpExpansion::createResultBlock() {
320 ResBlock.BB = BasicBlock::Create(Context&: CI->getContext(), Name: "res_block",
321 Parent: EndBlock->getParent(), InsertBefore: EndBlock);
322}
323
324MemCmpExpansion::LoadPair MemCmpExpansion::getLoadPair(Type *LoadSizeType,
325 Type *BSwapSizeType,
326 Type *CmpSizeType,
327 unsigned OffsetBytes) {
328 // Get the memory source at offset `OffsetBytes`.
329 Value *LhsSource = CI->getArgOperand(i: 0);
330 Value *RhsSource = CI->getArgOperand(i: 1);
331 Align LhsAlign = getMemCmpArgAlignment(CI, ArgNo: 0, DL);
332 Align RhsAlign = getMemCmpArgAlignment(CI, ArgNo: 1, DL);
333 if (OffsetBytes > 0) {
334 auto *ByteType = Type::getInt8Ty(C&: CI->getContext());
335 LhsSource = Builder.CreateConstGEP1_64(Ty: ByteType, Ptr: LhsSource, Idx0: OffsetBytes);
336 RhsSource = Builder.CreateConstGEP1_64(Ty: ByteType, Ptr: RhsSource, Idx0: OffsetBytes);
337 LhsAlign = commonAlignment(A: LhsAlign, Offset: OffsetBytes);
338 RhsAlign = commonAlignment(A: RhsAlign, Offset: OffsetBytes);
339 }
340
341 // Create a constant or a load from the source.
342 Value *Lhs = nullptr;
343 if (auto *C = dyn_cast<Constant>(Val: LhsSource))
344 Lhs = ConstantFoldLoadFromConstPtr(C, Ty: LoadSizeType, DL);
345 if (!Lhs)
346 Lhs = Builder.CreateAlignedLoad(Ty: LoadSizeType, Ptr: LhsSource, Align: LhsAlign);
347
348 Value *Rhs = nullptr;
349 if (auto *C = dyn_cast<Constant>(Val: RhsSource))
350 Rhs = ConstantFoldLoadFromConstPtr(C, Ty: LoadSizeType, DL);
351 if (!Rhs)
352 Rhs = Builder.CreateAlignedLoad(Ty: LoadSizeType, Ptr: RhsSource, Align: RhsAlign);
353
354 // Zero extend if Byte Swap intrinsic has different type
355 if (BSwapSizeType && LoadSizeType != BSwapSizeType) {
356 Lhs = Builder.CreateZExt(V: Lhs, DestTy: BSwapSizeType);
357 Rhs = Builder.CreateZExt(V: Rhs, DestTy: BSwapSizeType);
358 }
359
360 // Swap bytes if required.
361 if (BSwapSizeType) {
362 Function *Bswap = Intrinsic::getOrInsertDeclaration(
363 M: CI->getModule(), id: Intrinsic::bswap, OverloadTys: BSwapSizeType);
364 Lhs = Builder.CreateCall(Callee: Bswap, Args: Lhs);
365 Rhs = Builder.CreateCall(Callee: Bswap, Args: Rhs);
366 }
367
368 // Zero extend if required.
369 if (CmpSizeType != nullptr && CmpSizeType != Lhs->getType()) {
370 Lhs = Builder.CreateZExt(V: Lhs, DestTy: CmpSizeType);
371 Rhs = Builder.CreateZExt(V: Rhs, DestTy: CmpSizeType);
372 }
373 return {.Lhs: Lhs, .Rhs: Rhs};
374}
375
376// This function creates the IR instructions for loading and comparing 1 byte.
377// It loads 1 byte from each source of the memcmp parameters with the given
378// GEPIndex. It then subtracts the two loaded values and adds this result to the
379// final phi node for selecting the memcmp result.
380void MemCmpExpansion::emitLoadCompareByteBlock(unsigned BlockIndex,
381 unsigned OffsetBytes) {
382 BasicBlock *BB = LoadCmpBlocks[BlockIndex];
383 Builder.SetInsertPoint(BB);
384 const LoadPair Loads =
385 getLoadPair(LoadSizeType: Type::getInt8Ty(C&: CI->getContext()), BSwapSizeType: nullptr,
386 CmpSizeType: Type::getInt32Ty(C&: CI->getContext()), OffsetBytes);
387 Value *Diff = Builder.CreateSub(LHS: Loads.Lhs, RHS: Loads.Rhs);
388
389 PhiRes->addIncoming(V: Diff, BB);
390
391 if (BlockIndex < (LoadCmpBlocks.size() - 1)) {
392 // Early exit branch if difference found to EndBlock. Otherwise, continue to
393 // next LoadCmpBlock,
394 Value *Cmp = Builder.CreateICmp(P: ICmpInst::ICMP_NE, LHS: Diff,
395 RHS: ConstantInt::get(Ty: Diff->getType(), V: 0));
396 Builder.CreateCondBr(Cond: Cmp, True: EndBlock, False: LoadCmpBlocks[BlockIndex + 1]);
397 if (DTU)
398 DTU->applyUpdates(
399 Updates: {{DominatorTree::Insert, BB, EndBlock},
400 {DominatorTree::Insert, BB, LoadCmpBlocks[BlockIndex + 1]}});
401 } else {
402 // The last block has an unconditional branch to EndBlock.
403 Builder.CreateBr(Dest: EndBlock);
404 if (DTU)
405 DTU->applyUpdates(Updates: {{DominatorTree::Insert, BB, EndBlock}});
406 }
407}
408
409/// Generate an equality comparison for one or more pairs of loaded values.
410/// This is used in the case where the memcmp() call is compared equal or not
411/// equal to zero.
412Value *MemCmpExpansion::getCompareLoadPairs(unsigned BlockIndex,
413 unsigned &LoadIndex) {
414 assert(LoadIndex < getNumLoads() &&
415 "getCompareLoadPairs() called with no remaining loads");
416 std::vector<Value *> XorList, OrList;
417 Value *Diff = nullptr;
418
419 const unsigned NumLoads =
420 std::min(a: getNumLoads() - LoadIndex, b: NumLoadsPerBlockForZeroCmp);
421
422 // For a single-block expansion, start inserting before the memcmp call.
423 if (LoadCmpBlocks.empty())
424 Builder.SetInsertPoint(CI);
425 else
426 Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
427
428 Value *Cmp = nullptr;
429 // If we have multiple loads per block, we need to generate a composite
430 // comparison using xor+or. The type for the combinations is the largest load
431 // type.
432 IntegerType *const MaxLoadType =
433 NumLoads == 1 ? nullptr
434 : IntegerType::get(C&: CI->getContext(), NumBits: MaxLoadSize * 8);
435
436 for (unsigned i = 0; i < NumLoads; ++i, ++LoadIndex) {
437 const LoadEntry &CurLoadEntry = LoadSequence[LoadIndex];
438 const LoadPair Loads = getLoadPair(
439 LoadSizeType: IntegerType::get(C&: CI->getContext(), NumBits: CurLoadEntry.LoadSize * 8), BSwapSizeType: nullptr,
440 CmpSizeType: MaxLoadType, OffsetBytes: CurLoadEntry.Offset);
441
442 if (NumLoads != 1) {
443 // If we have multiple loads per block, we need to generate a composite
444 // comparison using xor+or.
445 Diff = Builder.CreateXor(LHS: Loads.Lhs, RHS: Loads.Rhs);
446 Diff = Builder.CreateZExt(V: Diff, DestTy: MaxLoadType);
447 XorList.push_back(x: Diff);
448 } else {
449 // If there's only one load per block, we just compare the loaded values.
450 Cmp = Builder.CreateICmpNE(LHS: Loads.Lhs, RHS: Loads.Rhs);
451 }
452 }
453
454 auto pairWiseOr = [&](std::vector<Value *> &InList) -> std::vector<Value *> {
455 std::vector<Value *> OutList;
456 for (unsigned i = 0; i < InList.size() - 1; i = i + 2) {
457 Value *Or = Builder.CreateOr(LHS: InList[i], RHS: InList[i + 1]);
458 OutList.push_back(x: Or);
459 }
460 if (InList.size() % 2 != 0)
461 OutList.push_back(x: InList.back());
462 return OutList;
463 };
464
465 if (!Cmp) {
466 // Pairwise OR the XOR results.
467 OrList = pairWiseOr(XorList);
468
469 // Pairwise OR the OR results until one result left.
470 while (OrList.size() != 1) {
471 OrList = pairWiseOr(OrList);
472 }
473
474 assert(Diff && "Failed to find comparison diff");
475 Cmp = Builder.CreateICmpNE(LHS: OrList[0], RHS: ConstantInt::get(Ty: Diff->getType(), V: 0));
476 }
477
478 return Cmp;
479}
480
481void MemCmpExpansion::emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
482 unsigned &LoadIndex) {
483 Value *Cmp = getCompareLoadPairs(BlockIndex, LoadIndex);
484
485 BasicBlock *NextBB = (BlockIndex == (LoadCmpBlocks.size() - 1))
486 ? EndBlock
487 : LoadCmpBlocks[BlockIndex + 1];
488 // Early exit branch if difference found to ResultBlock. Otherwise,
489 // continue to next LoadCmpBlock or EndBlock.
490 BasicBlock *BB = Builder.GetInsertBlock();
491 CondBrInst *CmpBr = Builder.CreateCondBr(Cond: Cmp, True: ResBlock.BB, False: NextBB);
492 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CmpBr, DEBUG_TYPE,
493 F: CI->getFunction());
494 if (DTU)
495 DTU->applyUpdates(Updates: {{DominatorTree::Insert, BB, ResBlock.BB},
496 {DominatorTree::Insert, BB, NextBB}});
497
498 // Add a phi edge for the last LoadCmpBlock to Endblock with a value of 0
499 // since early exit to ResultBlock was not taken (no difference was found in
500 // any of the bytes).
501 if (BlockIndex == LoadCmpBlocks.size() - 1) {
502 Value *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: CI->getContext()), V: 0);
503 PhiRes->addIncoming(V: Zero, BB: LoadCmpBlocks[BlockIndex]);
504 }
505}
506
507// This function creates the IR intructions for loading and comparing using the
508// given LoadSize. It loads the number of bytes specified by LoadSize from each
509// source of the memcmp parameters. It then does a subtract to see if there was
510// a difference in the loaded values. If a difference is found, it branches
511// with an early exit to the ResultBlock for calculating which source was
512// larger. Otherwise, it falls through to the either the next LoadCmpBlock or
513// the EndBlock if this is the last LoadCmpBlock. Loading 1 byte is handled with
514// a special case through emitLoadCompareByteBlock. The special handling can
515// simply subtract the loaded values and add it to the result phi node.
516void MemCmpExpansion::emitLoadCompareBlock(unsigned BlockIndex) {
517 // There is one load per block in this case, BlockIndex == LoadIndex.
518 const LoadEntry &CurLoadEntry = LoadSequence[BlockIndex];
519
520 if (CurLoadEntry.LoadSize == 1) {
521 MemCmpExpansion::emitLoadCompareByteBlock(BlockIndex, OffsetBytes: CurLoadEntry.Offset);
522 return;
523 }
524
525 Type *LoadSizeType =
526 IntegerType::get(C&: CI->getContext(), NumBits: CurLoadEntry.LoadSize * 8);
527 Type *BSwapSizeType =
528 DL.isLittleEndian()
529 ? IntegerType::get(C&: CI->getContext(),
530 NumBits: PowerOf2Ceil(A: CurLoadEntry.LoadSize * 8))
531 : nullptr;
532 Type *MaxLoadType = IntegerType::get(
533 C&: CI->getContext(),
534 NumBits: std::max(a: MaxLoadSize, b: (unsigned)PowerOf2Ceil(A: CurLoadEntry.LoadSize)) * 8);
535 assert(CurLoadEntry.LoadSize <= MaxLoadSize && "Unexpected load type");
536
537 Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
538
539 const LoadPair Loads = getLoadPair(LoadSizeType, BSwapSizeType, CmpSizeType: MaxLoadType,
540 OffsetBytes: CurLoadEntry.Offset);
541
542 // Add the loaded values to the phi nodes for calculating memcmp result only
543 // if result is not used in a zero equality.
544 if (!IsUsedForZeroCmp) {
545 ResBlock.PhiSrc1->addIncoming(V: Loads.Lhs, BB: LoadCmpBlocks[BlockIndex]);
546 ResBlock.PhiSrc2->addIncoming(V: Loads.Rhs, BB: LoadCmpBlocks[BlockIndex]);
547 }
548
549 Value *Cmp = Builder.CreateICmp(P: ICmpInst::ICMP_EQ, LHS: Loads.Lhs, RHS: Loads.Rhs);
550 BasicBlock *NextBB = (BlockIndex == (LoadCmpBlocks.size() - 1))
551 ? EndBlock
552 : LoadCmpBlocks[BlockIndex + 1];
553 // Early exit branch if difference found to ResultBlock. Otherwise, continue
554 // to next LoadCmpBlock or EndBlock.
555 BasicBlock *BB = Builder.GetInsertBlock();
556 CondBrInst *CmpBr = Builder.CreateCondBr(Cond: Cmp, True: NextBB, False: ResBlock.BB);
557 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CmpBr, DEBUG_TYPE,
558 F: CI->getFunction());
559 if (DTU)
560 DTU->applyUpdates(Updates: {{DominatorTree::Insert, BB, NextBB},
561 {DominatorTree::Insert, BB, ResBlock.BB}});
562
563 // Add a phi edge for the last LoadCmpBlock to Endblock with a value of 0
564 // since early exit to ResultBlock was not taken (no difference was found in
565 // any of the bytes).
566 if (BlockIndex == LoadCmpBlocks.size() - 1) {
567 Value *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: CI->getContext()), V: 0);
568 PhiRes->addIncoming(V: Zero, BB: LoadCmpBlocks[BlockIndex]);
569 }
570}
571
572// This function populates the ResultBlock with a sequence to calculate the
573// memcmp result. It compares the two loaded source values and returns -1 if
574// src1 < src2 and 1 if src1 > src2.
575void MemCmpExpansion::emitMemCmpResultBlock() {
576 // Special case: if memcmp result is used in a zero equality, result does not
577 // need to be calculated and can simply return 1.
578 if (IsUsedForZeroCmp) {
579 BasicBlock::iterator InsertPt = ResBlock.BB->getFirstInsertionPt();
580 Builder.SetInsertPoint(TheBB: ResBlock.BB, IP: InsertPt);
581 Value *Res = ConstantInt::get(Ty: Type::getInt32Ty(C&: CI->getContext()), V: 1);
582 PhiRes->addIncoming(V: Res, BB: ResBlock.BB);
583 Builder.CreateBr(Dest: EndBlock);
584 if (DTU)
585 DTU->applyUpdates(Updates: {{DominatorTree::Insert, ResBlock.BB, EndBlock}});
586 return;
587 }
588 BasicBlock::iterator InsertPt = ResBlock.BB->getFirstInsertionPt();
589 Builder.SetInsertPoint(TheBB: ResBlock.BB, IP: InsertPt);
590
591 Value *Cmp = Builder.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: ResBlock.PhiSrc1,
592 RHS: ResBlock.PhiSrc2);
593
594 Value *Res =
595 Builder.CreateSelect(C: Cmp, True: Constant::getAllOnesValue(Ty: Builder.getInt32Ty()),
596 False: ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
597 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *cast<Instruction>(Val: Res),
598 DEBUG_TYPE, F: CI->getFunction());
599
600 PhiRes->addIncoming(V: Res, BB: ResBlock.BB);
601 Builder.CreateBr(Dest: EndBlock);
602 if (DTU)
603 DTU->applyUpdates(Updates: {{DominatorTree::Insert, ResBlock.BB, EndBlock}});
604}
605
606void MemCmpExpansion::setupResultBlockPHINodes() {
607 Type *MaxLoadType = IntegerType::get(C&: CI->getContext(), NumBits: MaxLoadSize * 8);
608 Builder.SetInsertPoint(ResBlock.BB);
609 // Note: this assumes one load per block.
610 ResBlock.PhiSrc1 =
611 Builder.CreatePHI(Ty: MaxLoadType, NumReservedValues: NumLoadsNonOneByte, Name: "phi.src1");
612 ResBlock.PhiSrc2 =
613 Builder.CreatePHI(Ty: MaxLoadType, NumReservedValues: NumLoadsNonOneByte, Name: "phi.src2");
614}
615
616void MemCmpExpansion::setupEndBlockPHINodes() {
617 Builder.SetInsertPoint(TheBB: EndBlock, IP: EndBlock->begin());
618 PhiRes = Builder.CreatePHI(Ty: Type::getInt32Ty(C&: CI->getContext()), NumReservedValues: 2, Name: "phi.res");
619}
620
621Value *MemCmpExpansion::getMemCmpExpansionZeroCase() {
622 unsigned LoadIndex = 0;
623 // This loop populates each of the LoadCmpBlocks with the IR sequence to
624 // handle multiple loads per block.
625 for (unsigned I = 0; I < getNumBlocks(); ++I) {
626 emitLoadCompareBlockMultipleLoads(BlockIndex: I, LoadIndex);
627 }
628
629 emitMemCmpResultBlock();
630 return PhiRes;
631}
632
633/// A memcmp expansion that compares equality with 0 and only has one block of
634/// load and compare can bypass the compare, branch, and phi IR that is required
635/// in the general case.
636Value *MemCmpExpansion::getMemCmpEqZeroOneBlock() {
637 unsigned LoadIndex = 0;
638 Value *Cmp = getCompareLoadPairs(BlockIndex: 0, LoadIndex);
639 assert(LoadIndex == getNumLoads() && "some entries were not consumed");
640 return Builder.CreateZExt(V: Cmp, DestTy: Type::getInt32Ty(C&: CI->getContext()));
641}
642
643/// A memcmp expansion that only has one block of load and compare can bypass
644/// the compare, branch, and phi IR that is required in the general case.
645/// This function also analyses users of memcmp, and if there is only one user
646/// from which we can conclude that only 2 out of 3 memcmp outcomes really
647/// matter, then it generates more efficient code with only one comparison.
648Value *MemCmpExpansion::getMemCmpOneBlock() {
649 bool NeedsBSwap = DL.isLittleEndian() && Size != 1;
650 Type *LoadSizeType = IntegerType::get(C&: CI->getContext(), NumBits: Size * 8);
651 Type *BSwapSizeType =
652 NeedsBSwap ? IntegerType::get(C&: CI->getContext(), NumBits: PowerOf2Ceil(A: Size * 8))
653 : nullptr;
654 Type *MaxLoadType =
655 IntegerType::get(C&: CI->getContext(),
656 NumBits: std::max(a: MaxLoadSize, b: (unsigned)PowerOf2Ceil(A: Size)) * 8);
657
658 // The i8 and i16 cases don't need compares. We zext the loaded values and
659 // subtract them to get the suitable negative, zero, or positive i32 result.
660 if (Size == 1 || Size == 2) {
661 const LoadPair Loads = getLoadPair(LoadSizeType, BSwapSizeType,
662 CmpSizeType: Builder.getInt32Ty(), /*Offset*/ OffsetBytes: 0);
663 return Builder.CreateSub(LHS: Loads.Lhs, RHS: Loads.Rhs);
664 }
665
666 const LoadPair Loads = getLoadPair(LoadSizeType, BSwapSizeType, CmpSizeType: MaxLoadType,
667 /*Offset*/ OffsetBytes: 0);
668
669 // If a user of memcmp cares only about two outcomes, for example:
670 // bool result = memcmp(a, b, NBYTES) > 0;
671 // We can generate more optimal code with a smaller number of operations
672 if (CI->hasOneUser()) {
673 auto *UI = cast<Instruction>(Val: *CI->user_begin());
674 CmpPredicate Pred = ICmpInst::Predicate::BAD_ICMP_PREDICATE;
675 bool NeedsZExt = false;
676 // This is a special case because instead of checking if the result is less
677 // than zero:
678 // bool result = memcmp(a, b, NBYTES) < 0;
679 // Compiler is clever enough to generate the following code:
680 // bool result = memcmp(a, b, NBYTES) >> 31;
681 if (match(V: UI,
682 P: m_LShr(L: m_Value(),
683 R: m_SpecificInt(V: CI->getType()->getIntegerBitWidth() - 1)))) {
684 Pred = ICmpInst::ICMP_SLT;
685 NeedsZExt = true;
686 } else if (match(V: UI, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_SGT, L: m_Specific(V: CI),
687 R: m_AllOnes()))) {
688 // Adjust predicate as if it compared with 0.
689 Pred = ICmpInst::ICMP_SGE;
690 } else if (match(V: UI, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_SLT, L: m_Specific(V: CI),
691 R: m_One()))) {
692 // Adjust predicate as if it compared with 0.
693 Pred = ICmpInst::ICMP_SLE;
694 } else {
695 // In case of a successful match this call will set `Pred` variable
696 match(V: UI, P: m_ICmp(Pred, L: m_Specific(V: CI), R: m_Zero()));
697 }
698 // Generate new code and remove the original memcmp call and the user
699 if (ICmpInst::isSigned(Pred)) {
700 Value *Cmp = Builder.CreateICmp(P: ICmpInst::getUnsignedPredicate(Pred),
701 LHS: Loads.Lhs, RHS: Loads.Rhs);
702 auto *Result = NeedsZExt ? Builder.CreateZExt(V: Cmp, DestTy: UI->getType()) : Cmp;
703 UI->replaceAllUsesWith(V: Result);
704 UI->eraseFromParent();
705 CI->eraseFromParent();
706 return nullptr;
707 }
708 }
709
710 // The result of memcmp is negative, zero, or positive.
711 return Builder.CreateIntrinsic(RetTy: Builder.getInt32Ty(), ID: Intrinsic::ucmp,
712 Args: {Loads.Lhs, Loads.Rhs});
713}
714
715// This function expands the memcmp call into an inline expansion and returns
716// the memcmp result. Returns nullptr if the memcmp is already replaced.
717Value *MemCmpExpansion::getMemCmpExpansion() {
718 // Create the basic block framework for a multi-block expansion.
719 if (getNumBlocks() != 1) {
720 BasicBlock *StartBlock = CI->getParent();
721 EndBlock = SplitBlock(Old: StartBlock, SplitPt: CI, DTU, /*LI=*/nullptr,
722 /*MSSAU=*/nullptr, BBName: "endblock");
723 setupEndBlockPHINodes();
724 createResultBlock();
725
726 // If return value of memcmp is not used in a zero equality, we need to
727 // calculate which source was larger. The calculation requires the
728 // two loaded source values of each load compare block.
729 // These will be saved in the phi nodes created by setupResultBlockPHINodes.
730 if (!IsUsedForZeroCmp) setupResultBlockPHINodes();
731
732 // Create the number of required load compare basic blocks.
733 createLoadCmpBlocks();
734
735 // Update the terminator added by SplitBlock to branch to the first
736 // LoadCmpBlock.
737 StartBlock->getTerminator()->setSuccessor(Idx: 0, BB: LoadCmpBlocks[0]);
738 if (DTU)
739 DTU->applyUpdates(Updates: {{DominatorTree::Insert, StartBlock, LoadCmpBlocks[0]},
740 {DominatorTree::Delete, StartBlock, EndBlock}});
741 }
742
743 Builder.SetCurrentDebugLocation(CI->getDebugLoc());
744
745 if (IsUsedForZeroCmp)
746 return getNumBlocks() == 1 ? getMemCmpEqZeroOneBlock()
747 : getMemCmpExpansionZeroCase();
748
749 if (getNumBlocks() == 1)
750 return getMemCmpOneBlock();
751
752 for (unsigned I = 0; I < getNumBlocks(); ++I) {
753 emitLoadCompareBlock(BlockIndex: I);
754 }
755
756 emitMemCmpResultBlock();
757 return PhiRes;
758}
759
760// This function checks to see if an expansion of memcmp can be generated.
761// It checks for constant compare size that is less than the max inline size.
762// If an expansion cannot occur, returns false to leave as a library call.
763// Otherwise, the library call is replaced with a new IR instruction sequence.
764/// We want to transform:
765/// %call = call signext i32 @memcmp(i8* %0, i8* %1, i64 15)
766/// To:
767/// loadbb:
768/// %0 = bitcast i32* %buffer2 to i8*
769/// %1 = bitcast i32* %buffer1 to i8*
770/// %2 = bitcast i8* %1 to i64*
771/// %3 = bitcast i8* %0 to i64*
772/// %4 = load i64, i64* %2
773/// %5 = load i64, i64* %3
774/// %6 = call i64 @llvm.bswap.i64(i64 %4)
775/// %7 = call i64 @llvm.bswap.i64(i64 %5)
776/// %8 = sub i64 %6, %7
777/// %9 = icmp ne i64 %8, 0
778/// br i1 %9, label %res_block, label %loadbb1
779/// res_block: ; preds = %loadbb2,
780/// %loadbb1, %loadbb
781/// %phi.src1 = phi i64 [ %6, %loadbb ], [ %22, %loadbb1 ], [ %36, %loadbb2 ]
782/// %phi.src2 = phi i64 [ %7, %loadbb ], [ %23, %loadbb1 ], [ %37, %loadbb2 ]
783/// %10 = icmp ult i64 %phi.src1, %phi.src2
784/// %11 = select i1 %10, i32 -1, i32 1
785/// br label %endblock
786/// loadbb1: ; preds = %loadbb
787/// %12 = bitcast i32* %buffer2 to i8*
788/// %13 = bitcast i32* %buffer1 to i8*
789/// %14 = bitcast i8* %13 to i32*
790/// %15 = bitcast i8* %12 to i32*
791/// %16 = getelementptr i32, i32* %14, i32 2
792/// %17 = getelementptr i32, i32* %15, i32 2
793/// %18 = load i32, i32* %16
794/// %19 = load i32, i32* %17
795/// %20 = call i32 @llvm.bswap.i32(i32 %18)
796/// %21 = call i32 @llvm.bswap.i32(i32 %19)
797/// %22 = zext i32 %20 to i64
798/// %23 = zext i32 %21 to i64
799/// %24 = sub i64 %22, %23
800/// %25 = icmp ne i64 %24, 0
801/// br i1 %25, label %res_block, label %loadbb2
802/// loadbb2: ; preds = %loadbb1
803/// %26 = bitcast i32* %buffer2 to i8*
804/// %27 = bitcast i32* %buffer1 to i8*
805/// %28 = bitcast i8* %27 to i16*
806/// %29 = bitcast i8* %26 to i16*
807/// %30 = getelementptr i16, i16* %28, i16 6
808/// %31 = getelementptr i16, i16* %29, i16 6
809/// %32 = load i16, i16* %30
810/// %33 = load i16, i16* %31
811/// %34 = call i16 @llvm.bswap.i16(i16 %32)
812/// %35 = call i16 @llvm.bswap.i16(i16 %33)
813/// %36 = zext i16 %34 to i64
814/// %37 = zext i16 %35 to i64
815/// %38 = sub i64 %36, %37
816/// %39 = icmp ne i64 %38, 0
817/// br i1 %39, label %res_block, label %loadbb3
818/// loadbb3: ; preds = %loadbb2
819/// %40 = bitcast i32* %buffer2 to i8*
820/// %41 = bitcast i32* %buffer1 to i8*
821/// %42 = getelementptr i8, i8* %41, i8 14
822/// %43 = getelementptr i8, i8* %40, i8 14
823/// %44 = load i8, i8* %42
824/// %45 = load i8, i8* %43
825/// %46 = zext i8 %44 to i32
826/// %47 = zext i8 %45 to i32
827/// %48 = sub i32 %46, %47
828/// br label %endblock
829/// endblock: ; preds = %res_block,
830/// %loadbb3
831/// %phi.res = phi i32 [ %48, %loadbb3 ], [ %11, %res_block ]
832/// ret i32 %phi.res
833static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI,
834 const DataLayout *DL, ProfileSummaryInfo *PSI,
835 BlockFrequencyInfo *BFI, DomTreeUpdater *DTU,
836 const bool IsBCmp) {
837 NumMemCmpCalls++;
838
839 // Early exit from expansion if -Oz.
840 if (CI->getFunction()->hasMinSize())
841 return false;
842
843 // Early exit from expansion if size is not a constant.
844 ConstantInt *SizeCast = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
845 if (!SizeCast) {
846 NumMemCmpNotConstant++;
847 return false;
848 }
849 const uint64_t SizeVal = SizeCast->getZExtValue();
850
851 if (SizeVal == 0) {
852 return false;
853 }
854 // TTI call to check if target would like to expand memcmp. Also, get the
855 // available load sizes.
856 const bool IsUsedForZeroCmp =
857 IsBCmp || isOnlyUsedInZeroEqualityComparison(CxtI: CI);
858 bool OptForSize = llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI);
859 auto Options = TTI->enableMemCmpExpansion(OptSize: OptForSize,
860 IsZeroCmp: IsUsedForZeroCmp);
861 if (!Options) return false;
862
863 if (MemCmpEqZeroNumLoadsPerBlock.getNumOccurrences())
864 Options.NumLoadsPerBlock = MemCmpEqZeroNumLoadsPerBlock;
865
866 if (OptForSize &&
867 MaxLoadsPerMemcmpOptSize.getNumOccurrences())
868 Options.MaxNumLoads = MaxLoadsPerMemcmpOptSize;
869
870 if (!OptForSize && MaxLoadsPerMemcmp.getNumOccurrences())
871 Options.MaxNumLoads = MaxLoadsPerMemcmp;
872
873 // Keep only the load sizes the target can actually access given the
874 // statically known common alignment of both pointers: either the access is
875 // naturally aligned, or the target allows a misaligned access of that width.
876 // This lets strict-alignment targets expand compares whose pointers happen to
877 // be sufficiently aligned, while still falling back to the libcall when no
878 // load size fits. Because the greedy load sequence only places a load of size
879 // S at an offset that is a multiple of S, a load that does not exceed the
880 // base alignment is guaranteed to be naturally aligned.
881 //
882 // Note we query whether the access is *allowed*, not whether it is *fast*:
883 // this matches the historical behavior of forming unaligned loads whenever
884 // the target permits them, so it is a no-op for targets that allow unaligned
885 // access even when it is slow.
886 const Align LhsAlign = getMemCmpArgAlignment(CI, ArgNo: 0, DL: *DL);
887 const Align RhsAlign = getMemCmpArgAlignment(CI, ArgNo: 1, DL: *DL);
888 const Align MinAlign = std::min(a: LhsAlign, b: RhsAlign);
889 LLVMContext &Context = CI->getContext();
890 unsigned AS = CI->getArgOperand(i: 0)->getType()->getPointerAddressSpace();
891 llvm::erase_if(C&: Options.LoadSizes, P: [&](unsigned LoadSize) {
892 if (MinAlign >= LoadSize)
893 return false;
894 return !TTI->allowsMisalignedMemoryAccesses(Context, BitWidth: LoadSize * 8, AddressSpace: AS,
895 Alignment: MinAlign);
896 });
897 // If the filter removed every load size, bail out to the libcall: the
898 // MemCmpExpansion constructor asserts that at least one load size remains.
899 // In practice all in-tree targets include a byte load size, which is
900 // accessible at any alignment and therefore always survives the filter.
901 if (Options.LoadSizes.empty())
902 return false;
903
904 MemCmpExpansion Expansion(CI, SizeVal, Options, IsUsedForZeroCmp, *DL, DTU);
905
906 // Don't expand if this will require more loads than desired by the target.
907 if (Expansion.getNumLoads() == 0) {
908 NumMemCmpGreaterThanMax++;
909 return false;
910 }
911
912 NumMemCmpInlined++;
913
914 if (Value *Res = Expansion.getMemCmpExpansion()) {
915 // Replace call with result of expansion and erase call.
916 CI->replaceAllUsesWith(V: Res);
917 CI->eraseFromParent();
918 }
919
920 return true;
921}
922
923static PreservedAnalyses runImpl(Function &F, const TargetLibraryInfo *TLI,
924 const TargetTransformInfo *TTI,
925 ProfileSummaryInfo *PSI,
926 BlockFrequencyInfo *BFI, DominatorTree *DT) {
927 std::optional<DomTreeUpdater> DTU;
928 if (DT)
929 DTU.emplace(args&: DT, args: DomTreeUpdater::UpdateStrategy::Lazy);
930
931 const DataLayout& DL = F.getDataLayout();
932 SmallVector<std::pair<CallInst *, LibFunc>, 8> MemCmpCalls;
933 for (Instruction &I : instructions(F)) {
934 if (auto *CI = dyn_cast<CallInst>(Val: &I)) {
935 LibFunc Func;
936 if (TLI->getLibFunc(CB: *CI, F&: Func) &&
937 (Func == LibFunc_memcmp || Func == LibFunc_bcmp))
938 MemCmpCalls.push_back(Elt: {CI, Func});
939 }
940 }
941
942 bool MadeChanges = false;
943 for (const auto &[CI, Func] : MemCmpCalls) {
944 if (expandMemCmp(CI, TTI, DL: &DL, PSI, BFI, DTU: DTU ? &*DTU : nullptr,
945 IsBCmp: Func == LibFunc_bcmp))
946 MadeChanges = true;
947 }
948
949 if (MadeChanges)
950 for (BasicBlock &BB : F)
951 SimplifyInstructionsInBlock(BB: &BB);
952 if (!MadeChanges)
953 return PreservedAnalyses::all();
954 PreservedAnalyses PA;
955 PA.preserve<DominatorTreeAnalysis>();
956 return PA;
957}
958
959} // namespace
960
961PreservedAnalyses ExpandMemCmpPass::run(Function &F,
962 FunctionAnalysisManager &FAM) {
963 // Don't expand memcmp in sanitized functions — sanitizers intercept memcmp
964 // calls to check for memory errors, and expanding would bypass that.
965 if (F.hasFnAttribute(Kind: Attribute::SanitizeAddress) ||
966 F.hasFnAttribute(Kind: Attribute::SanitizeMemory) ||
967 F.hasFnAttribute(Kind: Attribute::SanitizeThread) ||
968 F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress))
969 return PreservedAnalyses::all();
970
971 const auto &TLI = FAM.getResult<TargetLibraryAnalysis>(IR&: F);
972 const auto &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
973 auto *PSI = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F)
974 .getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
975 BlockFrequencyInfo *BFI = (PSI && PSI->hasProfileSummary())
976 ? &FAM.getResult<BlockFrequencyAnalysis>(IR&: F)
977 : nullptr;
978 auto *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
979
980 return runImpl(F, TLI: &TLI, TTI: &TTI, PSI, BFI, DT);
981}
982