1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/SmallString.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/Analysis/ConstantFolding.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/Utils/Local.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/AttributeMask.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/MDBuilder.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/PatternMatch.h"
34#include "llvm/IR/ProfDataUtils.h"
35#include "llvm/Support/Casting.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Support/KnownBits.h"
38#include "llvm/Support/KnownFPClass.h"
39#include "llvm/Support/MathExtras.h"
40#include "llvm/TargetParser/Triple.h"
41#include "llvm/Transforms/Utils/BuildLibCalls.h"
42#include "llvm/Transforms/Utils/Local.h"
43#include "llvm/Transforms/Utils/SizeOpts.h"
44
45#include <cmath>
46
47using namespace llvm;
48using namespace PatternMatch;
49
50#define DEBUG_TYPE "simplify-lib-calls"
51
52static cl::opt<bool>
53 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
54 cl::init(Val: false),
55 cl::desc("Enable unsafe double to float "
56 "shrinking for math lib calls"));
57
58// Enable conversion of operator new calls with a MemProf hot or cold hint
59// to an operator new call that takes a hot/cold hint. Off by default since
60// not all allocators currently support this extension.
61static cl::opt<bool>
62 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(Val: false),
63 cl::desc("Enable hot/cold operator new library calls"));
64enum class OptimizeExistingHotColdNewKind {
65 None,
66 Cold,
67 Always,
68};
69static cl::opt<OptimizeExistingHotColdNewKind> OptimizeExistingHotColdNew(
70 "optimize-existing-hot-cold-new", cl::Hidden,
71 cl::desc(
72 "Enable optimization of existing hot/cold operator new library calls"),
73 cl::values(
74 clEnumValN(
75 OptimizeExistingHotColdNewKind::None, "none",
76 "Do not optimize existing hot/cold operator new library calls"),
77 clEnumValN(OptimizeExistingHotColdNewKind::Cold, "cold",
78 "Only optimize existing hot/cold operator new library calls "
79 "if determined to be cold"),
80 clEnumValN(
81 OptimizeExistingHotColdNewKind::Always, "always",
82 "Always optimize existing hot/cold operator new library calls"),
83 clEnumValN(
84 OptimizeExistingHotColdNewKind::Always, "",
85 "Always optimize existing hot/cold operator new library calls")),
86 cl::init(Val: OptimizeExistingHotColdNewKind::None), cl::ValueOptional);
87static cl::opt<bool> OptimizeNoBuiltinHotColdNew(
88 "optimize-nobuiltin-hot-cold-new-new", cl::Hidden, cl::init(Val: false),
89 cl::desc("Enable transformation of nobuiltin operator new library calls"));
90static cl::opt<bool> MinExistingHotColdNewHint(
91 "min-existing-hot-cold-new-hint", cl::Hidden, cl::init(Val: false),
92 cl::desc("Take the minimum of compiler hint and existing hint when "
93 "optimizing existing hot/cold operator new library calls"));
94
95namespace llvm {
96extern cl::opt<bool> ProfcheckDisableMetadataFixes;
97} // namespace llvm
98
99namespace {
100
101// Specialized parser to ensure the hint is an 8 bit value (we can't specify
102// uint8_t to opt<> as that is interpreted to mean that we are passing a char
103// option with a specific set of values.
104struct HotColdHintParser : public cl::parser<unsigned> {
105 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
106
107 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
108 if (Arg.getAsInteger(Radix: 0, Result&: Value))
109 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
110
111 if (Value > 255)
112 return O.error(Message: "'" + Arg + "' value must be in the range [0, 255]!");
113
114 return false;
115 }
116};
117
118} // end anonymous namespace
119
120// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
121// and 255 is the hottest. Default to 1 value away from the coldest and hottest
122// hints, so that the compiler hinted allocations are slightly less strong than
123// manually inserted hints at the two extremes.
124static cl::opt<unsigned, false, HotColdHintParser> ColdNewHintValue(
125 "cold-new-hint-value", cl::Hidden, cl::init(Val: 1),
126 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
127static cl::opt<unsigned, false, HotColdHintParser>
128 NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(Val: 128),
129 cl::desc("Value to pass to hot/cold operator new for "
130 "notcold (warm) allocation"));
131static cl::opt<unsigned, false, HotColdHintParser> HotNewHintValue(
132 "hot-new-hint-value", cl::Hidden, cl::init(Val: 254),
133 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
134static cl::opt<unsigned, false, HotColdHintParser> AmbiguousNewHintValue(
135 "ambiguous-new-hint-value", cl::Hidden, cl::init(Val: 222),
136 cl::desc(
137 "Value to pass to hot/cold operator new for ambiguous allocation"));
138
139//===----------------------------------------------------------------------===//
140// Helper Functions
141//===----------------------------------------------------------------------===//
142
143static bool ignoreCallingConv(LibFunc Func) {
144 return Func == LibFunc_abs || Func == LibFunc_labs ||
145 Func == LibFunc_llabs || Func == LibFunc_strlen;
146}
147
148/// Return true if it is only used in equality comparisons with With.
149static bool isOnlyUsedInEqualityComparison(Value *V, Value *With) {
150 for (User *U : V->users()) {
151 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
152 if (IC->isEquality() && IC->getOperand(i_nocapture: 1) == With)
153 continue;
154 // Unknown instruction.
155 return false;
156 }
157 return true;
158}
159
160static bool callHasFloatingPointArgument(const CallInst *CI) {
161 return any_of(Range: CI->operands(), P: [](const Use &OI) {
162 return OI->getType()->isFloatingPointTy();
163 });
164}
165
166static bool callHasFP128Argument(const CallInst *CI) {
167 return any_of(Range: CI->operands(), P: [](const Use &OI) {
168 return OI->getType()->isFP128Ty();
169 });
170}
171
172// Convert the entire string Str representing an integer in Base, up to
173// the terminating nul if present, to a constant according to the rules
174// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
175// return the result, otherwise null.
176// The function assumes the string is encoded in ASCII and carefully
177// avoids converting sequences (including "") that the corresponding
178// library call might fail and set errno for.
179static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
180 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
181 if (Base < 2 || Base > 36)
182 if (Base != 0)
183 // Fail for an invalid base (required by POSIX).
184 return nullptr;
185
186 // Current offset into the original string to reflect in EndPtr.
187 size_t Offset = 0;
188 // Strip leading whitespace.
189 for ( ; Offset != Str.size(); ++Offset)
190 if (!isSpace(C: (unsigned char)Str[Offset])) {
191 Str = Str.substr(Start: Offset);
192 break;
193 }
194
195 if (Str.empty())
196 // Fail for empty subject sequences (POSIX allows but doesn't require
197 // strtol[l]/strtoul[l] to fail with EINVAL).
198 return nullptr;
199
200 // Strip but remember the sign.
201 bool Negate = Str[0] == '-';
202 if (Str[0] == '-' || Str[0] == '+') {
203 Str = Str.drop_front();
204 if (Str.empty())
205 // Fail for a sign with nothing after it.
206 return nullptr;
207 ++Offset;
208 }
209
210 // Set Max to the absolute value of the minimum (for signed), or
211 // to the maximum (for unsigned) value representable in the type.
212 Type *RetTy = CI->getType();
213 unsigned NBits = RetTy->getPrimitiveSizeInBits();
214 uint64_t Max = AsSigned && Negate ? 1 : 0;
215 Max += AsSigned ? maxIntN(N: NBits) : maxUIntN(N: NBits);
216
217 // Autodetect Base if it's zero and consume the "0x" prefix.
218 if (Str.size() > 1) {
219 if (Str[0] == '0') {
220 if (toUpper(x: (unsigned char)Str[1]) == 'X') {
221 if (Str.size() == 2 || (Base && Base != 16))
222 // Fail if Base doesn't allow the "0x" prefix or for the prefix
223 // alone that implementations like BSD set errno to EINVAL for.
224 return nullptr;
225
226 Str = Str.drop_front(N: 2);
227 Offset += 2;
228 Base = 16;
229 }
230 else if (Base == 0)
231 Base = 8;
232 } else if (Base == 0)
233 Base = 10;
234 }
235 else if (Base == 0)
236 Base = 10;
237
238 // Convert the rest of the subject sequence, not including the sign,
239 // to its uint64_t representation (this assumes the source character
240 // set is ASCII).
241 uint64_t Result = 0;
242 for (unsigned i = 0; i != Str.size(); ++i) {
243 unsigned char DigVal = Str[i];
244 if (isDigit(C: DigVal))
245 DigVal = DigVal - '0';
246 else {
247 DigVal = toUpper(x: DigVal);
248 if (isAlpha(C: DigVal))
249 DigVal = DigVal - 'A' + 10;
250 else
251 return nullptr;
252 }
253
254 if (DigVal >= Base)
255 // Fail if the digit is not valid in the Base.
256 return nullptr;
257
258 // Add the digit and fail if the result is not representable in
259 // the (unsigned form of the) destination type.
260 bool VFlow;
261 Result = SaturatingMultiplyAdd(X: Result, Y: Base, A: (uint64_t)DigVal, ResultOverflowed: &VFlow);
262 if (VFlow || Result > Max)
263 return nullptr;
264 }
265
266 if (EndPtr) {
267 // Store the pointer to the end.
268 Value *Off = B.getInt64(C: Offset + Str.size());
269 Value *StrBeg = CI->getArgOperand(i: 0);
270 Value *StrEnd = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: StrBeg, IdxList: Off, Name: "endptr");
271 B.CreateStore(Val: StrEnd, Ptr: EndPtr);
272 }
273
274 if (Negate) {
275 // Unsigned negation doesn't overflow.
276 Result = -Result;
277 // For unsigned numbers, discard sign bits.
278 if (!AsSigned)
279 Result &= maxUIntN(N: NBits);
280 }
281
282 return ConstantInt::get(Ty: RetTy, V: Result, IsSigned: AsSigned);
283}
284
285static bool isOnlyUsedInComparisonWithZero(Value *V) {
286 for (User *U : V->users()) {
287 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
288 if (Constant *C = dyn_cast<Constant>(Val: IC->getOperand(i_nocapture: 1)))
289 if (C->isNullValue())
290 continue;
291 // Unknown instruction.
292 return false;
293 }
294 return true;
295}
296
297static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
298 const SimplifyQuery &SQ) {
299 if (!isOnlyUsedInComparisonWithZero(V: CI))
300 return false;
301
302 if (!isDereferenceablePointer(V: Str, Size: APInt(64, Len), Q: SQ))
303 return false;
304
305 if (CI->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemory))
306 return false;
307
308 return true;
309}
310
311static void annotateDereferenceableBytes(CallInst *CI,
312 ArrayRef<unsigned> ArgNos,
313 uint64_t DereferenceableBytes) {
314 const Function *F = CI->getCaller();
315 if (!F)
316 return;
317 for (unsigned ArgNo : ArgNos) {
318 uint64_t DerefBytes = DereferenceableBytes;
319 unsigned AS = CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
320 if (!llvm::NullPointerIsDefined(F, AS) ||
321 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
322 DerefBytes = std::max(a: CI->getParamDereferenceableOrNullBytes(i: ArgNo),
323 b: DereferenceableBytes);
324
325 if (CI->getParamDereferenceableBytes(i: ArgNo) < DerefBytes) {
326 CI->removeParamAttr(ArgNo, Kind: Attribute::Dereferenceable);
327 if (!llvm::NullPointerIsDefined(F, AS) ||
328 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
329 CI->removeParamAttr(ArgNo, Kind: Attribute::DereferenceableOrNull);
330 CI->addParamAttr(ArgNo, Attr: Attribute::getWithDereferenceableBytes(
331 Context&: CI->getContext(), Bytes: DerefBytes));
332 }
333 }
334}
335
336static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI,
337 ArrayRef<unsigned> ArgNos,
338 uint64_t CharSize = 1) {
339 Function *F = CI->getCaller();
340 if (!F)
341 return;
342
343 for (unsigned ArgNo : ArgNos) {
344 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NoUndef))
345 CI->addParamAttr(ArgNo, Kind: Attribute::NoUndef);
346
347 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull)) {
348 unsigned AS =
349 CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
350 if (llvm::NullPointerIsDefined(F, AS))
351 continue;
352 CI->addParamAttr(ArgNo, Kind: Attribute::NonNull);
353 }
354
355 annotateDereferenceableBytes(CI, ArgNos: ArgNo, DereferenceableBytes: CharSize);
356 }
357}
358
359static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> ArgNos,
360 Value *Size, const DataLayout &DL) {
361 if (ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size)) {
362 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
363 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: LenC->getZExtValue());
364 } else if (isKnownNonZero(V: Size, Q: DL)) {
365 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
366 uint64_t X, Y;
367 uint64_t DerefMin = 1;
368 if (match(V: Size, P: m_Select(C: m_Value(), L: m_ConstantInt(V&: X), R: m_ConstantInt(V&: Y)))) {
369 DerefMin = std::min(a: X, b: Y);
370 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: DerefMin);
371 }
372 }
373}
374
375// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
376// easier chaining. Calls to emit* and B.createCall should probably be wrapped
377// in this function when New is created to replace Old. Callers should take
378// care to check Old.isMustTailCall() if they aren't replacing Old directly
379// with New.
380static Value *copyFlags(const CallInst &Old, Value *New) {
381 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
382 assert(!Old.isNoTailCall() && "do not copy notail call flags");
383 if (auto *NewCI = dyn_cast_or_null<CallInst>(Val: New))
384 NewCI->setTailCallKind(Old.getTailCallKind());
385 return New;
386}
387
388static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
389 NewCI->setAttributes(AttributeList::get(
390 C&: NewCI->getContext(), Attrs: {NewCI->getAttributes(), Old.getAttributes()}));
391 if (NewCI->getType()->isVoidTy()) {
392 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
393 NewCI->removeParamAttr(ArgNo: I, Kind: Attribute::Returned);
394 }
395 NewCI->removeRetAttrs(AttrsToRemove: AttributeFuncs::typeIncompatible(
396 Ty: NewCI->getType(), AS: NewCI->getRetAttributes()));
397 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
398 NewCI->removeParamAttrs(
399 ArgNo: I, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewCI->getArgOperand(i: I)->getType(),
400 AS: NewCI->getParamAttributes(ArgNo: I)));
401
402 return copyFlags(Old, New: NewCI);
403}
404
405// Helper to avoid truncating the length if size_t is 32-bits.
406static StringRef substr(StringRef Str, uint64_t Len) {
407 return Len >= Str.size() ? Str : Str.substr(Start: 0, N: Len);
408}
409
410//===----------------------------------------------------------------------===//
411// String and Memory Library Call Optimizations
412//===----------------------------------------------------------------------===//
413
414Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
415 // Extract some information from the instruction
416 Value *Dst = CI->getArgOperand(i: 0);
417 Value *Src = CI->getArgOperand(i: 1);
418 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
419
420 // See if we can get the length of the input string.
421 uint64_t Len = GetStringLength(V: Src);
422 if (Len)
423 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
424 else
425 return nullptr;
426 --Len; // Unbias length.
427
428 // Handle the simple, do-nothing case: strcat(x, "") -> x
429 if (Len == 0)
430 return Dst;
431
432 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len, B));
433}
434
435Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
436 IRBuilderBase &B) {
437 // We need to find the end of the destination string. That's where the
438 // memory is to be moved to. We just generate a call to strlen.
439 Value *DstLen = emitStrLen(Ptr: Dst, B, DL, TLI);
440 if (!DstLen)
441 return nullptr;
442
443 // Now that we have the destination's length, we must index into the
444 // destination's pointer to get the actual memcpy destination (end of
445 // the string .. we're concatenating).
446 Value *CpyDst = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: DstLen, Name: "endptr");
447
448 // We have enough information to now generate the memcpy call to do the
449 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
450 B.CreateMemCpy(Dst: CpyDst, DstAlign: Align(1), Src, SrcAlign: Align(1),
451 Size: TLI->getAsSizeT(V: Len + 1, M: *B.getModule()));
452 return Dst;
453}
454
455Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
456 // Extract some information from the instruction.
457 Value *Dst = CI->getArgOperand(i: 0);
458 Value *Src = CI->getArgOperand(i: 1);
459 Value *Size = CI->getArgOperand(i: 2);
460 uint64_t Len;
461 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
462 if (isKnownNonZero(V: Size, Q: DL))
463 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
464
465 // We don't do anything if length is not constant.
466 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size);
467 if (LengthArg) {
468 Len = LengthArg->getZExtValue();
469 // strncat(x, c, 0) -> x
470 if (!Len)
471 return Dst;
472 } else {
473 return nullptr;
474 }
475
476 // See if we can get the length of the input string.
477 uint64_t SrcLen = GetStringLength(V: Src);
478 if (SrcLen) {
479 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
480 --SrcLen; // Unbias length.
481 } else {
482 return nullptr;
483 }
484
485 // strncat(x, "", c) -> x
486 if (SrcLen == 0)
487 return Dst;
488
489 // We don't optimize this case.
490 if (Len < SrcLen)
491 return nullptr;
492
493 // strncat(x, s, c) -> strcat(x, s)
494 // s is constant so the strcat can be optimized further.
495 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len: SrcLen, B));
496}
497
498// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
499// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
500// is that either S is dereferenceable or the value of N is nonzero.
501static Value* memChrToCharCompare(CallInst *CI, Value *NBytes,
502 IRBuilderBase &B, const DataLayout &DL)
503{
504 Value *Src = CI->getArgOperand(i: 0);
505 Value *CharVal = CI->getArgOperand(i: 1);
506
507 // Fold memchr(A, C, N) == A to N && *A == C.
508 Type *CharTy = B.getInt8Ty();
509 Value *Char0 = B.CreateLoad(Ty: CharTy, Ptr: Src);
510 CharVal = B.CreateTrunc(V: CharVal, DestTy: CharTy);
511 Value *Cmp = B.CreateICmpEQ(LHS: Char0, RHS: CharVal, Name: "char0cmp");
512
513 if (NBytes) {
514 Value *Zero = ConstantInt::get(Ty: NBytes->getType(), V: 0);
515 Value *And = B.CreateICmpNE(LHS: NBytes, RHS: Zero);
516 Cmp = B.CreateLogicalAnd(Cond1: And, Cond2: Cmp);
517 // The and above is based on the byte count and the query, neither of which
518 // we know without value profiling, so mark the profile as unknown.
519 if (auto *SI = dyn_cast<SelectInst>(Val: Cmp))
520 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE);
521 }
522
523 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
524 return B.CreateSelect(C: Cmp, True: Src, False: NullPtr);
525}
526
527Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
528 Value *SrcStr = CI->getArgOperand(i: 0);
529 Value *CharVal = CI->getArgOperand(i: 1);
530 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
531
532 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
533 return memChrToCharCompare(CI, NBytes: nullptr, B, DL);
534
535 // If the second operand is non-constant, see if we can compute the length
536 // of the input string and turn this into memchr.
537 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
538 if (!CharC) {
539 uint64_t Len = GetStringLength(V: SrcStr);
540 if (Len)
541 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len);
542 else
543 return nullptr;
544
545 Function *Callee = CI->getCalledFunction();
546 FunctionType *FT = Callee->getFunctionType();
547 unsigned IntBits = TLI->getIntSize();
548 if (!FT->getParamType(i: 1)->isIntegerTy(BitWidth: IntBits)) // memchr needs 'int'.
549 return nullptr;
550
551 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
552 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
553 return copyFlags(Old: *CI,
554 New: emitMemChr(Ptr: SrcStr, Val: CharVal, // include nul.
555 Len: ConstantInt::get(Ty: SizeTTy, V: Len), B,
556 DL, TLI));
557 }
558
559 if (CharC->isZero()) {
560 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
561 if (isOnlyUsedInEqualityComparison(V: CI, With: NullPtr))
562 // Pre-empt the transformation to strlen below and fold
563 // strchr(A, '\0') == null to false.
564 return B.CreateIntToPtr(V: B.getTrue(), DestTy: CI->getType());
565 }
566
567 // Otherwise, the character is a constant, see if the first argument is
568 // a string literal. If so, we can constant fold.
569 StringRef Str;
570 if (!getConstantStringInfo(V: SrcStr, Str)) {
571 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
572 if (Value *StrLen = emitStrLen(Ptr: SrcStr, B, DL, TLI))
573 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: StrLen, Name: "strchr");
574 return nullptr;
575 }
576
577 // Compute the offset, make sure to handle the case when we're searching for
578 // zero (a weird way to spell strlen).
579 size_t I = (0xFF & CharC->getSExtValue()) == 0
580 ? Str.size()
581 : Str.find(C: CharC->getSExtValue());
582 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
583 return Constant::getNullValue(Ty: CI->getType());
584
585 // strchr(s+n,c) -> gep(s+n+i,c)
586 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: I), Name: "strchr");
587}
588
589Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
590 Value *SrcStr = CI->getArgOperand(i: 0);
591 Value *CharVal = CI->getArgOperand(i: 1);
592 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
593 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
594
595 StringRef Str;
596 if (!getConstantStringInfo(V: SrcStr, Str)) {
597 // strrchr(s, 0) -> strchr(s, 0)
598 if (CharC && CharC->isZero())
599 return copyFlags(Old: *CI, New: emitStrChr(Ptr: SrcStr, C: '\0', B, TLI));
600 return nullptr;
601 }
602
603 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
604 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
605
606 // Try to expand strrchr to the memrchr nonstandard extension if it's
607 // available, or simply fail otherwise.
608 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
609 Value *Size = ConstantInt::get(Ty: SizeTTy, V: NBytes);
610 return copyFlags(Old: *CI, New: emitMemRChr(Ptr: SrcStr, Val: CharVal, Len: Size, B, DL, TLI));
611}
612
613Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
614 Value *Str1P = CI->getArgOperand(i: 0), *Str2P = CI->getArgOperand(i: 1);
615 if (Str1P == Str2P) // strcmp(x,x) -> 0
616 return ConstantInt::get(Ty: CI->getType(), V: 0);
617
618 StringRef Str1, Str2;
619 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
620 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
621
622 // strcmp(x, y) -> cnst (if both x and y are constant strings)
623 if (HasStr1 && HasStr2)
624 return ConstantInt::getSigned(Ty: CI->getType(),
625 V: std::clamp(val: Str1.compare(RHS: Str2), lo: -1, hi: 1));
626
627 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
628 return B.CreateNeg(V: B.CreateZExt(
629 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
630
631 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
632 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
633 DestTy: CI->getType());
634
635 // strcmp(P, "x") -> memcmp(P, "x", 2)
636 uint64_t Len1 = GetStringLength(V: Str1P);
637 if (Len1)
638 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
639 uint64_t Len2 = GetStringLength(V: Str2P);
640 if (Len2)
641 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
642
643 if (Len1 && Len2) {
644 return copyFlags(
645 Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
646 Len: TLI->getAsSizeT(V: std::min(a: Len1, b: Len2), M: *CI->getModule()),
647 B, DL, TLI));
648 }
649
650 // strcmp to memcmp
651 SimplifyQuery SQ(DL, TLI, DT, AC, CI);
652 if (!HasStr1 && HasStr2) {
653 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ))
654 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
655 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
656 B, DL, TLI));
657 } else if (HasStr1 && !HasStr2) {
658 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ))
659 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
660 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
661 B, DL, TLI));
662 }
663
664 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
665 return nullptr;
666}
667
668// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
669// arrays LHS and RHS and nonconstant Size.
670static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
671 Value *Size, bool StrNCmp,
672 IRBuilderBase &B, const DataLayout &DL);
673
674Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
675 Value *Str1P = CI->getArgOperand(i: 0);
676 Value *Str2P = CI->getArgOperand(i: 1);
677 Value *Size = CI->getArgOperand(i: 2);
678 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
679 return ConstantInt::get(Ty: CI->getType(), V: 0);
680
681 if (isKnownNonZero(V: Size, Q: DL))
682 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
683 // Get the length argument if it is constant.
684 uint64_t Length;
685 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size))
686 Length = LengthArg->getZExtValue();
687 else
688 return optimizeMemCmpVarSize(CI, LHS: Str1P, RHS: Str2P, Size, StrNCmp: true, B, DL);
689
690 if (Length == 0) // strncmp(x,y,0) -> 0
691 return ConstantInt::get(Ty: CI->getType(), V: 0);
692
693 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
694 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P, Len: Size, B, DL, TLI));
695
696 StringRef Str1, Str2;
697 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
698 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
699
700 // strncmp(x, y) -> cnst (if both x and y are constant strings)
701 if (HasStr1 && HasStr2) {
702 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
703 StringRef SubStr1 = substr(Str: Str1, Len: Length);
704 StringRef SubStr2 = substr(Str: Str2, Len: Length);
705 return ConstantInt::getSigned(Ty: CI->getType(),
706 V: std::clamp(val: SubStr1.compare(RHS: SubStr2), lo: -1, hi: 1));
707 }
708
709 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
710 return B.CreateNeg(V: B.CreateZExt(
711 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
712
713 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
714 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
715 DestTy: CI->getType());
716
717 uint64_t Len1 = GetStringLength(V: Str1P);
718 if (Len1)
719 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
720 uint64_t Len2 = GetStringLength(V: Str2P);
721 if (Len2)
722 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
723
724 // strncmp to memcmp
725 if (!HasStr1 && HasStr2) {
726 Len2 = std::min(a: Len2, b: Length);
727 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ: DL))
728 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
729 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
730 B, DL, TLI));
731 } else if (HasStr1 && !HasStr2) {
732 Len1 = std::min(a: Len1, b: Length);
733 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ: DL))
734 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
735 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
736 B, DL, TLI));
737 }
738
739 return nullptr;
740}
741
742Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
743 Value *Src = CI->getArgOperand(i: 0);
744 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
745 uint64_t SrcLen = GetStringLength(V: Src);
746 if (SrcLen && Size) {
747 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: SrcLen);
748 if (SrcLen <= Size->getZExtValue() + 1)
749 return copyFlags(Old: *CI, New: emitStrDup(Ptr: Src, B, TLI));
750 }
751
752 return nullptr;
753}
754
755Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
756 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
757 if (Dst == Src) // strcpy(x,x) -> x
758 return Src;
759
760 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
761 // See if we can get the length of the input string.
762 uint64_t Len = GetStringLength(V: Src);
763 if (Len)
764 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
765 else
766 return nullptr;
767
768 // We have enough information to now generate the memcpy call to do the
769 // copy for us. Make a memcpy to copy the nul byte with align = 1.
770 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
771 Size: TLI->getAsSizeT(V: Len, M: *CI->getModule()));
772 mergeAttributesAndFlags(NewCI, Old: *CI);
773 return Dst;
774}
775
776Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
777 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
778
779 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
780 if (CI->use_empty())
781 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
782
783 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
784 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
785 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
786 }
787
788 // See if we can get the length of the input string.
789 uint64_t Len = GetStringLength(V: Src);
790 if (Len)
791 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
792 else
793 return nullptr;
794
795 Value *LenV = TLI->getAsSizeT(V: Len, M: *CI->getModule());
796 Value *DstEnd = B.CreateInBoundsGEP(
797 Ty: B.getInt8Ty(), Ptr: Dst, IdxList: TLI->getAsSizeT(V: Len - 1, M: *CI->getModule()));
798
799 // We have enough information to now generate the memcpy call to do the
800 // copy for us. Make a memcpy to copy the nul byte with align = 1.
801 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: LenV);
802 mergeAttributesAndFlags(NewCI, Old: *CI);
803 return DstEnd;
804}
805
806// Optimize a call to size_t strlcpy(char*, const char*, size_t).
807
808Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
809 Value *Size = CI->getArgOperand(i: 2);
810 if (isKnownNonZero(V: Size, Q: DL))
811 // Like snprintf, the function stores into the destination only when
812 // the size argument is nonzero.
813 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
814 // The function reads the source argument regardless of Size (it returns
815 // its length).
816 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
817
818 uint64_t NBytes;
819 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
820 NBytes = SizeC->getZExtValue();
821 else
822 return nullptr;
823
824 Value *Dst = CI->getArgOperand(i: 0);
825 Value *Src = CI->getArgOperand(i: 1);
826 if (NBytes <= 1) {
827 if (NBytes == 1)
828 // For a call to strlcpy(D, S, 1) first store a nul in *D.
829 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
830
831 // Transform strlcpy(D, S, 0) to a call to strlen(S).
832 return copyFlags(Old: *CI, New: emitStrLen(Ptr: Src, B, DL, TLI));
833 }
834
835 // Try to determine the length of the source, substituting its size
836 // when it's not nul-terminated (as it's required to be) to avoid
837 // reading past its end.
838 StringRef Str;
839 if (!getConstantStringInfo(V: Src, Str, /*TrimAtNul=*/false))
840 return nullptr;
841
842 uint64_t SrcLen = Str.find(C: '\0');
843 // Set if the terminating nul should be copied by the call to memcpy
844 // below.
845 bool NulTerm = SrcLen < NBytes;
846
847 if (NulTerm)
848 // Overwrite NBytes with the number of bytes to copy, including
849 // the terminating nul.
850 NBytes = SrcLen + 1;
851 else {
852 // Set the length of the source for the function to return to its
853 // size, and cap NBytes at the same.
854 SrcLen = std::min(a: SrcLen, b: uint64_t(Str.size()));
855 NBytes = std::min(a: NBytes - 1, b: SrcLen);
856 }
857
858 if (SrcLen == 0) {
859 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
860 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
861 return ConstantInt::get(Ty: CI->getType(), V: 0);
862 }
863
864 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
865 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
866 // D[N' - 1] if necessary.
867 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
868 Size: TLI->getAsSizeT(V: NBytes, M: *CI->getModule()));
869 mergeAttributesAndFlags(NewCI, Old: *CI);
870
871 if (!NulTerm) {
872 Value *EndOff = ConstantInt::get(Ty: CI->getType(), V: NBytes);
873 Value *EndPtr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: EndOff);
874 B.CreateStore(Val: B.getInt8(C: 0), Ptr: EndPtr);
875 }
876
877 // Like snprintf, strlcpy returns the number of nonzero bytes that would
878 // have been copied if the bound had been sufficiently big (which in this
879 // case is strlen(Src)).
880 return ConstantInt::get(Ty: CI->getType(), V: SrcLen);
881}
882
883// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
884// otherwise.
885Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
886 IRBuilderBase &B) {
887 Value *Dst = CI->getArgOperand(i: 0);
888 Value *Src = CI->getArgOperand(i: 1);
889 Value *Size = CI->getArgOperand(i: 2);
890
891 if (isKnownNonZero(V: Size, Q: DL)) {
892 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
893 // only when N is nonzero.
894 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
895 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
896 }
897
898 // If the "bound" argument is known set N to it. Otherwise set it to
899 // UINT64_MAX and handle it later.
900 uint64_t N = UINT64_MAX;
901 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
902 N = SizeC->getZExtValue();
903
904 if (N == 0)
905 // Fold st{p,r}ncpy(D, S, 0) to D.
906 return Dst;
907
908 if (N == 1) {
909 Type *CharTy = B.getInt8Ty();
910 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "stxncpy.char0");
911 B.CreateStore(Val: CharVal, Ptr: Dst);
912 if (!RetEnd)
913 // Transform strncpy(D, S, 1) to return (*D = *S), D.
914 return Dst;
915
916 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
917 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
918 Value *Cmp = B.CreateICmpEQ(LHS: CharVal, RHS: ZeroChar, Name: "stpncpy.char0cmp");
919
920 Value *Off1 = B.getInt32(C: 1);
921 Value *EndPtr = B.CreateInBoundsGEP(Ty: CharTy, Ptr: Dst, IdxList: Off1, Name: "stpncpy.end");
922 return B.CreateSelect(C: Cmp, True: Dst, False: EndPtr, Name: "stpncpy.sel");
923 }
924
925 // If the length of the input string is known set SrcLen to it.
926 uint64_t SrcLen = GetStringLength(V: Src);
927 if (SrcLen)
928 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
929 else
930 return nullptr;
931
932 --SrcLen; // Unbias length.
933
934 if (SrcLen == 0) {
935 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
936 Align MemSetAlign =
937 CI->getAttributes().getParamAttrs(ArgNo: 0).getAlignment().valueOrOne();
938 CallInst *NewCI = B.CreateMemSet(Ptr: Dst, Val: B.getInt8(C: '\0'), Size, Align: MemSetAlign);
939 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(ArgNo: 0));
940 NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
941 C&: CI->getContext(), ArgNo: 0, B: ArgAttrs));
942 copyFlags(Old: *CI, New: NewCI);
943 return Dst;
944 }
945
946 if (N > SrcLen + 1) {
947 if (N > 128)
948 // Bail if N is large or unknown.
949 return nullptr;
950
951 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
952 StringRef Str;
953 if (!getConstantStringInfo(V: Src, Str))
954 return nullptr;
955 std::string SrcStr = Str.str();
956 // Create a bigger, nul-padded array with the same length, SrcLen,
957 // as the original string.
958 SrcStr.resize(n: N, c: '\0');
959 Src = B.CreateGlobalString(Str: SrcStr, Name: "str", /*AddressSpace=*/0,
960 /*M=*/nullptr, /*AddNull=*/false);
961 }
962
963 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
964 // S and N are constant.
965 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
966 Size: TLI->getAsSizeT(V: N, M: *CI->getModule()));
967 mergeAttributesAndFlags(NewCI, Old: *CI);
968 if (!RetEnd)
969 return Dst;
970
971 // stpncpy(D, S, N) returns the address of the first null in D if it writes
972 // one, otherwise D + N.
973 Value *Off = B.getInt64(C: std::min(a: SrcLen, b: N));
974 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: Off, Name: "endptr");
975}
976
977Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
978 unsigned CharSize,
979 Value *Bound) {
980 Value *Src = CI->getArgOperand(i: 0);
981 Type *CharTy = B.getIntNTy(N: CharSize);
982
983 if (isOnlyUsedInZeroEqualityComparison(CtxI: CI) &&
984 (!Bound || isKnownNonZero(V: Bound, Q: DL))) {
985 // Fold strlen:
986 // strlen(x) != 0 --> *x != 0
987 // strlen(x) == 0 --> *x == 0
988 // and likewise strnlen with constant N > 0:
989 // strnlen(x, N) != 0 --> *x != 0
990 // strnlen(x, N) == 0 --> *x == 0
991 return B.CreateZExt(V: B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "char0"),
992 DestTy: CI->getType());
993 }
994
995 if (Bound) {
996 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Val: Bound)) {
997 if (BoundCst->isZero())
998 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
999 return ConstantInt::get(Ty: CI->getType(), V: 0);
1000
1001 if (BoundCst->isOne()) {
1002 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
1003 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "strnlen.char0");
1004 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
1005 Value *Cmp = B.CreateICmpNE(LHS: CharVal, RHS: ZeroChar, Name: "strnlen.char0cmp");
1006 return B.CreateZExt(V: Cmp, DestTy: CI->getType());
1007 }
1008 }
1009 }
1010
1011 if (uint64_t Len = GetStringLength(V: Src, CharSize)) {
1012 Value *LenC = ConstantInt::get(Ty: CI->getType(), V: Len - 1);
1013 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
1014 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
1015 if (Bound)
1016 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: LenC, RHS: Bound);
1017 return LenC;
1018 }
1019
1020 if (Bound)
1021 // Punt for strnlen for now.
1022 return nullptr;
1023
1024 // If s is a constant pointer pointing to a string literal, we can fold
1025 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
1026 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
1027 // We only try to simplify strlen when the pointer s points to an array
1028 // of CharSize elements. Otherwise, we would need to scale the offset x before
1029 // doing the subtraction. This will make the optimization more complex, and
1030 // it's not very useful because calling strlen for a pointer of other types is
1031 // very uncommon.
1032 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: Src)) {
1033 unsigned BW = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1034 SmallMapVector<Value *, APInt, 4> VarOffsets;
1035 APInt ConstOffset(BW, 0);
1036 assert(CharSize % 8 == 0 && "Expected a multiple of 8 sized CharSize");
1037 // Check the gep is a single variable offset.
1038 if (!GEP->collectOffset(DL, BitWidth: BW, VariableOffsets&: VarOffsets, ConstantOffset&: ConstOffset) ||
1039 VarOffsets.size() != 1 || ConstOffset != 0 ||
1040 VarOffsets.begin()->second != CharSize / 8)
1041 return nullptr;
1042
1043 ConstantDataArraySlice Slice;
1044 if (getConstantDataArrayInfo(V: GEP->getOperand(i_nocapture: 0), Slice, ElementSize: CharSize)) {
1045 uint64_t NullTermIdx;
1046 if (Slice.Array == nullptr) {
1047 NullTermIdx = 0;
1048 } else {
1049 NullTermIdx = ~((uint64_t)0);
1050 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
1051 if (Slice.Array->getElementAsInteger(i: I + Slice.Offset) == 0) {
1052 NullTermIdx = I;
1053 break;
1054 }
1055 }
1056 // If the string does not have '\0', leave it to strlen to compute
1057 // its length.
1058 if (NullTermIdx == ~((uint64_t)0))
1059 return nullptr;
1060 }
1061
1062 Value *Offset = VarOffsets.begin()->first;
1063 KnownBits Known = computeKnownBits(V: Offset, DL, AC: nullptr, CtxI: CI, DT: nullptr);
1064
1065 // If Offset is not provably in the range [0, NullTermIdx], we can still
1066 // optimize if we can prove that the program has undefined behavior when
1067 // Offset is outside that range. That is the case when GEP->getOperand(0)
1068 // is a pointer to an object whose memory extent is NullTermIdx+1.
1069 if ((Known.isNonNegative() && Known.getMaxValue().ule(RHS: NullTermIdx)) ||
1070 (isa<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)) &&
1071 NullTermIdx == Slice.Length - 1)) {
1072 Offset = B.CreateSExtOrTrunc(V: Offset, DestTy: CI->getType());
1073 return B.CreateSub(LHS: ConstantInt::get(Ty: CI->getType(), V: NullTermIdx),
1074 RHS: Offset);
1075 }
1076 }
1077 }
1078
1079 // strlen(x?"foo":"bars") --> x ? 3 : 4
1080 if (SelectInst *SI = dyn_cast<SelectInst>(Val: Src)) {
1081 uint64_t LenTrue = GetStringLength(V: SI->getTrueValue(), CharSize);
1082 uint64_t LenFalse = GetStringLength(V: SI->getFalseValue(), CharSize);
1083 if (LenTrue && LenFalse) {
1084 ORE.emit(RemarkBuilder: [&]() {
1085 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1086 << "folded strlen(select) to select of constants";
1087 });
1088 return B.CreateSelect(C: SI->getCondition(),
1089 True: ConstantInt::get(Ty: CI->getType(), V: LenTrue - 1),
1090 False: ConstantInt::get(Ty: CI->getType(), V: LenFalse - 1), Name: "",
1091 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : SI);
1092 }
1093 }
1094
1095 return nullptr;
1096}
1097
1098Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1099 if (Value *V = optimizeStringLength(CI, B, CharSize: 8))
1100 return V;
1101 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1102 return nullptr;
1103}
1104
1105Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1106 Value *Bound = CI->getArgOperand(i: 1);
1107 if (Value *V = optimizeStringLength(CI, B, CharSize: 8, Bound))
1108 return V;
1109
1110 if (isKnownNonZero(V: Bound, Q: DL))
1111 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1112 return nullptr;
1113}
1114
1115Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1116 Module &M = *CI->getModule();
1117 unsigned WCharSize = M.getWCharSize();
1118
1119 if (Value *V = optimizeStringLength(CI, B, CharSize: WCharSize * 8))
1120 return V;
1121 // At least the terminating wide character is read.
1122 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0, CharSize: WCharSize);
1123 return nullptr;
1124}
1125
1126Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1127 StringRef S1, S2;
1128 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1129 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1130
1131 // strpbrk(s, "") -> nullptr
1132 // strpbrk("", s) -> nullptr
1133 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1134 return Constant::getNullValue(Ty: CI->getType());
1135
1136 // Constant folding.
1137 if (HasS1 && HasS2) {
1138 size_t I = S1.find_first_of(Chars: S2);
1139 if (I == StringRef::npos) // No match.
1140 return Constant::getNullValue(Ty: CI->getType());
1141
1142 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1143 IdxList: B.getInt64(C: I), Name: "strpbrk");
1144 }
1145
1146 // strpbrk(s, "a") -> strchr(s, 'a')
1147 if (HasS2 && S2.size() == 1)
1148 return copyFlags(Old: *CI, New: emitStrChr(Ptr: CI->getArgOperand(i: 0), C: S2[0], B, TLI));
1149
1150 return nullptr;
1151}
1152
1153Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1154 Value *EndPtr = CI->getArgOperand(i: 1);
1155 if (isa<ConstantPointerNull>(Val: EndPtr)) {
1156 // With a null EndPtr, this function won't capture the main argument.
1157 // It would be readonly too, except that it still may write to errno.
1158 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
1159 CI: CaptureInfo::none()));
1160 }
1161
1162 return nullptr;
1163}
1164
1165Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1166 StringRef S1, S2;
1167 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1168 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1169
1170 // strspn(s, "") -> 0
1171 // strspn("", s) -> 0
1172 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1173 return Constant::getNullValue(Ty: CI->getType());
1174
1175 // Constant folding.
1176 if (HasS1 && HasS2) {
1177 size_t Pos = S1.find_first_not_of(Chars: S2);
1178 if (Pos == StringRef::npos)
1179 Pos = S1.size();
1180 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1181 }
1182
1183 return nullptr;
1184}
1185
1186Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1187 StringRef S1, S2;
1188 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1189 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1190
1191 // strcspn("", s) -> 0
1192 if (HasS1 && S1.empty())
1193 return Constant::getNullValue(Ty: CI->getType());
1194
1195 // Constant folding.
1196 if (HasS1 && HasS2) {
1197 size_t Pos = S1.find_first_of(Chars: S2);
1198 if (Pos == StringRef::npos)
1199 Pos = S1.size();
1200 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1201 }
1202
1203 // strcspn(s, "") -> strlen(s)
1204 if (HasS2 && S2.empty())
1205 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B, DL, TLI));
1206
1207 return nullptr;
1208}
1209
1210Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1211 // fold strstr(x, x) -> x.
1212 if (CI->getArgOperand(i: 0) == CI->getArgOperand(i: 1))
1213 return CI->getArgOperand(i: 0);
1214
1215 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1216 if (isOnlyUsedInEqualityComparison(V: CI, With: CI->getArgOperand(i: 0))) {
1217 Value *StrLen = emitStrLen(Ptr: CI->getArgOperand(i: 1), B, DL, TLI);
1218 if (!StrLen)
1219 return nullptr;
1220 Value *StrNCmp = emitStrNCmp(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
1221 Len: StrLen, B, DL, TLI);
1222 if (!StrNCmp)
1223 return nullptr;
1224 for (User *U : llvm::make_early_inc_range(Range: CI->users())) {
1225 ICmpInst *Old = cast<ICmpInst>(Val: U);
1226 Value *Cmp =
1227 B.CreateICmp(P: Old->getPredicate(), LHS: StrNCmp,
1228 RHS: ConstantInt::getNullValue(Ty: StrNCmp->getType()), Name: "cmp");
1229 replaceAllUsesWith(I: Old, With: Cmp);
1230 }
1231 return CI;
1232 }
1233
1234 // See if either input string is a constant string.
1235 StringRef SearchStr, ToFindStr;
1236 bool HasStr1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: SearchStr);
1237 bool HasStr2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: ToFindStr);
1238
1239 // fold strstr(x, "") -> x.
1240 if (HasStr2 && ToFindStr.empty())
1241 return CI->getArgOperand(i: 0);
1242
1243 // If both strings are known, constant fold it.
1244 if (HasStr1 && HasStr2) {
1245 size_t Offset = SearchStr.find(Str: ToFindStr);
1246
1247 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1248 return Constant::getNullValue(Ty: CI->getType());
1249
1250 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1251 return B.CreateConstInBoundsGEP1_64(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1252 Idx0: Offset, Name: "strstr");
1253 }
1254
1255 // fold strstr(x, "y") -> strchr(x, 'y').
1256 if (HasStr2 && ToFindStr.size() == 1) {
1257 return emitStrChr(Ptr: CI->getArgOperand(i: 0), C: ToFindStr[0], B, TLI);
1258 }
1259
1260 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
1261 return nullptr;
1262}
1263
1264Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1265 Value *SrcStr = CI->getArgOperand(i: 0);
1266 Value *Size = CI->getArgOperand(i: 2);
1267 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1268 Value *CharVal = CI->getArgOperand(i: 1);
1269 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1270 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1271
1272 if (LenC) {
1273 if (LenC->isZero())
1274 // Fold memrchr(x, y, 0) --> null.
1275 return NullPtr;
1276
1277 if (LenC->isOne()) {
1278 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1279 // constant or otherwise.
1280 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memrchr.char0");
1281 // Slice off the character's high end bits.
1282 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1283 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memrchr.char0cmp");
1284 return B.CreateSelectWithUnknownProfile(C: Cmp, True: SrcStr, False: NullPtr, DEBUG_TYPE,
1285 Name: "memrchr.sel");
1286 }
1287 }
1288
1289 StringRef Str;
1290 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1291 return nullptr;
1292
1293 if (Str.size() == 0)
1294 // If the array is empty fold memrchr(A, C, N) to null for any value
1295 // of C and N on the basis that the only valid value of N is zero
1296 // (otherwise the call is undefined).
1297 return NullPtr;
1298
1299 uint64_t EndOff = UINT64_MAX;
1300 if (LenC) {
1301 EndOff = LenC->getZExtValue();
1302 if (Str.size() < EndOff)
1303 // Punt out-of-bounds accesses to sanitizers and/or libc.
1304 return nullptr;
1305 }
1306
1307 if (ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal)) {
1308 // Fold memrchr(S, C, N) for a constant C.
1309 size_t Pos = Str.rfind(C: CharC->getZExtValue(), From: EndOff);
1310 if (Pos == StringRef::npos)
1311 // When the character is not in the source array fold the result
1312 // to null regardless of Size.
1313 return NullPtr;
1314
1315 if (LenC)
1316 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1317 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos));
1318
1319 if (Str.find(C: Str[Pos]) == Pos) {
1320 // When there is just a single occurrence of C in S, i.e., the one
1321 // in Str[Pos], fold
1322 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1323 // for nonconstant N.
1324 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1325 Name: "memrchr.cmp");
1326 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr,
1327 IdxList: B.getInt64(C: Pos), Name: "memrchr.ptr_plus");
1328 return B.CreateSelectWithUnknownProfile(C: Cmp, True: NullPtr, False: SrcPlus, DEBUG_TYPE,
1329 Name: "memrchr.sel");
1330 }
1331 }
1332
1333 // Truncate the string to search at most EndOff characters.
1334 Str = Str.substr(Start: 0, N: EndOff);
1335 if (Str.find_first_not_of(C: Str[0]) != StringRef::npos)
1336 return nullptr;
1337
1338 // If the source array consists of all equal characters, then for any
1339 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1340 // N != 0 && *S == C ? S + N - 1 : null
1341 Type *SizeTy = Size->getType();
1342 Type *Int8Ty = B.getInt8Ty();
1343 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1344 // Slice off the sought character's high end bits.
1345 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1346 Value *CEqS0 =
1347 B.CreateICmpEQ(LHS: ConstantInt::get(Ty: Int8Ty, V: (unsigned char)Str[0]), RHS: CharVal);
1348 Value *And = B.CreateLogicalAnd(Cond1: NNeZ, Cond2: CEqS0);
1349 if (auto *AndSI = dyn_cast<SelectInst>(Val: And))
1350 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *AndSI, DEBUG_TYPE);
1351 Value *SizeM1 = B.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
1352 Value *SrcPlus =
1353 B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: SrcStr, IdxList: SizeM1, Name: "memrchr.ptr_plus");
1354 return B.CreateSelectWithUnknownProfile(C: And, True: SrcPlus, False: NullPtr, DEBUG_TYPE,
1355 Name: "memrchr.sel");
1356}
1357
1358Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1359 Value *SrcStr = CI->getArgOperand(i: 0);
1360 Value *Size = CI->getArgOperand(i: 2);
1361
1362 if (isKnownNonZero(V: Size, Q: DL)) {
1363 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1364 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1365 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1366 }
1367
1368 Value *CharVal = CI->getArgOperand(i: 1);
1369 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
1370 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1371 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1372
1373 // memchr(x, y, 0) -> null
1374 if (LenC) {
1375 if (LenC->isZero())
1376 return NullPtr;
1377
1378 if (LenC->isOne()) {
1379 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1380 // constant or otherwise.
1381 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memchr.char0");
1382 // Slice off the character's high end bits.
1383 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1384 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memchr.char0cmp");
1385 // The condition depends on the value of the string being equal to the
1386 // query, neither of which we know without value profiling, so mark the
1387 // profile unknown.
1388 return B.CreateSelectWithUnknownProfile(C: Cmp, True: SrcStr, False: NullPtr, DEBUG_TYPE,
1389 Name: "memchr.sel");
1390 }
1391 }
1392
1393 StringRef Str;
1394 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1395 return nullptr;
1396
1397 if (CharC) {
1398 size_t Pos = Str.find(C: CharC->getZExtValue());
1399 if (Pos == StringRef::npos)
1400 // When the character is not in the source array fold the result
1401 // to null regardless of Size.
1402 return NullPtr;
1403
1404 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1405 // When the constant Size is less than or equal to the character
1406 // position also fold the result to null.
1407 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1408 Name: "memchr.cmp");
1409 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos),
1410 Name: "memchr.ptr");
1411 // The condition is dependent upon the value of n, which we cannot infer
1412 // without value profiling, so mark the profile unknown.
1413 return B.CreateSelectWithUnknownProfile(C: Cmp, True: NullPtr, False: SrcPlus, DEBUG_TYPE);
1414 }
1415
1416 if (Str.size() == 0)
1417 // If the array is empty fold memchr(A, C, N) to null for any value
1418 // of C and N on the basis that the only valid value of N is zero
1419 // (otherwise the call is undefined).
1420 return NullPtr;
1421
1422 if (LenC)
1423 Str = substr(Str, Len: LenC->getZExtValue());
1424
1425 size_t Pos = Str.find_first_not_of(C: Str[0]);
1426 if (Pos == StringRef::npos
1427 || Str.find_first_not_of(C: Str[Pos], From: Pos) == StringRef::npos) {
1428 // If the source array consists of at most two consecutive sequences
1429 // of the same characters, then for any C and N (whether in bounds or
1430 // not), fold memchr(S, C, N) to
1431 // N != 0 && *S == C ? S : null
1432 // or for the two sequences to:
1433 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1434 // ^Sel2 ^Sel1 are denoted above.
1435 // The latter makes it also possible to fold strchr() calls with strings
1436 // of the same characters.
1437 Type *SizeTy = Size->getType();
1438 Type *Int8Ty = B.getInt8Ty();
1439
1440 // Slice off the sought character's high end bits.
1441 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1442
1443 Value *Sel1 = NullPtr;
1444 if (Pos != StringRef::npos) {
1445 // Handle two consecutive sequences of the same characters.
1446 Value *PosVal = ConstantInt::get(Ty: SizeTy, V: Pos);
1447 Value *StrPos = ConstantInt::get(Ty: Int8Ty, V: (unsigned char)Str[Pos]);
1448 Value *CEqSPos = B.CreateICmpEQ(LHS: CharVal, RHS: StrPos);
1449 Value *NGtPos = B.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: Size, RHS: PosVal);
1450 Value *And = B.CreateAnd(LHS: CEqSPos, RHS: NGtPos);
1451 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: PosVal);
1452 // The condition depends on the value of the query and size, neither of
1453 // which we know without value profiling, so mark the profile unknown.
1454 Sel1 = B.CreateSelectWithUnknownProfile(C: And, True: SrcPlus, False: NullPtr, DEBUG_TYPE,
1455 Name: "memchr.sel1");
1456 }
1457
1458 Value *Str0 = ConstantInt::get(Ty: Int8Ty, V: (unsigned char)Str[0]);
1459 Value *CEqS0 = B.CreateICmpEQ(LHS: Str0, RHS: CharVal);
1460 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1461 Value *And = B.CreateAnd(LHS: NNeZ, RHS: CEqS0);
1462 // The condition depends on the value of the query and size, neither of
1463 // which we know without value profiling, so mark the profile unknown.
1464 return B.CreateSelectWithUnknownProfile(C: And, True: SrcStr, False: Sel1, DEBUG_TYPE,
1465 Name: "memchr.sel2");
1466 }
1467
1468 if (!LenC) {
1469 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1470 // S is dereferenceable so it's safe to load from it and fold
1471 // memchr(S, C, N) == S to N && *S == C for any C and N.
1472 // TODO: This is safe even for nonconstant S.
1473 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1474
1475 // From now on we need a constant length and constant array.
1476 return nullptr;
1477 }
1478
1479 bool OptForSize = llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
1480 QueryType: PGSOQueryType::IRPass);
1481
1482 // If the char is variable but the input str and length are not we can turn
1483 // this memchr call into a simple bit field test. Of course this only works
1484 // when the return value is only checked against null.
1485 //
1486 // It would be really nice to reuse switch lowering here but we can't change
1487 // the CFG at this point.
1488 //
1489 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1490 // != 0
1491 // after bounds check.
1492 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CtxI: CI))
1493 return nullptr;
1494
1495 unsigned char Max =
1496 *std::max_element(first: reinterpret_cast<const unsigned char *>(Str.begin()),
1497 last: reinterpret_cast<const unsigned char *>(Str.end()));
1498
1499 // Make sure the bit field we're about to create fits in a register on the
1500 // target.
1501 // FIXME: On a 64 bit architecture this prevents us from using the
1502 // interesting range of alpha ascii chars. We could do better by emitting
1503 // two bitfields or shifting the range by 64 if no lower chars are used.
1504 if (!DL.fitsInLegalInteger(Width: Max + 1)) {
1505 // Build chain of ORs
1506 // Transform:
1507 // memchr("abcd", C, 4) != nullptr
1508 // to:
1509 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1510 std::string SortedStr = Str.str();
1511 llvm::sort(C&: SortedStr);
1512 // Compute the number of of non-contiguous ranges.
1513 unsigned NonContRanges = 1;
1514 for (size_t i = 1; i < SortedStr.size(); ++i) {
1515 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1516 NonContRanges++;
1517 }
1518 }
1519
1520 // Restrict this optimization to profitable cases with one or two range
1521 // checks.
1522 if (NonContRanges > 2)
1523 return nullptr;
1524
1525 // Slice off the character's high end bits.
1526 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1527
1528 SmallVector<Value *> CharCompares;
1529 for (unsigned char C : SortedStr)
1530 CharCompares.push_back(Elt: B.CreateICmpEQ(LHS: CharVal, RHS: B.getInt8(C)));
1531
1532 return B.CreateIntToPtr(V: B.CreateOr(Ops: CharCompares), DestTy: CI->getType());
1533 }
1534
1535 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1536 // creating unnecessary illegal types.
1537 unsigned char Width = NextPowerOf2(A: std::max(a: (unsigned char)7, b: Max));
1538
1539 // Now build the bit field.
1540 APInt Bitfield(Width, 0);
1541 for (char C : Str)
1542 Bitfield.setBit((unsigned char)C);
1543 Value *BitfieldC = B.getInt(AI: Bitfield);
1544
1545 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1546 Value *C = B.CreateZExtOrTrunc(V: CharVal, DestTy: BitfieldC->getType());
1547 C = B.CreateAnd(LHS: C, RHS: B.getIntN(N: Width, C: 0xFF));
1548
1549 // First check that the bit field access is within bounds.
1550 Value *Bounds = B.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: C, RHS: B.getIntN(N: Width, C: Width),
1551 Name: "memchr.bounds");
1552
1553 // Create code that checks if the given bit is set in the field.
1554 Value *Shl = B.CreateShl(LHS: B.getIntN(N: Width, C: 1ULL), RHS: C);
1555 Value *Bits = B.CreateIsNotNull(Arg: B.CreateAnd(LHS: Shl, RHS: BitfieldC), Name: "memchr.bits");
1556
1557 // Finally merge both checks and cast to pointer type. The inttoptr
1558 // implicitly zexts the i1 to intptr type.
1559 Value *Memchr = B.CreateLogicalAnd(Cond1: Bounds, Cond2: Bits, Name: "memchr");
1560 // We construct an and between the value of the memory and the bytes to search
1561 // for. We cannot infer how often this would be true without value profiling
1562 // for the query, so mark the profile unknown.
1563 if (auto *SI = dyn_cast<SelectInst>(Val: Memchr))
1564 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE);
1565 return B.CreateIntToPtr(V: Memchr, DestTy: CI->getType());
1566}
1567
1568// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1569// arrays LHS and RHS and nonconstant Size.
1570static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
1571 Value *Size, bool StrNCmp,
1572 IRBuilderBase &B, const DataLayout &DL) {
1573 if (LHS == RHS) // memcmp(s,s,x) -> 0
1574 return Constant::getNullValue(Ty: CI->getType());
1575
1576 StringRef LStr, RStr;
1577 if (!getConstantStringInfo(V: LHS, Str&: LStr, /*TrimAtNul=*/false) ||
1578 !getConstantStringInfo(V: RHS, Str&: RStr, /*TrimAtNul=*/false))
1579 return nullptr;
1580
1581 // If the contents of both constant arrays are known, fold a call to
1582 // memcmp(A, B, N) to
1583 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1584 // where Pos is the first mismatch between A and B, determined below.
1585
1586 uint64_t Pos = 0;
1587 Value *Zero = ConstantInt::get(Ty: CI->getType(), V: 0);
1588 for (uint64_t MinSize = std::min(a: LStr.size(), b: RStr.size()); ; ++Pos) {
1589 if (Pos == MinSize ||
1590 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1591 // One array is a leading part of the other of equal or greater
1592 // size, or for strncmp, the arrays are equal strings.
1593 // Fold the result to zero. Size is assumed to be in bounds, since
1594 // otherwise the call would be undefined.
1595 return Zero;
1596 }
1597
1598 if (LStr[Pos] != RStr[Pos])
1599 break;
1600 }
1601
1602 // Normalize the result.
1603 typedef unsigned char UChar;
1604 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1605 Value *MaxSize = ConstantInt::get(Ty: Size->getType(), V: Pos);
1606 Value *Cmp = B.CreateICmp(P: ICmpInst::ICMP_ULE, LHS: Size, RHS: MaxSize);
1607 Value *Res = ConstantInt::getSigned(Ty: CI->getType(), V: IRes);
1608 return B.CreateSelect(C: Cmp, True: Zero, False: Res);
1609}
1610
1611// Optimize a memcmp call CI with constant size Len.
1612static Value *optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS,
1613 uint64_t Len, IRBuilderBase &B,
1614 const DataLayout &DL) {
1615 if (Len == 0) // memcmp(s1,s2,0) -> 0
1616 return Constant::getNullValue(Ty: CI->getType());
1617
1618 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1619 if (Len == 1) {
1620 Value *LHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: LHS, Name: "lhsc"),
1621 DestTy: CI->getType(), Name: "lhsv");
1622 Value *RHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: RHS, Name: "rhsc"),
1623 DestTy: CI->getType(), Name: "rhsv");
1624 return B.CreateSub(LHS: LHSV, RHS: RHSV, Name: "chardiff");
1625 }
1626
1627 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1628 // TODO: The case where both inputs are constants does not need to be limited
1629 // to legal integers or equality comparison. See block below this.
1630 if (DL.isLegalInteger(Width: Len * 8) && isOnlyUsedInZeroEqualityComparison(CtxI: CI)) {
1631 IntegerType *IntType = IntegerType::get(C&: CI->getContext(), NumBits: Len * 8);
1632 Align PrefAlignment = DL.getPrefTypeAlign(Ty: IntType);
1633
1634 // First, see if we can fold either argument to a constant.
1635 Value *LHSV = nullptr;
1636 if (auto *LHSC = dyn_cast<Constant>(Val: LHS))
1637 LHSV = ConstantFoldLoadFromConstPtr(C: LHSC, Ty: IntType, DL);
1638
1639 Value *RHSV = nullptr;
1640 if (auto *RHSC = dyn_cast<Constant>(Val: RHS))
1641 RHSV = ConstantFoldLoadFromConstPtr(C: RHSC, Ty: IntType, DL);
1642
1643 // Don't generate unaligned loads. If either source is constant data,
1644 // alignment doesn't matter for that source because there is no load.
1645 if ((LHSV || getKnownAlignment(V: LHS, DL, CtxI: CI) >= PrefAlignment) &&
1646 (RHSV || getKnownAlignment(V: RHS, DL, CtxI: CI) >= PrefAlignment)) {
1647 if (!LHSV)
1648 LHSV = B.CreateLoad(Ty: IntType, Ptr: LHS, Name: "lhsv");
1649 if (!RHSV)
1650 RHSV = B.CreateLoad(Ty: IntType, Ptr: RHS, Name: "rhsv");
1651 return B.CreateZExt(V: B.CreateICmpNE(LHS: LHSV, RHS: RHSV), DestTy: CI->getType(), Name: "memcmp");
1652 }
1653 }
1654
1655 return nullptr;
1656}
1657
1658// Most simplifications for memcmp also apply to bcmp.
1659Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1660 IRBuilderBase &B) {
1661 Value *LHS = CI->getArgOperand(i: 0), *RHS = CI->getArgOperand(i: 1);
1662 Value *Size = CI->getArgOperand(i: 2);
1663
1664 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1665
1666 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, StrNCmp: false, B, DL))
1667 return Res;
1668
1669 // Handle constant Size.
1670 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1671 if (!LenC)
1672 return nullptr;
1673
1674 return optimizeMemCmpConstantSize(CI, LHS, RHS, Len: LenC->getZExtValue(), B, DL);
1675}
1676
1677Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1678 Module *M = CI->getModule();
1679 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1680 return V;
1681
1682 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1683 // bcmp can be more efficient than memcmp because it only has to know that
1684 // there is a difference, not how different one is to the other.
1685 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_bcmp) &&
1686 isOnlyUsedInZeroEqualityComparison(CtxI: CI)) {
1687 Value *LHS = CI->getArgOperand(i: 0);
1688 Value *RHS = CI->getArgOperand(i: 1);
1689 Value *Size = CI->getArgOperand(i: 2);
1690 return copyFlags(Old: *CI, New: emitBCmp(Ptr1: LHS, Ptr2: RHS, Len: Size, B, DL, TLI));
1691 }
1692
1693 return nullptr;
1694}
1695
1696Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1697 return optimizeMemCmpBCmpCommon(CI, B);
1698}
1699
1700Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1701 Value *Size = CI->getArgOperand(i: 2);
1702 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1703 if (isa<IntrinsicInst>(Val: CI))
1704 return nullptr;
1705
1706 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1707 CallInst *NewCI = B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1708 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1709 mergeAttributesAndFlags(NewCI, Old: *CI);
1710 return CI->getArgOperand(i: 0);
1711}
1712
1713Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1714 Value *Dst = CI->getArgOperand(i: 0);
1715 Value *Src = CI->getArgOperand(i: 1);
1716 ConstantInt *StopChar = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
1717 ConstantInt *N = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
1718 StringRef SrcStr;
1719 if (CI->use_empty() && Dst == Src)
1720 return Dst;
1721 // memccpy(d, s, c, 0) -> nullptr
1722 if (N) {
1723 if (N->isNullValue())
1724 return Constant::getNullValue(Ty: CI->getType());
1725 if (!getConstantStringInfo(V: Src, Str&: SrcStr, /*TrimAtNul=*/false) ||
1726 // TODO: Handle zeroinitializer.
1727 !StopChar)
1728 return nullptr;
1729 } else {
1730 return nullptr;
1731 }
1732
1733 // Wrap arg 'c' of type int to char
1734 size_t Pos = SrcStr.find(C: StopChar->getSExtValue() & 0xFF);
1735 if (Pos == StringRef::npos) {
1736 if (N->getZExtValue() <= SrcStr.size()) {
1737 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
1738 Size: CI->getArgOperand(i: 3)));
1739 return Constant::getNullValue(Ty: CI->getType());
1740 }
1741 return nullptr;
1742 }
1743
1744 Value *NewN =
1745 ConstantInt::get(Ty: N->getType(), V: std::min(a: uint64_t(Pos + 1), b: N->getZExtValue()));
1746 // memccpy -> llvm.memcpy
1747 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: NewN));
1748 return Pos + 1 <= N->getZExtValue()
1749 ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: NewN)
1750 : Constant::getNullValue(Ty: CI->getType());
1751}
1752
1753Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1754 Value *Dst = CI->getArgOperand(i: 0);
1755 Value *N = CI->getArgOperand(i: 2);
1756 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1757 CallInst *NewCI =
1758 B.CreateMemCpy(Dst, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size: N);
1759 // Propagate attributes, but memcpy has no return value, so make sure that
1760 // any return attributes are compliant.
1761 // TODO: Attach return value attributes to the 1st operand to preserve them?
1762 mergeAttributesAndFlags(NewCI, Old: *CI);
1763 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: N);
1764}
1765
1766Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1767 Value *Size = CI->getArgOperand(i: 2);
1768 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1769 if (isa<IntrinsicInst>(Val: CI))
1770 return nullptr;
1771
1772 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1773 CallInst *NewCI = B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1774 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1775 mergeAttributesAndFlags(NewCI, Old: *CI);
1776 return CI->getArgOperand(i: 0);
1777}
1778
1779Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1780 Value *Size = CI->getArgOperand(i: 2);
1781 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1782 if (isa<IntrinsicInst>(Val: CI))
1783 return nullptr;
1784
1785 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1786 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
1787 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val, Size, Align: Align(1));
1788 mergeAttributesAndFlags(NewCI, Old: *CI);
1789 return CI->getArgOperand(i: 0);
1790}
1791
1792Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1793 if (isa<ConstantPointerNull>(Val: CI->getArgOperand(i: 0))) {
1794 Value *Malloc = emitMalloc(Num: CI->getArgOperand(i: 1), B, DL, TLI);
1795 if (auto *MallocCI = dyn_cast_or_null<CallInst>(Val: Malloc))
1796 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_alloc_token))
1797 MallocCI->setMetadata(KindID: LLVMContext::MD_alloc_token, Node: MD);
1798 return copyFlags(Old: *CI, New: Malloc);
1799 }
1800
1801 return nullptr;
1802}
1803
1804// Optionally allow optimization of nobuiltin calls to operator new and its
1805// variants.
1806Value *LibCallSimplifier::maybeOptimizeNoBuiltinOperatorNew(CallInst *CI,
1807 IRBuilderBase &B) {
1808 if (!OptimizeHotColdNew)
1809 return nullptr;
1810 Function *Callee = CI->getCalledFunction();
1811 if (!Callee)
1812 return nullptr;
1813 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
1814 if (Func == NotLibFunc)
1815 return nullptr;
1816 switch (Func) {
1817 case LibFunc_Znwm:
1818 case LibFunc_ZnwmRKSt9nothrow_t:
1819 case LibFunc_ZnwmSt11align_val_t:
1820 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1821 case LibFunc_Znam:
1822 case LibFunc_ZnamRKSt9nothrow_t:
1823 case LibFunc_ZnamSt11align_val_t:
1824 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1825 case LibFunc_size_returning_new:
1826 case LibFunc_size_returning_new_aligned:
1827 // By default normal operator new calls (not already passing a hot_cold_t
1828 // parameter) are not mutated if the call is not marked builtin. Optionally
1829 // enable that in cases where it is known to be safe.
1830 if (!OptimizeNoBuiltinHotColdNew)
1831 return nullptr;
1832 break;
1833 case LibFunc_Znwm12__hot_cold_t:
1834 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1835 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1836 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1837 case LibFunc_Znam12__hot_cold_t:
1838 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1839 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1840 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1841 case LibFunc_size_returning_new_hot_cold:
1842 case LibFunc_size_returning_new_aligned_hot_cold:
1843 // If the nobuiltin call already passes a hot_cold_t parameter, allow update
1844 // of that parameter when enabled.
1845 if (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::None)
1846 return nullptr;
1847 break;
1848 default:
1849 return nullptr;
1850 }
1851 return optimizeNew(CI, B, Func);
1852}
1853
1854// When enabled, replace operator new() calls marked with a hot or cold memprof
1855// attribute with an operator new() call that takes a __hot_cold_t parameter.
1856// Currently this is supported by the open source version of tcmalloc, see:
1857// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1858Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1859 LibFunc &Func) {
1860 if (!OptimizeHotColdNew)
1861 return nullptr;
1862
1863 uint8_t HotCold;
1864 bool IsCold = false;
1865 if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold") {
1866 HotCold = ColdNewHintValue;
1867 IsCold = true;
1868 } else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1869 "notcold")
1870 HotCold = NotColdNewHintValue;
1871 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "hot")
1872 HotCold = HotNewHintValue;
1873 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1874 "ambiguous")
1875 HotCold = AmbiguousNewHintValue;
1876 else
1877 return nullptr;
1878
1879 bool ShouldOptimizeExistingHotColdNew =
1880 OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Always ||
1881 (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Cold &&
1882 IsCold);
1883
1884 Value *HotColdVal = B.getInt8(C: HotCold);
1885 auto getHotColdHintForExisting = [&](uint8_t HotCold) -> Value * {
1886 // If not taking the minimum, simply use the compiler hint value.
1887 if (!MinExistingHotColdNewHint)
1888 return HotColdVal;
1889 Value *ExistingHint = CI->getArgOperand(i: CI->arg_size() - 1);
1890 if (ExistingHint->getType() != B.getInt8Ty())
1891 ExistingHint = B.CreateTruncOrBitCast(V: ExistingHint, DestTy: B.getInt8Ty());
1892 // Emit a umin intrinsic to take the minimum of the existing hint and the
1893 // compiler hint. When the existing hint is a compile-time constant, the
1894 // IRBuilder folder will automatically constant-fold this into a constant.
1895 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: ExistingHint, RHS: HotColdVal);
1896 };
1897
1898 // For calls that already pass a hot/cold hint, only update the hint if
1899 // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
1900 // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
1901 // Note that in cases where we decide it is "notcold", it might be slightly
1902 // better to replace the hinted call with a non hinted call, to avoid the
1903 // extra parameter and the if condition check of the hint value in the
1904 // allocator. This can be considered in the future.
1905 Value *NewCall = nullptr;
1906 switch (Func) {
1907 case LibFunc_Znwm12__hot_cold_t:
1908 if (ShouldOptimizeExistingHotColdNew)
1909 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1910 NewFunc: LibFunc_Znwm12__hot_cold_t,
1911 HotCold: getHotColdHintForExisting(HotCold));
1912 break;
1913 case LibFunc_Znwm:
1914 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1915 NewFunc: LibFunc_Znwm12__hot_cold_t, HotCold: HotColdVal);
1916 break;
1917 case LibFunc_Znam12__hot_cold_t:
1918 if (ShouldOptimizeExistingHotColdNew)
1919 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1920 NewFunc: LibFunc_Znam12__hot_cold_t,
1921 HotCold: getHotColdHintForExisting(HotCold));
1922 break;
1923 case LibFunc_Znam:
1924 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1925 NewFunc: LibFunc_Znam12__hot_cold_t, HotCold: HotColdVal);
1926 break;
1927 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1928 if (ShouldOptimizeExistingHotColdNew)
1929 NewCall =
1930 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1931 TLI, NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
1932 HotCold: getHotColdHintForExisting(HotCold));
1933 break;
1934 case LibFunc_ZnwmRKSt9nothrow_t:
1935 NewCall = emitHotColdNewNoThrow(
1936 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1937 NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1938 break;
1939 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1940 if (ShouldOptimizeExistingHotColdNew)
1941 NewCall =
1942 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1943 TLI, NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
1944 HotCold: getHotColdHintForExisting(HotCold));
1945 break;
1946 case LibFunc_ZnamRKSt9nothrow_t:
1947 NewCall = emitHotColdNewNoThrow(
1948 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1949 NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1950 break;
1951 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1952 if (ShouldOptimizeExistingHotColdNew)
1953 NewCall =
1954 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1955 TLI, NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
1956 HotCold: getHotColdHintForExisting(HotCold));
1957 break;
1958 case LibFunc_ZnwmSt11align_val_t:
1959 NewCall = emitHotColdNewAligned(
1960 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1961 NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1962 break;
1963 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1964 if (ShouldOptimizeExistingHotColdNew)
1965 NewCall =
1966 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1967 TLI, NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t,
1968 HotCold: getHotColdHintForExisting(HotCold));
1969 break;
1970 case LibFunc_ZnamSt11align_val_t:
1971 NewCall = emitHotColdNewAligned(
1972 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1973 NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1974 break;
1975 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1976 if (ShouldOptimizeExistingHotColdNew)
1977 NewCall = emitHotColdNewAlignedNoThrow(
1978 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1979 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1980 HotCold: getHotColdHintForExisting(HotCold));
1981 break;
1982 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1983 NewCall = emitHotColdNewAlignedNoThrow(
1984 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1985 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1986 HotCold: HotColdVal);
1987 break;
1988 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1989 if (ShouldOptimizeExistingHotColdNew)
1990 NewCall = emitHotColdNewAlignedNoThrow(
1991 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1992 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1993 HotCold: getHotColdHintForExisting(HotCold));
1994 break;
1995 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1996 NewCall = emitHotColdNewAlignedNoThrow(
1997 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1998 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1999 HotCold: HotColdVal);
2000 break;
2001 case LibFunc_size_returning_new:
2002 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
2003 NewFunc: LibFunc_size_returning_new_hot_cold,
2004 HotCold: HotColdVal, RetTy: CI->getType());
2005 break;
2006 case LibFunc_size_returning_new_hot_cold:
2007 if (ShouldOptimizeExistingHotColdNew)
2008 NewCall = emitHotColdSizeReturningNew(
2009 Num: CI->getArgOperand(i: 0), B, TLI, NewFunc: LibFunc_size_returning_new_hot_cold,
2010 HotCold: getHotColdHintForExisting(HotCold), RetTy: CI->getType());
2011 break;
2012 case LibFunc_size_returning_new_aligned:
2013 NewCall = emitHotColdSizeReturningNewAligned(
2014 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
2015 NewFunc: LibFunc_size_returning_new_aligned_hot_cold, HotCold: HotColdVal, RetTy: CI->getType());
2016 break;
2017 case LibFunc_size_returning_new_aligned_hot_cold:
2018 if (ShouldOptimizeExistingHotColdNew)
2019 NewCall = emitHotColdSizeReturningNewAligned(
2020 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
2021 NewFunc: LibFunc_size_returning_new_aligned_hot_cold,
2022 HotCold: getHotColdHintForExisting(HotCold), RetTy: CI->getType());
2023 break;
2024 default:
2025 return nullptr;
2026 }
2027
2028 if (auto *NewCI = dyn_cast_or_null<CallInst>(Val: NewCall)) {
2029 NewCI->copyMetadata(SrcInst: *CI);
2030 mergeAttributesAndFlags(NewCI, Old: *CI);
2031 NewCI->removeFnAttr(Kind: "memprof");
2032 // Call-site speculatable is only valid if the new callee is speculatable.
2033 NewCI->removeFnAttr(Kind: Attribute::Speculatable);
2034 if (CI->arg_size() == NewCI->arg_size()) {
2035 // The __hot_cold_t hint argument was replaced; drop any old range.
2036 NewCI->removeParamAttr(ArgNo: NewCI->arg_size() - 1, Kind: Attribute::Range);
2037 } else {
2038 // Adding __hot_cold_t changes the signature, invalidating this attribute.
2039 NewCI->removeFnAttr(Kind: "alloc-variant-zeroed");
2040 }
2041 }
2042
2043 return NewCall;
2044}
2045
2046//===----------------------------------------------------------------------===//
2047// Math Library Optimizations
2048//===----------------------------------------------------------------------===//
2049
2050/// Preserve the accuracy requirement of \p Old on the replacement \p New.
2051static void copyFPMath(const CallInst &Old, Value *New) {
2052 if (auto *NewI = dyn_cast<Instruction>(Val: New))
2053 if (MDNode *MD = Old.getMetadata(KindID: LLVMContext::MD_fpmath))
2054 NewI->setMetadata(KindID: LLVMContext::MD_fpmath, Node: MD);
2055}
2056
2057// Replace a libcall \p CI with a call to intrinsic \p IID
2058static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
2059 Intrinsic::ID IID) {
2060 Value *NewCall = B.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0), FMFSource: CI);
2061 NewCall->takeName(V: CI);
2062 copyFPMath(Old: *CI, New: NewCall);
2063 return copyFlags(Old: *CI, New: NewCall);
2064}
2065
2066static Value *replaceBinaryCall(CallInst *CI, IRBuilderBase &B,
2067 Intrinsic::ID IID) {
2068 Value *NewCall = B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2069 RHS: CI->getArgOperand(i: 1), FMFSource: CI);
2070 NewCall->takeName(V: CI);
2071 copyFPMath(Old: *CI, New: NewCall);
2072 return copyFlags(Old: *CI, New: NewCall);
2073}
2074
2075/// Return a variant of Val with float type.
2076/// Currently this works in two cases: If Val is an FPExtension of a float
2077/// value to something bigger, simply return the operand.
2078/// If Val is a ConstantFP but can be converted to a float ConstantFP without
2079/// loss of precision do so.
2080static Value *valueHasFloatPrecision(Value *Val) {
2081 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
2082 Value *Op = Cast->getOperand(i_nocapture: 0);
2083 if (Op->getType()->isFloatTy())
2084 return Op;
2085 }
2086 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
2087 APFloat F = Const->getValueAPF();
2088 bool losesInfo;
2089 (void)F.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
2090 losesInfo: &losesInfo);
2091 if (!losesInfo)
2092 return ConstantFP::get(Context&: Const->getContext(), V: F);
2093 }
2094 return nullptr;
2095}
2096
2097/// Shrink double -> float functions.
2098static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
2099 bool isBinary, const TargetLibraryInfo *TLI,
2100 bool isPrecise = false) {
2101 Function *CalleeFn = CI->getCalledFunction();
2102 if (!CI->getType()->isDoubleTy() || !CalleeFn)
2103 return nullptr;
2104
2105 // If not all the uses of the function are converted to float, then bail out.
2106 // This matters if the precision of the result is more important than the
2107 // precision of the arguments.
2108 if (isPrecise)
2109 for (User *U : CI->users()) {
2110 FPTruncInst *Cast = dyn_cast<FPTruncInst>(Val: U);
2111 if (!Cast || !Cast->getType()->isFloatTy())
2112 return nullptr;
2113 }
2114
2115 // If this is something like 'g((double) float)', convert to 'gf(float)'.
2116 Value *V[2];
2117 V[0] = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2118 V[1] = isBinary ? valueHasFloatPrecision(Val: CI->getArgOperand(i: 1)) : nullptr;
2119 if (!V[0] || (isBinary && !V[1]))
2120 return nullptr;
2121
2122 // If call isn't an intrinsic, check that it isn't within a function with the
2123 // same name as the float version of this call, otherwise the result is an
2124 // infinite loop. For example, from MinGW-w64:
2125 //
2126 // float expf(float val) { return (float) exp((double) val); }
2127 StringRef CalleeName = CalleeFn->getName();
2128 bool IsIntrinsic = CalleeFn->isIntrinsic();
2129 if (!IsIntrinsic) {
2130 StringRef CallerName = CI->getFunction()->getName();
2131 if (CallerName.ends_with(Suffix: 'f') &&
2132 CallerName.size() == (CalleeName.size() + 1) &&
2133 CallerName.starts_with(Prefix: CalleeName))
2134 return nullptr;
2135 }
2136
2137 // Propagate the math semantics from the current function to the new function.
2138 IRBuilderBase::FastMathFlagGuard Guard(B);
2139 B.setFastMathFlags(CI->getFastMathFlags());
2140
2141 // g((double) float) -> (double) gf(float)
2142 Value *R;
2143 if (IsIntrinsic) {
2144 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
2145 R = isBinary ? B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V)
2146 : B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V[0]);
2147 } else {
2148 AttributeList CallsiteAttrs = CI->getAttributes();
2149 R = isBinary
2150 ? emitBinaryFloatFnCall(Op1: V[0], Op2: V[1], TLI, Name: CalleeName, B,
2151 Attrs: CallsiteAttrs)
2152 : emitUnaryFloatFnCall(Op: V[0], TLI, Name: CalleeName, B, Attrs: CallsiteAttrs);
2153 }
2154 return B.CreateFPExt(V: R, DestTy: B.getDoubleTy());
2155}
2156
2157/// Shrink double -> float for unary functions.
2158static Value *optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2159 const TargetLibraryInfo *TLI,
2160 bool isPrecise = false) {
2161 return optimizeDoubleFP(CI, B, isBinary: false, TLI, isPrecise);
2162}
2163
2164/// Shrink double -> float for binary functions.
2165static Value *optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2166 const TargetLibraryInfo *TLI,
2167 bool isPrecise = false) {
2168 return optimizeDoubleFP(CI, B, isBinary: true, TLI, isPrecise);
2169}
2170
2171/// Shrink double -> float for llvm.sincos.
2172static Value *optimizeSinCosDoubleFP(CallInst *CI, IRBuilderBase &B) {
2173 auto *RetTy = dyn_cast<StructType>(Val: CI->getType());
2174 if (!RetTy || RetTy->getNumElements() != 2 ||
2175 !RetTy->getElementType(N: 0)->getScalarType()->isDoubleTy())
2176 return nullptr;
2177
2178 Value *X = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2179 if (!X)
2180 if (auto *Ext = dyn_cast<FPExtInst>(Val: CI->getArgOperand(i: 0)))
2181 if (Ext->getOperand(i_nocapture: 0)->getType()->getScalarType()->isFloatTy())
2182 X = Ext->getOperand(i_nocapture: 0);
2183 if (!X)
2184 return nullptr;
2185
2186 for (User *U : CI->users()) {
2187 auto *EV = dyn_cast<ExtractValueInst>(Val: U);
2188 if (!EV)
2189 return nullptr;
2190 for (User *EVU : EV->users()) {
2191 auto *Cast = dyn_cast<FPTruncInst>(Val: EVU);
2192 if (!Cast || !Cast->getType()->getScalarType()->isFloatTy())
2193 return nullptr;
2194 }
2195 }
2196
2197 IRBuilderBase::FastMathFlagGuard Guard(B);
2198 B.setFastMathFlags(CI->getFastMathFlags());
2199
2200 Value *NewCall = B.CreateIntrinsic(ID: Intrinsic::sincos, OverloadTypes: X->getType(), Args: X);
2201 cast<Instruction>(Val: NewCall)->setMetadata(
2202 KindID: LLVMContext::MD_fpmath, Node: CI->getMetadata(KindID: LLVMContext::MD_fpmath));
2203 Value *Res = PoisonValue::get(T: RetTy);
2204 for (unsigned I = 0; I != 2; ++I) {
2205 Value *Ext = B.CreateFPExt(V: B.CreateExtractValue(Agg: NewCall, Idxs: I),
2206 DestTy: RetTy->getElementType(N: I));
2207 Res = B.CreateInsertValue(Agg: Res, Val: Ext, Idxs: I);
2208 }
2209 return Res;
2210}
2211
2212// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
2213Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
2214 Value *Real, *Imag;
2215
2216 if (CI->arg_size() == 1) {
2217
2218 if (!CI->isFast())
2219 return nullptr;
2220
2221 Value *Op = CI->getArgOperand(i: 0);
2222 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
2223
2224 Real = B.CreateExtractValue(Agg: Op, Idxs: 0, Name: "real");
2225 Imag = B.CreateExtractValue(Agg: Op, Idxs: 1, Name: "imag");
2226
2227 } else {
2228 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
2229
2230 Real = CI->getArgOperand(i: 0);
2231 Imag = CI->getArgOperand(i: 1);
2232
2233 // if real or imaginary part is zero, simplify to abs(cimag(z))
2234 // or abs(creal(z))
2235 Value *AbsOp = nullptr;
2236 if (ConstantFP *ConstReal = dyn_cast<ConstantFP>(Val: Real)) {
2237 if (ConstReal->isZero())
2238 AbsOp = Imag;
2239
2240 } else if (ConstantFP *ConstImag = dyn_cast<ConstantFP>(Val: Imag)) {
2241 if (ConstImag->isZero())
2242 AbsOp = Real;
2243 }
2244
2245 if (AbsOp)
2246 return copyFlags(Old: *CI, New: B.CreateFAbs(V: AbsOp, FMFSource: CI, Name: "cabs"));
2247
2248 if (!CI->isFast())
2249 return nullptr;
2250 }
2251
2252 // Propagate fast-math flags from the existing call to new instructions.
2253 Value *RealReal = B.CreateFMulFMF(L: Real, R: Real, FMFSource: CI);
2254 Value *ImagImag = B.CreateFMulFMF(L: Imag, R: Imag, FMFSource: CI);
2255 return copyFlags(
2256 Old: *CI, New: B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt,
2257 Op: B.CreateFAddFMF(L: RealReal, R: ImagImag, FMFSource: CI), FMFSource: CI,
2258 Name: "cabs"));
2259}
2260
2261// Return a properly extended integer (DstWidth bits wide) if the operation is
2262// an itofp.
2263static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
2264 if (isa<SIToFPInst>(Val: I2F) || isa<UIToFPInst>(Val: I2F)) {
2265 Value *Op = cast<Instruction>(Val: I2F)->getOperand(i: 0);
2266 // Make sure that the exponent fits inside an "int" of size DstWidth,
2267 // thus avoiding any range issues that FP has not.
2268 unsigned BitWidth = Op->getType()->getScalarSizeInBits();
2269 if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(Val: I2F))) {
2270 Type *IntTy = Op->getType()->getWithNewBitWidth(NewBitWidth: DstWidth);
2271 return isa<SIToFPInst>(Val: I2F) ? B.CreateSExt(V: Op, DestTy: IntTy)
2272 : B.CreateZExt(V: Op, DestTy: IntTy);
2273 }
2274 }
2275
2276 return nullptr;
2277}
2278
2279/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
2280/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
2281/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
2282Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
2283 Module *M = Pow->getModule();
2284 Value *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2285 Type *Ty = Pow->getType();
2286 bool Ignored;
2287
2288 // Evaluate special cases related to a nested function as the base.
2289
2290 // pow(exp(x), y) -> exp(x * y)
2291 // pow(exp2(x), y) -> exp2(x * y)
2292 // If exp{,2}() is used only once, it is better to fold two transcendental
2293 // math functions into one. If used again, exp{,2}() would still have to be
2294 // called with the original argument, then keep both original transcendental
2295 // functions. However, this transformation is only safe with fully relaxed
2296 // math semantics, since, besides rounding differences, it changes overflow
2297 // and underflow behavior quite dramatically. For example:
2298 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
2299 // Whereas:
2300 // exp(1000 * 0.001) = exp(1)
2301 // TODO: Loosen the requirement for fully relaxed math semantics.
2302 // TODO: Handle exp10() when more targets have it available.
2303 CallInst *BaseFn = dyn_cast<CallInst>(Val: Base);
2304 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
2305 Function *CalleeFn = BaseFn->getCalledFunction();
2306 LibFunc LibFn =
2307 CalleeFn ? TLI->getLibFunc(funcName: CalleeFn->getName()) : NotLibFunc;
2308 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFn)) {
2309 StringRef ExpName;
2310 Intrinsic::ID ID;
2311 Value *ExpFn;
2312 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2313
2314 switch (LibFn) {
2315 default:
2316 return nullptr;
2317 case LibFunc_expf:
2318 case LibFunc_exp:
2319 case LibFunc_expl:
2320 ExpName = TLI->getName(F: LibFunc_exp);
2321 ID = Intrinsic::exp;
2322 LibFnFloat = LibFunc_expf;
2323 LibFnDouble = LibFunc_exp;
2324 LibFnLongDouble = LibFunc_expl;
2325 break;
2326 case LibFunc_exp2f:
2327 case LibFunc_exp2:
2328 case LibFunc_exp2l:
2329 ExpName = TLI->getName(F: LibFunc_exp2);
2330 ID = Intrinsic::exp2;
2331 LibFnFloat = LibFunc_exp2f;
2332 LibFnDouble = LibFunc_exp2;
2333 LibFnLongDouble = LibFunc_exp2l;
2334 break;
2335 }
2336
2337 // Create new exp{,2}() with the product as its argument.
2338 Value *FMul = B.CreateFMul(L: BaseFn->getArgOperand(i: 0), R: Expo, Name: "mul");
2339 ExpFn = BaseFn->doesNotAccessMemory()
2340 ? B.CreateUnaryIntrinsic(ID, Op: FMul, FMFSource: nullptr, Name: ExpName)
2341 : emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFnDouble, FloatFn: LibFnFloat,
2342 LongDoubleFn: LibFnLongDouble, B,
2343 Attrs: BaseFn->getAttributes());
2344
2345 // Since the new exp{,2}() is different from the original one, dead code
2346 // elimination cannot be trusted to remove it, since it may have side
2347 // effects (e.g., errno). When the only consumer for the original
2348 // exp{,2}() is pow(), then it has to be explicitly erased.
2349 substituteInParent(I: BaseFn, With: ExpFn);
2350 return ExpFn;
2351 }
2352 }
2353
2354 // Evaluate special cases related to a constant base.
2355
2356 const APFloat *BaseF;
2357 if (!match(V: Base, P: m_APFloat(Res&: BaseF)))
2358 return nullptr;
2359
2360 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2361
2362 const bool UseIntrinsic = Pow->doesNotAccessMemory();
2363
2364 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2365 if ((UseIntrinsic || !Ty->isVectorTy()) && BaseF->isExactlyValue(V: 2.0) &&
2366 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo)) &&
2367 (UseIntrinsic ||
2368 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2369
2370 // TODO: Shouldn't really need to depend on getIntToFPVal for intrinsic. Can
2371 // just directly use the original integer type.
2372 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize())) {
2373 Constant *One = ConstantFP::get(Ty, V: 1.0);
2374
2375 if (UseIntrinsic) {
2376 return copyFlags(Old: *Pow, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2377 OverloadTypes: {Ty, ExpoI->getType()},
2378 Args: {One, ExpoI}, FMFSource: Pow, Name: "exp2"));
2379 }
2380
2381 return copyFlags(Old: *Pow, New: emitBinaryFloatFnCall(
2382 Op1: One, Op2: ExpoI, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2383 LongDoubleFn: LibFunc_ldexpl, B, Attrs: NoAttrs));
2384 }
2385 }
2386
2387 // pow(2.0 ** n, x) -> exp2(n * x)
2388 if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f, LongDoubleFn: LibFunc_exp2l)) {
2389 APFloat BaseR = APFloat(1.0);
2390 BaseR.convert(ToSemantics: BaseF->getSemantics(), RM: APFloat::rmTowardZero, losesInfo: &Ignored);
2391 BaseR = BaseR / *BaseF;
2392 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2393 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2394 APSInt NI(64, false);
2395 if ((IsInteger || IsReciprocal) &&
2396 NF->convertToInteger(Result&: NI, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2397 APFloat::opOK &&
2398 NI > 1 && NI.isPowerOf2()) {
2399 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2400 Value *FMul = B.CreateFMul(L: Expo, R: ConstantFP::get(Ty, V: N), Name: "mul");
2401 if (Pow->doesNotAccessMemory())
2402 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2403 FMFSource: nullptr, Name: "exp2"));
2404 else
2405 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2406 FloatFn: LibFunc_exp2f,
2407 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2408 }
2409 }
2410
2411 // pow(10.0, x) -> exp10(x)
2412 if (BaseF->isExactlyValue(V: 10.0) &&
2413 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp10, FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l)) {
2414
2415 if (Pow->doesNotAccessMemory()) {
2416 return B.CreateIntrinsic(ID: Intrinsic::exp10, OverloadTypes: {Ty}, Args: {Expo}, FMFSource: Pow, Name: "exp10", OpBundles: {},
2417 SetFn: [Pow](CallInst *CI) { CI->copyIRFlags(V: Pow); });
2418 }
2419
2420 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: Expo, TLI, DoubleFn: LibFunc_exp10,
2421 FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l,
2422 B, Attrs: NoAttrs));
2423 }
2424
2425 // pow(x, y) -> exp2(log2(x) * y)
2426 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2427 !BaseF->isNegative()) {
2428 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2429 // Luckily optimizePow has already handled the x == 1 case.
2430 assert(!match(Base, m_FPOne()) &&
2431 "pow(1.0, y) should have been simplified earlier!");
2432
2433 Value *Log = nullptr;
2434 if (Ty->isFloatTy())
2435 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToFloat()));
2436 else if (Ty->isDoubleTy())
2437 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToDouble()));
2438
2439 if (Log) {
2440 Value *FMul = B.CreateFMul(L: Log, R: Expo, Name: "mul");
2441 if (Pow->doesNotAccessMemory())
2442 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2443 FMFSource: nullptr, Name: "exp2"));
2444 else if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f,
2445 LongDoubleFn: LibFunc_exp2l))
2446 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2447 FloatFn: LibFunc_exp2f,
2448 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2449 }
2450 }
2451
2452 return nullptr;
2453}
2454
2455static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2456 Module *M, IRBuilderBase &B,
2457 const TargetLibraryInfo *TLI) {
2458 // If errno is never set, then use the intrinsic for sqrt().
2459 if (NoErrno)
2460 return B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: V, FMFSource: nullptr, Name: "sqrt");
2461
2462 // Otherwise, use the libcall for sqrt().
2463 if (hasFloatFn(M, TLI, Ty: V->getType(), DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2464 LongDoubleFn: LibFunc_sqrtl))
2465 // TODO: We also should check that the target can in fact lower the sqrt()
2466 // libcall. We currently have no way to ask this question, so we ask if
2467 // the target has a sqrt() libcall, which is not exactly the same.
2468 return emitUnaryFloatFnCall(Op: V, TLI, DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2469 LongDoubleFn: LibFunc_sqrtl, B, Attrs);
2470
2471 return nullptr;
2472}
2473
2474/// Use square root in place of pow(x, +/-0.5).
2475Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2476 Value *Sqrt, *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2477 Module *Mod = Pow->getModule();
2478 Type *Ty = Pow->getType();
2479
2480 const APFloat *ExpoF;
2481 if (!match(V: Expo, P: m_APFloat(Res&: ExpoF)) ||
2482 (!ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)))
2483 return nullptr;
2484
2485 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2486 // so that requires fast-math-flags (afn or reassoc).
2487 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2488 return nullptr;
2489
2490 // If we have a pow() library call (accesses memory) and we can't guarantee
2491 // that the base is not an infinity, give up:
2492 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2493 // errno), but sqrt(-Inf) is required by various standards to set errno.
2494 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2495 !isKnownNeverInfinity(
2496 V: Base, SQ: SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
2497 return nullptr;
2498
2499 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M: Mod, B,
2500 TLI);
2501 if (!Sqrt)
2502 return nullptr;
2503
2504 // Handle signed zero base by expanding to fabs(sqrt(x)).
2505 if (!Pow->hasNoSignedZeros())
2506 Sqrt = B.CreateFAbs(V: Sqrt, FMFSource: nullptr, Name: "abs");
2507
2508 Sqrt = copyFlags(Old: *Pow, New: Sqrt);
2509
2510 // Handle non finite base by expanding to
2511 // (x == -infinity ? +infinity : sqrt(x)).
2512 if (!Pow->hasNoInfs()) {
2513 Value *PosInf = ConstantFP::getInfinity(Ty),
2514 *NegInf = ConstantFP::getInfinity(Ty, Negative: true);
2515 Value *FCmp = B.CreateFCmpOEQ(LHS: Base, RHS: NegInf, Name: "isinf");
2516 Sqrt = B.CreateSelect(C: FCmp, True: PosInf, False: Sqrt);
2517 // We assume that the case where x == -infinity is unlikely, so we assign
2518 // unlikely branch weights to that arm of the select.
2519 if (!ProfcheckDisableMetadataFixes) {
2520 if (auto *SqrtSI = dyn_cast<SelectInst>(Val: Sqrt))
2521 setBranchWeights(
2522 I&: *SqrtSI,
2523 Weights: {MDBuilder::kUnlikelyBranchWeight, MDBuilder::kLikelyBranchWeight},
2524 /*IsExpected=*/false);
2525 }
2526 }
2527
2528 // If the exponent is negative, then get the reciprocal.
2529 if (ExpoF->isNegative())
2530 Sqrt = B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Sqrt, Name: "reciprocal");
2531
2532 return Sqrt;
2533}
2534
2535static Value *createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M,
2536 IRBuilderBase &B) {
2537 Value *Args[] = {Base, Expo};
2538 Type *Types[] = {Base->getType(), Expo->getType()};
2539 return B.CreateIntrinsic(ID: Intrinsic::powi, OverloadTypes: Types, Args);
2540}
2541
2542Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2543 Value *Base = Pow->getArgOperand(i: 0);
2544 Value *Expo = Pow->getArgOperand(i: 1);
2545 Function *Callee = Pow->getCalledFunction();
2546 StringRef Name = Callee->getName();
2547 Type *Ty = Pow->getType();
2548 Module *M = Pow->getModule();
2549 bool AllowApprox = Pow->hasApproxFunc();
2550 bool Ignored;
2551
2552 // Propagate the math semantics from the call to any created instructions.
2553 IRBuilderBase::FastMathFlagGuard Guard(B);
2554 B.setFastMathFlags(Pow->getFastMathFlags());
2555 // Evaluate special cases related to the base.
2556
2557 // pow(1.0, x) -> 1.0
2558 if (match(V: Base, P: m_FPOne()))
2559 return Base;
2560
2561 if (Value *Exp = replacePowWithExp(Pow, B))
2562 return Exp;
2563
2564 // Evaluate special cases related to the exponent.
2565
2566 // pow(x, -1.0) -> 1.0 / x
2567 if (match(V: Expo, P: m_SpecificFP(V: -1.0)))
2568 return B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Base, Name: "reciprocal");
2569
2570 // pow(x, +/-0.0) -> 1.0
2571 if (match(V: Expo, P: m_AnyZeroFP()))
2572 return ConstantFP::get(Ty, V: 1.0);
2573
2574 // pow(x, 1.0) -> x
2575 if (match(V: Expo, P: m_FPOne()))
2576 return Base;
2577
2578 // pow(x, 2.0) -> x * x
2579 if (match(V: Expo, P: m_SpecificFP(V: 2.0)) && Pow->doesNotAccessMemory())
2580 return B.CreateFMul(L: Base, R: Base, Name: "square");
2581
2582 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2583 return Sqrt;
2584
2585 // If we can approximate pow:
2586 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2587 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2588 const APFloat *ExpoF;
2589 if (AllowApprox && match(V: Expo, P: m_APFloat(Res&: ExpoF)) &&
2590 !ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)) {
2591 APFloat ExpoA(abs(X: *ExpoF));
2592 APFloat ExpoI(*ExpoF);
2593 Value *Sqrt = nullptr;
2594 if (!ExpoA.isInteger()) {
2595 APFloat Expo2 = ExpoA;
2596 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2597 // is no floating point exception and the result is an integer, then
2598 // ExpoA == integer + 0.5
2599 if (Expo2.add(RHS: ExpoA, RM: APFloat::rmNearestTiesToEven) != APFloat::opOK)
2600 return nullptr;
2601
2602 if (!Expo2.isInteger())
2603 return nullptr;
2604
2605 if (ExpoI.roundToIntegral(RM: APFloat::rmTowardNegative) !=
2606 APFloat::opInexact)
2607 return nullptr;
2608 if (!ExpoI.isInteger())
2609 return nullptr;
2610 ExpoF = &ExpoI;
2611
2612 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M,
2613 B, TLI);
2614 if (!Sqrt)
2615 return nullptr;
2616 }
2617
2618 // 0.5 fraction is now optionally handled.
2619 // Do pow -> powi for remaining integer exponent
2620 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2621 if (ExpoF->isInteger() &&
2622 ExpoF->convertToInteger(Result&: IntExpo, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2623 APFloat::opOK) {
2624 Value *PowI = copyFlags(
2625 Old: *Pow,
2626 New: createPowWithIntegerExponent(
2627 Base, Expo: ConstantInt::get(Ty: B.getIntNTy(N: TLI->getIntSize()), V: IntExpo),
2628 M, B));
2629
2630 if (PowI && Sqrt)
2631 return B.CreateFMul(L: PowI, R: Sqrt);
2632
2633 return PowI;
2634 }
2635 }
2636
2637 // powf(x, itofp(y)) -> powi(x, y)
2638 // The powi exponent must be a scalar integer, so a vector y is not usable.
2639 if (AllowApprox && !Expo->getType()->isVectorTy() &&
2640 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo))) {
2641 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize()))
2642 return copyFlags(Old: *Pow, New: createPowWithIntegerExponent(Base, Expo: ExpoI, M, B));
2643 }
2644
2645 // Shrink pow() to powf() if the arguments are single precision,
2646 // unless the result is expected to be double precision.
2647 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_pow) &&
2648 hasFloatVersion(M, FuncName: Name)) {
2649 if (Value *Shrunk = optimizeBinaryDoubleFP(CI: Pow, B, TLI, isPrecise: true))
2650 return Shrunk;
2651 }
2652
2653 return nullptr;
2654}
2655
2656Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2657 Module *M = CI->getModule();
2658 Function *Callee = CI->getCalledFunction();
2659 StringRef Name = Callee->getName();
2660 Value *Ret = nullptr;
2661 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_exp2) &&
2662 hasFloatVersion(M, FuncName: Name))
2663 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2664
2665 // If we have an llvm.exp2 intrinsic, emit the llvm.ldexp intrinsic. If we
2666 // have the libcall, emit the libcall.
2667 //
2668 // TODO: In principle we should be able to just always use the intrinsic for
2669 // any doesNotAccessMemory callsite.
2670
2671 const bool UseIntrinsic = Callee->isIntrinsic();
2672 // Bail out for vectors because the code below only expects scalars.
2673 Type *Ty = CI->getType();
2674 if (!UseIntrinsic && Ty->isVectorTy())
2675 return Ret;
2676
2677 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2678 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2679 Value *Op = CI->getArgOperand(i: 0);
2680 if ((isa<SIToFPInst>(Val: Op) || isa<UIToFPInst>(Val: Op)) &&
2681 (UseIntrinsic ||
2682 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2683 if (Value *Exp = getIntToFPVal(I2F: Op, B, DstWidth: TLI->getIntSize())) {
2684 Constant *One = ConstantFP::get(Ty, V: 1.0);
2685
2686 if (UseIntrinsic) {
2687 return copyFlags(Old: *CI, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2688 OverloadTypes: {Ty, Exp->getType()},
2689 Args: {One, Exp}, FMFSource: CI));
2690 }
2691
2692 IRBuilderBase::FastMathFlagGuard Guard(B);
2693 B.setFastMathFlags(CI->getFastMathFlags());
2694 return copyFlags(Old: *CI, New: emitBinaryFloatFnCall(
2695 Op1: One, Op2: Exp, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2696 LongDoubleFn: LibFunc_ldexpl, B, Attrs: AttributeList()));
2697 }
2698 }
2699
2700 return Ret;
2701}
2702
2703Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B,
2704 Intrinsic::ID IID) {
2705 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2706 // the intrinsics for improved optimization (for example, vectorization).
2707 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2708 // From the C standard draft WG14/N1256:
2709 // "Ideally, fmax would be sensitive to the sign of zero, for example
2710 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2711 // might be impractical."
2712 FastMathFlags FMF = CI->getFastMathFlags();
2713 FMF.setNoSignedZeros();
2714 return copyFlags(Old: *CI, New: B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2715 RHS: CI->getArgOperand(i: 1), FMFSource: FMF));
2716}
2717
2718Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2719 Function *LogFn = Log->getCalledFunction();
2720 StringRef LogNm = LogFn->getName();
2721 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2722 Module *Mod = Log->getModule();
2723 Type *Ty = Log->getType();
2724
2725 if (UnsafeFPShrink && (LogFn->isIntrinsic() || hasFloatVersion(M: Mod, FuncName: LogNm)))
2726 if (Value *Ret = optimizeUnaryDoubleFP(CI: Log, B, TLI, isPrecise: true))
2727 return Ret;
2728
2729 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2730
2731 // This is only applicable to log(), log2(), log10().
2732 LogLb = TLI->getLibFunc(funcName: LogNm);
2733 if (LogLb != NotLibFunc) {
2734 switch (LogLb) {
2735 case LibFunc_logf:
2736 LogID = Intrinsic::log;
2737 ExpLb = LibFunc_expf;
2738 Exp2Lb = LibFunc_exp2f;
2739 Exp10Lb = LibFunc_exp10f;
2740 PowLb = LibFunc_powf;
2741 break;
2742 case LibFunc_log:
2743 LogID = Intrinsic::log;
2744 ExpLb = LibFunc_exp;
2745 Exp2Lb = LibFunc_exp2;
2746 Exp10Lb = LibFunc_exp10;
2747 PowLb = LibFunc_pow;
2748 break;
2749 case LibFunc_logl:
2750 LogID = Intrinsic::log;
2751 ExpLb = LibFunc_expl;
2752 Exp2Lb = LibFunc_exp2l;
2753 Exp10Lb = LibFunc_exp10l;
2754 PowLb = LibFunc_powl;
2755 break;
2756 case LibFunc_log2f:
2757 LogID = Intrinsic::log2;
2758 ExpLb = LibFunc_expf;
2759 Exp2Lb = LibFunc_exp2f;
2760 Exp10Lb = LibFunc_exp10f;
2761 PowLb = LibFunc_powf;
2762 break;
2763 case LibFunc_log2:
2764 LogID = Intrinsic::log2;
2765 ExpLb = LibFunc_exp;
2766 Exp2Lb = LibFunc_exp2;
2767 Exp10Lb = LibFunc_exp10;
2768 PowLb = LibFunc_pow;
2769 break;
2770 case LibFunc_log2l:
2771 LogID = Intrinsic::log2;
2772 ExpLb = LibFunc_expl;
2773 Exp2Lb = LibFunc_exp2l;
2774 Exp10Lb = LibFunc_exp10l;
2775 PowLb = LibFunc_powl;
2776 break;
2777 case LibFunc_log10f:
2778 LogID = Intrinsic::log10;
2779 ExpLb = LibFunc_expf;
2780 Exp2Lb = LibFunc_exp2f;
2781 Exp10Lb = LibFunc_exp10f;
2782 PowLb = LibFunc_powf;
2783 break;
2784 case LibFunc_log10:
2785 LogID = Intrinsic::log10;
2786 ExpLb = LibFunc_exp;
2787 Exp2Lb = LibFunc_exp2;
2788 Exp10Lb = LibFunc_exp10;
2789 PowLb = LibFunc_pow;
2790 break;
2791 case LibFunc_log10l:
2792 LogID = Intrinsic::log10;
2793 ExpLb = LibFunc_expl;
2794 Exp2Lb = LibFunc_exp2l;
2795 Exp10Lb = LibFunc_exp10l;
2796 PowLb = LibFunc_powl;
2797 break;
2798 default:
2799 return nullptr;
2800 }
2801
2802 // Convert libcall to intrinsic if the value is known > 0.
2803 bool IsKnownNoErrno = Log->hasNoNaNs() && Log->hasNoInfs();
2804 if (!IsKnownNoErrno) {
2805 SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
2806 KnownFPClass Known = computeKnownFPClass(
2807 V: Log->getOperand(i_nocapture: 0),
2808 InterestedClasses: KnownFPClass::OrderedLessThanZeroMask | fcSubnormal, SQ);
2809 Function *F = Log->getParent()->getParent();
2810 const fltSemantics &FltSem = Ty->getScalarType()->getFltSemantics();
2811 IsKnownNoErrno =
2812 Known.cannotBeOrderedLessThanZero() &&
2813 Known.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
2814 }
2815 if (IsKnownNoErrno) {
2816 Value *NewLog = B.CreateUnaryIntrinsic(ID: LogID, Op: Log->getArgOperand(i: 0), FMFSource: Log);
2817 if (auto *I = dyn_cast<Instruction>(Val: NewLog)) {
2818 I->copyMetadata(SrcInst: *Log);
2819 return copyFlags(Old: *Log, New: I);
2820 }
2821 return NewLog;
2822 }
2823 } else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2824 LogID == Intrinsic::log10) {
2825 if (Ty->getScalarType()->isFloatTy()) {
2826 ExpLb = LibFunc_expf;
2827 Exp2Lb = LibFunc_exp2f;
2828 Exp10Lb = LibFunc_exp10f;
2829 PowLb = LibFunc_powf;
2830 } else if (Ty->getScalarType()->isDoubleTy()) {
2831 ExpLb = LibFunc_exp;
2832 Exp2Lb = LibFunc_exp2;
2833 Exp10Lb = LibFunc_exp10;
2834 PowLb = LibFunc_pow;
2835 } else
2836 return nullptr;
2837 } else
2838 return nullptr;
2839
2840 // The earlier call must also be 'fast' in order to do these transforms.
2841 CallInst *Arg = dyn_cast<CallInst>(Val: Log->getArgOperand(i: 0));
2842 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2843 return nullptr;
2844
2845 IRBuilderBase::FastMathFlagGuard Guard(B);
2846 B.setFastMathFlags(FastMathFlags::getFast());
2847
2848 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2849 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2850
2851 // log(pow(x,y)) -> y*log(x)
2852 AttributeList NoAttrs;
2853 if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2854 Value *LogX =
2855 Log->doesNotAccessMemory()
2856 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Arg->getOperand(i_nocapture: 0), FMFSource: nullptr, Name: "log")
2857 : emitUnaryFloatFnCall(Op: Arg->getOperand(i_nocapture: 0), TLI, Name: LogNm, B, Attrs: NoAttrs);
2858 Value *Y = Arg->getArgOperand(i: 1);
2859 // Cast exponent to FP if integer.
2860 if (ArgID == Intrinsic::powi)
2861 Y = B.CreateSIToFP(V: Y, DestTy: Ty, Name: "cast");
2862 Value *MulY = B.CreateFMul(L: Y, R: LogX, Name: "mul");
2863 // Since pow() may have side effects, e.g. errno,
2864 // dead code elimination may not be trusted to remove it.
2865 substituteInParent(I: Arg, With: MulY);
2866 return MulY;
2867 }
2868
2869 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2870 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2871 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2 ||
2872 ArgID == Intrinsic::exp10) {
2873 Constant *Eul;
2874 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2875 // FIXME: Add more precise value of e for long double.
2876 Eul = ConstantFP::get(Ty: Log->getType(), V: numbers::e);
2877 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2878 Eul = ConstantFP::get(Ty: Log->getType(), V: 2.0);
2879 else
2880 Eul = ConstantFP::get(Ty: Log->getType(), V: 10.0);
2881 Value *LogE = Log->doesNotAccessMemory()
2882 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Eul, FMFSource: nullptr, Name: "log")
2883 : emitUnaryFloatFnCall(Op: Eul, TLI, Name: LogNm, B, Attrs: NoAttrs);
2884 Value *MulY = B.CreateFMul(L: Arg->getArgOperand(i: 0), R: LogE, Name: "mul");
2885 // Since exp() may have side effects, e.g. errno,
2886 // dead code elimination may not be trusted to remove it.
2887 substituteInParent(I: Arg, With: MulY);
2888 return MulY;
2889 }
2890
2891 return nullptr;
2892}
2893
2894// sqrt(exp(X)) -> exp(X * 0.5)
2895Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2896 if (!CI->hasAllowReassoc())
2897 return nullptr;
2898
2899 Function *SqrtFn = CI->getCalledFunction();
2900 CallInst *Arg = dyn_cast<CallInst>(Val: CI->getArgOperand(i: 0));
2901 if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2902 return nullptr;
2903 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2904 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2905
2906 LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2907
2908 SqrtLb = TLI->getLibFunc(funcName: SqrtFn->getName());
2909 if (SqrtLb != NotLibFunc)
2910 switch (SqrtLb) {
2911 case LibFunc_sqrtf:
2912 ExpLb = LibFunc_expf;
2913 Exp2Lb = LibFunc_exp2f;
2914 Exp10Lb = LibFunc_exp10f;
2915 break;
2916 case LibFunc_sqrt:
2917 ExpLb = LibFunc_exp;
2918 Exp2Lb = LibFunc_exp2;
2919 Exp10Lb = LibFunc_exp10;
2920 break;
2921 case LibFunc_sqrtl:
2922 ExpLb = LibFunc_expl;
2923 Exp2Lb = LibFunc_exp2l;
2924 Exp10Lb = LibFunc_exp10l;
2925 break;
2926 default:
2927 return nullptr;
2928 }
2929 else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2930 if (CI->getType()->getScalarType()->isFloatTy()) {
2931 ExpLb = LibFunc_expf;
2932 Exp2Lb = LibFunc_exp2f;
2933 Exp10Lb = LibFunc_exp10f;
2934 } else if (CI->getType()->getScalarType()->isDoubleTy()) {
2935 ExpLb = LibFunc_exp;
2936 Exp2Lb = LibFunc_exp2;
2937 Exp10Lb = LibFunc_exp10;
2938 } else
2939 return nullptr;
2940 } else
2941 return nullptr;
2942
2943 if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2944 ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2945 return nullptr;
2946
2947 IRBuilderBase::InsertPointGuard Guard(B);
2948 B.SetInsertPoint(Arg);
2949 auto *ExpOperand = Arg->getOperand(i_nocapture: 0);
2950 auto *FMul =
2951 B.CreateFMulFMF(L: ExpOperand, R: ConstantFP::get(Ty: ExpOperand->getType(), V: 0.5),
2952 FMFSource: CI, Name: "merged.sqrt");
2953
2954 Arg->setOperand(i_nocapture: 0, Val_nocapture: FMul);
2955 return Arg;
2956}
2957
2958Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2959 Module *M = CI->getModule();
2960 Function *Callee = CI->getCalledFunction();
2961 Value *Ret = nullptr;
2962 // TODO: Once we have a way (other than checking for the existince of the
2963 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2964 // condition below.
2965 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_sqrtf) &&
2966 (Callee->getName() == "sqrt" ||
2967 Callee->getIntrinsicID() == Intrinsic::sqrt))
2968 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2969
2970 if (Value *Opt = mergeSqrtToExp(CI, B))
2971 return Opt;
2972
2973 if (!CI->isFast())
2974 return Ret;
2975
2976 Instruction *I = dyn_cast<Instruction>(Val: CI->getArgOperand(i: 0));
2977 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2978 return Ret;
2979
2980 // We're looking for a repeated factor in a multiplication tree,
2981 // so we can do this fold: sqrt(x * x) -> fabs(x);
2982 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2983 Value *Op0 = I->getOperand(i: 0);
2984 Value *Op1 = I->getOperand(i: 1);
2985 Value *RepeatOp = nullptr;
2986 Value *OtherOp = nullptr;
2987 if (Op0 == Op1) {
2988 // Simple match: the operands of the multiply are identical.
2989 RepeatOp = Op0;
2990 } else {
2991 // Look for a more complicated pattern: one of the operands is itself
2992 // a multiply, so search for a common factor in that multiply.
2993 // Note: We don't bother looking any deeper than this first level or for
2994 // variations of this pattern because instcombine's visitFMUL and/or the
2995 // reassociation pass should give us this form.
2996 Value *MulOp;
2997 if (match(V: Op0, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2998 cast<Instruction>(Val: Op0)->isFast()) {
2999 // Pattern: sqrt((x * x) * z)
3000 RepeatOp = MulOp;
3001 OtherOp = Op1;
3002 } else if (match(V: Op1, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
3003 cast<Instruction>(Val: Op1)->isFast()) {
3004 // Pattern: sqrt(z * (x * x))
3005 RepeatOp = MulOp;
3006 OtherOp = Op0;
3007 }
3008 }
3009 if (!RepeatOp)
3010 return Ret;
3011
3012 // Fast math flags for any created instructions should match the sqrt
3013 // and multiply.
3014
3015 // If we found a repeated factor, hoist it out of the square root and
3016 // replace it with the fabs of that factor.
3017 Value *FabsCall = B.CreateFAbs(V: RepeatOp, FMFSource: I, Name: "fabs");
3018 if (OtherOp) {
3019 // If we found a non-repeated factor, we still need to get its square
3020 // root. We then multiply that by the value that was simplified out
3021 // of the square root calculation.
3022 Value *SqrtCall =
3023 B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: OtherOp, FMFSource: I, Name: "sqrt");
3024 return copyFlags(Old: *CI, New: B.CreateFMulFMF(L: FabsCall, R: SqrtCall, FMFSource: I));
3025 }
3026 return copyFlags(Old: *CI, New: FabsCall);
3027}
3028
3029Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
3030
3031 // fmod(x,y) sets errno if y == 0 or x == +/-inf. frem does not set errno,
3032 // so the fold is valid only when we can prove fmod wouldn't either.
3033 bool IsNoErrno = CI->hasNoNaNs();
3034 if (!IsNoErrno) {
3035 SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
3036 KnownFPClass Known0 = computeKnownFPClass(V: CI->getOperand(i_nocapture: 0), InterestedClasses: fcInf, SQ);
3037 if (Known0.isKnownNeverInfinity()) {
3038 KnownFPClass Known1 =
3039 computeKnownFPClass(V: CI->getOperand(i_nocapture: 1), InterestedClasses: fcZero | fcSubnormal, SQ);
3040 Function *F = CI->getParent()->getParent();
3041 const fltSemantics &FltSem =
3042 CI->getType()->getScalarType()->getFltSemantics();
3043 IsNoErrno = Known1.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
3044 }
3045 }
3046
3047 if (IsNoErrno)
3048 return B.CreateFRemFMF(L: CI->getOperand(i_nocapture: 0), R: CI->getOperand(i_nocapture: 1), FMFSource: CI);
3049 return nullptr;
3050}
3051
3052Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
3053 IRBuilderBase &B) {
3054 Module *M = CI->getModule();
3055 Function *Callee = CI->getCalledFunction();
3056 Value *Ret = nullptr;
3057 StringRef Name = Callee->getName();
3058 if (UnsafeFPShrink &&
3059 (Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
3060 Name == "asinh") &&
3061 hasFloatVersion(M, FuncName: Name))
3062 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
3063
3064 Value *Op1 = CI->getArgOperand(i: 0);
3065 auto *OpC = dyn_cast<CallInst>(Val: Op1);
3066 if (!OpC)
3067 return Ret;
3068
3069 // Both calls must be 'fast' in order to remove them.
3070 if (!CI->isFast() || !OpC->isFast())
3071 return Ret;
3072
3073 // tan(atan(x)) -> x
3074 // atanh(tanh(x)) -> x
3075 // sinh(asinh(x)) -> x
3076 // asinh(sinh(x)) -> x
3077 // cosh(acosh(x)) -> x
3078 Function *F = OpC->getCalledFunction();
3079 LibFunc Func = F ? TLI->getLibFunc(funcName: F->getName()) : NotLibFunc;
3080 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3081 LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
3082 .Case(S: "tan", Value: LibFunc_atan)
3083 .Case(S: "atanh", Value: LibFunc_tanh)
3084 .Case(S: "sinh", Value: LibFunc_asinh)
3085 .Case(S: "cosh", Value: LibFunc_acosh)
3086 .Case(S: "tanf", Value: LibFunc_atanf)
3087 .Case(S: "atanhf", Value: LibFunc_tanhf)
3088 .Case(S: "sinhf", Value: LibFunc_asinhf)
3089 .Case(S: "coshf", Value: LibFunc_acoshf)
3090 .Case(S: "tanl", Value: LibFunc_atanl)
3091 .Case(S: "atanhl", Value: LibFunc_tanhl)
3092 .Case(S: "sinhl", Value: LibFunc_asinhl)
3093 .Case(S: "coshl", Value: LibFunc_acoshl)
3094 .Case(S: "asinh", Value: LibFunc_sinh)
3095 .Case(S: "asinhf", Value: LibFunc_sinhf)
3096 .Case(S: "asinhl", Value: LibFunc_sinhl)
3097 .Default(Value: NotLibFunc); // Used as error value
3098 if (Func == inverseFunc)
3099 Ret = OpC->getArgOperand(i: 0);
3100 }
3101 return Ret;
3102}
3103
3104static bool isTrigLibCall(CallInst *CI) {
3105 // We can only hope to do anything useful if we can ignore things like errno
3106 // and floating-point exceptions.
3107 // We already checked the prototype.
3108 return CI->doesNotThrow() && CI->doesNotAccessMemory();
3109}
3110
3111static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
3112 bool UseFloat, Value *&Sin, Value *&Cos,
3113 Value *&SinCos, const TargetLibraryInfo *TLI) {
3114 Module *M = OrigCallee->getParent();
3115 Type *ArgTy = Arg->getType();
3116 Type *ResTy;
3117 StringRef Name;
3118
3119 Triple T(OrigCallee->getParent()->getTargetTriple());
3120 if (UseFloat) {
3121 Name = "__sincospif_stret";
3122
3123 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
3124 // x86_64 can't use {float, float} since that would be returned in both
3125 // xmm0 and xmm1, which isn't what a real struct would do.
3126 ResTy = T.getArch() == Triple::x86_64
3127 ? static_cast<Type *>(FixedVectorType::get(ElementType: ArgTy, NumElts: 2))
3128 : static_cast<Type *>(StructType::get(elt1: ArgTy, elts: ArgTy));
3129 } else {
3130 Name = "__sincospi_stret";
3131 ResTy = StructType::get(elt1: ArgTy, elts: ArgTy);
3132 }
3133
3134 if (!isLibFuncEmittable(M, TLI, Name))
3135 return false;
3136 LibFunc TheLibFunc = TLI->getLibFunc(funcName: Name);
3137 FunctionCallee Callee = getOrInsertLibFunc(
3138 M, TLI: *TLI, TheLibFunc, AttributeList: OrigCallee->getAttributes(), RetTy: ResTy, Args: ArgTy);
3139
3140 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
3141 // If the argument is an instruction, it must dominate all uses so put our
3142 // sincos call there.
3143 B.SetInsertPoint(++ArgInst->getIterator());
3144 } else {
3145 // Otherwise (e.g. for a constant) the beginning of the function is as
3146 // good a place as any.
3147 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
3148 B.SetInsertPoint(EntryBB.begin());
3149 }
3150
3151 SinCos = B.CreateCall(Callee, Args: Arg, Name: "sincospi");
3152
3153 if (SinCos->getType()->isStructTy()) {
3154 Sin = B.CreateExtractValue(Agg: SinCos, Idxs: 0, Name: "sinpi");
3155 Cos = B.CreateExtractValue(Agg: SinCos, Idxs: 1, Name: "cospi");
3156 } else {
3157 Sin = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{0}, Name: "sinpi");
3158 Cos = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{1}, Name: "cospi");
3159 }
3160
3161 return true;
3162}
3163
3164/// Flushing a denormal to +0.0 breaks f(-x) = -f(x) for odd f.
3165static bool mayFlushDenormalsToPositiveZero(const CallInst *CI) {
3166 DenormalMode Mode = CI->getFunction()->getDenormalMode(
3167 FPType: CI->getType()->getScalarType()->getFltSemantics());
3168 return Mode.inputsMayBePositiveZero() || Mode.outputsMayBePositiveZero();
3169}
3170
3171static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
3172 IRBuilderBase &B) {
3173 Value *X;
3174 Value *Src = CI->getArgOperand(i: 0);
3175
3176 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
3177 if (match(V: Src, P: m_OneUse(SubPattern: m_FNegOrFSubNegZero(X: m_Value(V&: X)))) &&
3178 (IsEven || !mayFlushDenormalsToPositiveZero(CI))) {
3179 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3180 auto *CallInst = copyFlags(Old: *CI, New: Call);
3181 if (IsEven) {
3182 // Even function: f(-x) = f(x)
3183 return CallInst;
3184 }
3185 // Odd function: f(-x) = -f(x)
3186 return B.CreateFNegFMF(V: CallInst, FMFSource: CI);
3187 }
3188
3189 // Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
3190 if (IsEven && (match(V: Src, P: m_FAbs(Op0: m_Value(V&: X))) ||
3191 match(V: Src, P: m_CopySign(Op0: m_Value(V&: X), Op1: m_Value())))) {
3192 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3193 return copyFlags(Old: *CI, New: Call);
3194 }
3195
3196 return nullptr;
3197}
3198
3199Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
3200 IRBuilderBase &B) {
3201 switch (Func) {
3202 case LibFunc_cos:
3203 case LibFunc_cosf:
3204 case LibFunc_cosl:
3205
3206 case LibFunc_cosh:
3207 case LibFunc_coshf:
3208 case LibFunc_coshl:
3209 return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
3210
3211 case LibFunc_cbrt:
3212 case LibFunc_cbrtf:
3213 case LibFunc_cbrtl:
3214
3215 case LibFunc_sin:
3216 case LibFunc_sinf:
3217 case LibFunc_sinl:
3218
3219 case LibFunc_sinh:
3220 case LibFunc_sinhf:
3221 case LibFunc_sinhl:
3222
3223 case LibFunc_asin:
3224 case LibFunc_asinf:
3225 case LibFunc_asinl:
3226
3227 case LibFunc_asinh:
3228 case LibFunc_asinhf:
3229 case LibFunc_asinhl:
3230
3231 case LibFunc_tan:
3232 case LibFunc_tanf:
3233 case LibFunc_tanl:
3234
3235 case LibFunc_tanh:
3236 case LibFunc_tanhf:
3237 case LibFunc_tanhl:
3238
3239 case LibFunc_atan:
3240 case LibFunc_atanf:
3241 case LibFunc_atanl:
3242
3243 case LibFunc_erf:
3244 case LibFunc_erff:
3245 case LibFunc_erfl:
3246 return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
3247
3248 default:
3249 return nullptr;
3250 }
3251}
3252
3253Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
3254 // Make sure the prototype is as expected, otherwise the rest of the
3255 // function is probably invalid and likely to abort.
3256 if (!isTrigLibCall(CI))
3257 return nullptr;
3258
3259 Value *Arg = CI->getArgOperand(i: 0);
3260 if (isa<ConstantData>(Val: Arg))
3261 return nullptr;
3262
3263 SmallVector<CallInst *, 1> SinCalls;
3264 SmallVector<CallInst *, 1> CosCalls;
3265 SmallVector<CallInst *, 1> SinCosCalls;
3266
3267 bool IsFloat = Arg->getType()->isFloatTy();
3268
3269 // Look for all compatible sinpi, cospi and sincospi calls with the same
3270 // argument. If there are enough (in some sense) we can make the
3271 // substitution.
3272 Function *F = CI->getFunction();
3273 for (User *U : Arg->users())
3274 classifyArgUse(Val: U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
3275
3276 // It's only worthwhile if both sinpi and cospi are actually used.
3277 if (SinCalls.empty() || CosCalls.empty())
3278 return nullptr;
3279
3280 Value *Sin, *Cos, *SinCos;
3281 if (!insertSinCosCall(B, OrigCallee: CI->getCalledFunction(), Arg, UseFloat: IsFloat, Sin, Cos,
3282 SinCos, TLI))
3283 return nullptr;
3284
3285 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
3286 Value *Res) {
3287 for (CallInst *C : Calls)
3288 replaceAllUsesWith(I: C, With: Res);
3289 };
3290
3291 replaceTrigInsts(SinCalls, Sin);
3292 replaceTrigInsts(CosCalls, Cos);
3293 replaceTrigInsts(SinCosCalls, SinCos);
3294
3295 return IsSin ? Sin : Cos;
3296}
3297
3298void LibCallSimplifier::classifyArgUse(
3299 Value *Val, Function *F, bool IsFloat,
3300 SmallVectorImpl<CallInst *> &SinCalls,
3301 SmallVectorImpl<CallInst *> &CosCalls,
3302 SmallVectorImpl<CallInst *> &SinCosCalls) {
3303 auto *CI = dyn_cast<CallInst>(Val);
3304 if (!CI || CI->use_empty())
3305 return;
3306
3307 // Don't consider calls in other functions.
3308 if (CI->getFunction() != F)
3309 return;
3310
3311 Module *M = CI->getModule();
3312 Function *Callee = CI->getCalledFunction();
3313 LibFunc Func = Callee ? TLI->getLibFunc(FDecl: *Callee) : NotLibFunc;
3314 if (!isLibFuncEmittable(M, TLI, TheLibFunc: Func) || !isTrigLibCall(CI))
3315 return;
3316
3317 if (IsFloat) {
3318 if (Func == LibFunc_sinpif)
3319 SinCalls.push_back(Elt: CI);
3320 else if (Func == LibFunc_cospif)
3321 CosCalls.push_back(Elt: CI);
3322 else if (Func == LibFunc_sincospif_stret)
3323 SinCosCalls.push_back(Elt: CI);
3324 } else {
3325 if (Func == LibFunc_sinpi)
3326 SinCalls.push_back(Elt: CI);
3327 else if (Func == LibFunc_cospi)
3328 CosCalls.push_back(Elt: CI);
3329 else if (Func == LibFunc_sincospi_stret)
3330 SinCosCalls.push_back(Elt: CI);
3331 }
3332}
3333
3334/// Constant folds remquo
3335Value *LibCallSimplifier::optimizeRemquo(CallInst *CI, IRBuilderBase &B) {
3336 const APFloat *X, *Y;
3337 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3338 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3339 return nullptr;
3340
3341 APFloat::opStatus Status;
3342 APFloat Quot = *X;
3343 Status = Quot.divide(RHS: *Y, RM: APFloat::rmNearestTiesToEven);
3344 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3345 return nullptr;
3346 APFloat Rem = *X;
3347 if (Rem.remainder(RHS: *Y) != APFloat::opOK)
3348 return nullptr;
3349
3350 // TODO: We can only keep at least the three of the last bits of x/y
3351 unsigned IntBW = TLI->getIntSize();
3352 APSInt QuotInt(IntBW, /*isUnsigned=*/false);
3353 bool IsExact;
3354 Status =
3355 Quot.convertToInteger(Result&: QuotInt, RM: APFloat::rmNearestTiesToEven, IsExact: &IsExact);
3356 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3357 return nullptr;
3358
3359 B.CreateAlignedStore(
3360 Val: ConstantInt::getSigned(Ty: B.getIntNTy(N: IntBW), V: QuotInt.getExtValue()),
3361 Ptr: CI->getArgOperand(i: 2), Align: CI->getParamAlign(ArgNo: 2));
3362 return ConstantFP::get(Ty: CI->getType(), V: Rem);
3363}
3364
3365/// Constant folds fdim
3366Value *LibCallSimplifier::optimizeFdim(CallInst *CI, IRBuilderBase &B) {
3367 // Cannot perform the fold unless the call has attribute memory(none)
3368 if (!CI->doesNotAccessMemory())
3369 return nullptr;
3370
3371 // TODO : Handle undef values
3372 // Propagate poison if any
3373 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 0)))
3374 return CI->getArgOperand(i: 0);
3375 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 1)))
3376 return CI->getArgOperand(i: 1);
3377
3378 const APFloat *X, *Y;
3379 // Check if both values are constants
3380 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3381 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3382 return nullptr;
3383
3384 // C99 fdim(x, y) = (x > y) ? x - y : +0.
3385 if (X->compare(RHS: *Y) != APFloat::cmpGreaterThan && !X->isNaN() && !Y->isNaN())
3386 return ConstantFP::getZero(Ty: CI->getType());
3387 APFloat Difference = *X;
3388 Difference.subtract(RHS: *Y, RM: RoundingMode::NearestTiesToEven);
3389 return ConstantFP::get(Ty: CI->getType(), V: Difference);
3390}
3391
3392//===----------------------------------------------------------------------===//
3393// Integer Library Call Optimizations
3394//===----------------------------------------------------------------------===//
3395
3396Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
3397 // All variants of ffs return int which need not be 32 bits wide.
3398 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
3399 Type *RetType = CI->getType();
3400 Value *Op = CI->getArgOperand(i: 0);
3401 Type *ArgType = Op->getType();
3402 Value *V = B.CreateIntrinsic(ID: Intrinsic::cttz, OverloadTypes: {ArgType}, Args: {Op, B.getTrue()},
3403 FMFSource: nullptr, Name: "cttz");
3404 V = B.CreateAdd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 1));
3405 V = B.CreateIntCast(V, DestTy: RetType, isSigned: false);
3406
3407 Value *Cond = B.CreateICmpNE(LHS: Op, RHS: Constant::getNullValue(Ty: ArgType));
3408 Value *S = B.CreateSelect(C: Cond, True: V, False: ConstantInt::get(Ty: RetType, V: 0));
3409 if (ProfcheckDisableMetadataFixes)
3410 return S;
3411 if (auto *SI = dyn_cast<SelectInst>(Val: S))
3412 setBranchWeights(
3413 I&: *SI, Weights: {MDBuilder::kLikelyBranchWeight, MDBuilder::kUnlikelyBranchWeight},
3414 /*IsExpected=*/false);
3415 return S;
3416}
3417
3418Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
3419 // All variants of fls return int which need not be 32 bits wide.
3420 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
3421 Value *Op = CI->getArgOperand(i: 0);
3422 Type *ArgType = Op->getType();
3423 Value *V = B.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {ArgType}, Args: {Op, B.getFalse()},
3424 FMFSource: nullptr, Name: "ctlz");
3425 V = B.CreateSub(LHS: ConstantInt::get(Ty: V->getType(), V: ArgType->getIntegerBitWidth()),
3426 RHS: V);
3427 return B.CreateIntCast(V, DestTy: CI->getType(), isSigned: false);
3428}
3429
3430Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
3431 // abs(x) -> x <s 0 ? -x : x
3432 // The negation has 'nsw' because abs of INT_MIN is undefined.
3433 Value *X = CI->getArgOperand(i: 0);
3434 Value *IsNeg = B.CreateIsNeg(Arg: X);
3435 Value *NegX = B.CreateNSWNeg(V: X, Name: "neg");
3436 return B.CreateSelect(C: IsNeg, True: NegX, False: X);
3437}
3438
3439Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
3440 // isdigit(c) -> (c-'0') <u 10
3441 Value *Op = CI->getArgOperand(i: 0);
3442 Type *ArgType = Op->getType();
3443 Op = B.CreateSub(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: '0'), Name: "isdigittmp");
3444 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 10), Name: "isdigit");
3445 return B.CreateZExt(V: Op, DestTy: CI->getType());
3446}
3447
3448Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
3449 // isascii(c) -> c <u 128
3450 Value *Op = CI->getArgOperand(i: 0);
3451 Type *ArgType = Op->getType();
3452 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 128), Name: "isascii");
3453 return B.CreateZExt(V: Op, DestTy: CI->getType());
3454}
3455
3456Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
3457 // toascii(c) -> c & 0x7f
3458 return B.CreateAnd(LHS: CI->getArgOperand(i: 0),
3459 RHS: ConstantInt::get(Ty: CI->getType(), V: 0x7F));
3460}
3461
3462// Fold calls to atoi, atol, and atoll.
3463Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
3464 StringRef Str;
3465 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3466 return nullptr;
3467
3468 return convertStrToInt(CI, Str, EndPtr: nullptr, Base: 10, /*AsSigned=*/true, B);
3469}
3470
3471// Fold calls to strtol, strtoll, strtoul, and strtoull.
3472Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
3473 bool AsSigned) {
3474 Value *EndPtr = CI->getArgOperand(i: 1);
3475 if (isa<ConstantPointerNull>(Val: EndPtr)) {
3476 // With a null EndPtr, this function won't capture the main argument.
3477 // It would be readonly too, except that it still may write to errno.
3478 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
3479 CI: CaptureInfo::none()));
3480 EndPtr = nullptr;
3481 } else if (!isKnownNonZero(V: EndPtr, Q: DL))
3482 return nullptr;
3483
3484 StringRef Str;
3485 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3486 return nullptr;
3487
3488 if (ConstantInt *CInt = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2))) {
3489 return convertStrToInt(CI, Str, EndPtr, Base: CInt->getSExtValue(), AsSigned, B);
3490 }
3491
3492 return nullptr;
3493}
3494
3495//===----------------------------------------------------------------------===//
3496// Formatting and IO Library Call Optimizations
3497//===----------------------------------------------------------------------===//
3498
3499static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
3500
3501Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3502 int StreamArg) {
3503 Function *Callee = CI->getCalledFunction();
3504 // Error reporting calls should be cold, mark them as such.
3505 // This applies even to non-builtin calls: it is only a hint and applies to
3506 // functions that the frontend might not understand as builtins.
3507
3508 // This heuristic was suggested in:
3509 // Improving Static Branch Prediction in a Compiler
3510 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3511 // Proceedings of PACT'98, Oct. 1998, IEEE
3512 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
3513 isReportingError(Callee, CI, StreamArg)) {
3514 CI->addFnAttr(Kind: Attribute::Cold);
3515 }
3516
3517 return nullptr;
3518}
3519
3520static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3521 if (!Callee || !Callee->isDeclaration())
3522 return false;
3523
3524 if (StreamArg < 0)
3525 return true;
3526
3527 // These functions might be considered cold, but only if their stream
3528 // argument is stderr.
3529
3530 if (StreamArg >= (int)CI->arg_size())
3531 return false;
3532 LoadInst *LI = dyn_cast<LoadInst>(Val: CI->getArgOperand(i: StreamArg));
3533 if (!LI)
3534 return false;
3535 GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: LI->getPointerOperand());
3536 if (!GV || !GV->isDeclaration())
3537 return false;
3538 return GV->getName() == "stderr";
3539}
3540
3541Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3542 // Check for a fixed format string.
3543 StringRef FormatStr;
3544 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: FormatStr))
3545 return nullptr;
3546
3547 // Empty format string -> noop.
3548 if (FormatStr.empty()) // Tolerate printf's declared void.
3549 return CI->use_empty() ? (Value *)CI : ConstantInt::get(Ty: CI->getType(), V: 0);
3550
3551 // Do not do any of the following transformations if the printf return value
3552 // is used, in general the printf return value is not compatible with either
3553 // putchar() or puts().
3554 if (!CI->use_empty())
3555 return nullptr;
3556
3557 Type *IntTy = CI->getType();
3558 // printf("x") -> putchar('x'), even for "%" and "%%".
3559 if (FormatStr.size() == 1 || FormatStr == "%%") {
3560 // Convert the character to unsigned char before passing it to putchar
3561 // to avoid host-specific sign extension in the IR. Putchar converts
3562 // it to unsigned char regardless.
3563 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)FormatStr[0]);
3564 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3565 }
3566
3567 // Try to remove call or emit putchar/puts.
3568 if (FormatStr == "%s" && CI->arg_size() > 1) {
3569 StringRef OperandStr;
3570 if (!getConstantStringInfo(V: CI->getOperand(i_nocapture: 1), Str&: OperandStr))
3571 return nullptr;
3572 // printf("%s", "") --> NOP
3573 if (OperandStr.empty())
3574 return (Value *)CI;
3575 // printf("%s", "a") --> putchar('a')
3576 if (OperandStr.size() == 1) {
3577 // Convert the character to unsigned char before passing it to putchar
3578 // to avoid host-specific sign extension in the IR. Putchar converts
3579 // it to unsigned char regardless.
3580 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)OperandStr[0]);
3581 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3582 }
3583 // printf("%s", str"\n") --> puts(str)
3584 if (OperandStr.back() == '\n') {
3585 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3586 return nullptr;
3587 OperandStr = OperandStr.drop_back();
3588 Value *GV = B.CreateGlobalString(Str: OperandStr, Name: "str");
3589 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3590 }
3591 return nullptr;
3592 }
3593
3594 // printf("foo\n") --> puts("foo")
3595 if (FormatStr.back() == '\n' &&
3596 !FormatStr.contains(C: '%')) { // No format characters.
3597 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3598 return nullptr;
3599 // Create a string literal with no \n on it. We expect the constant merge
3600 // pass to be run after this pass, to merge duplicate strings.
3601 FormatStr = FormatStr.drop_back();
3602 Value *GV = B.CreateGlobalString(Str: FormatStr, Name: "str");
3603 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3604 }
3605
3606 // Optimize specific format strings.
3607 // printf("%c", chr) --> putchar(chr)
3608 if (FormatStr == "%c" && CI->arg_size() > 1 &&
3609 CI->getArgOperand(i: 1)->getType()->isIntegerTy()) {
3610 // Convert the argument to the type expected by putchar, i.e., int, which
3611 // need not be 32 bits wide but which is the same as printf's return type.
3612 Value *IntChar = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: IntTy, isSigned: false);
3613 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3614 }
3615
3616 // printf("%s\n", str) --> puts(str)
3617 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3618 CI->getArgOperand(i: 1)->getType()->isPointerTy())
3619 return copyFlags(Old: *CI, New: emitPutS(Str: CI->getArgOperand(i: 1), B, TLI));
3620 return nullptr;
3621}
3622
3623Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3624
3625 Module *M = CI->getModule();
3626 Function *Callee = CI->getCalledFunction();
3627 FunctionType *FT = Callee->getFunctionType();
3628 if (Value *V = optimizePrintFString(CI, B)) {
3629 return V;
3630 }
3631
3632 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3633
3634 // printf(format, ...) -> iprintf(format, ...) if no floating point
3635 // arguments.
3636 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_iprintf) &&
3637 !callHasFloatingPointArgument(CI)) {
3638 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_iprintf, T: FT,
3639 AttributeList: Callee->getAttributes());
3640 CallInst *New = cast<CallInst>(Val: CI->clone());
3641 New->setCalledFunction(IPrintFFn);
3642 B.Insert(I: New);
3643 return New;
3644 }
3645
3646 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3647 // arguments.
3648 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_printf) &&
3649 !callHasFP128Argument(CI)) {
3650 auto SmallPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_printf, T: FT,
3651 AttributeList: Callee->getAttributes());
3652 CallInst *New = cast<CallInst>(Val: CI->clone());
3653 New->setCalledFunction(SmallPrintFFn);
3654 B.Insert(I: New);
3655 return New;
3656 }
3657
3658 return nullptr;
3659}
3660
3661Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3662 IRBuilderBase &B) {
3663 // Check for a fixed format string.
3664 StringRef FormatStr;
3665 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3666 return nullptr;
3667
3668 // If we just have a format string (nothing else crazy) transform it.
3669 Value *Dest = CI->getArgOperand(i: 0);
3670 if (CI->arg_size() == 2) {
3671 // Make sure there's no % in the constant array. We could try to handle
3672 // %% -> % in the future if we cared.
3673 if (FormatStr.contains(C: '%'))
3674 return nullptr; // we found a format specifier, bail out.
3675
3676 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3677 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1),
3678 // Copy the null byte.
3679 Size: TLI->getAsSizeT(V: FormatStr.size() + 1, M: *CI->getModule()));
3680 return ConstantInt::get(Ty: CI->getType(), V: FormatStr.size());
3681 }
3682
3683 // The remaining optimizations require the format string to be "%s" or "%c"
3684 // and have an extra operand.
3685 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3686 return nullptr;
3687
3688 // Decode the second character of the format string.
3689 if (FormatStr[1] == 'c') {
3690 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3691 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3692 return nullptr;
3693 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 2), DestTy: B.getInt8Ty(), Name: "char");
3694 Value *Ptr = Dest;
3695 B.CreateStore(Val: V, Ptr);
3696 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3697 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3698
3699 return ConstantInt::get(Ty: CI->getType(), V: 1);
3700 }
3701
3702 if (FormatStr[1] == 's') {
3703 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3704 // strlen(str)+1)
3705 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3706 return nullptr;
3707
3708 if (CI->use_empty())
3709 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3710 return copyFlags(Old: *CI, New: emitStrCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI));
3711
3712 uint64_t SrcLen = GetStringLength(V: CI->getArgOperand(i: 2));
3713 if (SrcLen) {
3714 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1),
3715 Size: TLI->getAsSizeT(V: SrcLen, M: *CI->getModule()));
3716 // Returns total number of characters written without null-character.
3717 return ConstantInt::get(Ty: CI->getType(), V: SrcLen - 1);
3718 } else if (Value *V = emitStpCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI)) {
3719 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3720 Value *PtrDiff = B.CreatePtrDiff(LHS: V, RHS: Dest);
3721 return B.CreateIntCast(V: PtrDiff, DestTy: CI->getType(), isSigned: false);
3722 }
3723
3724 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3725 QueryType: PGSOQueryType::IRPass))
3726 return nullptr;
3727
3728 Value *Len = emitStrLen(Ptr: CI->getArgOperand(i: 2), B, DL, TLI);
3729 if (!Len)
3730 return nullptr;
3731 Value *IncLen =
3732 B.CreateAdd(LHS: Len, RHS: ConstantInt::get(Ty: Len->getType(), V: 1), Name: "leninc");
3733 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1), Size: IncLen);
3734
3735 // The sprintf result is the unincremented number of bytes in the string.
3736 return B.CreateIntCast(V: Len, DestTy: CI->getType(), isSigned: false);
3737 }
3738 return nullptr;
3739}
3740
3741Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3742 Module *M = CI->getModule();
3743 Function *Callee = CI->getCalledFunction();
3744 FunctionType *FT = Callee->getFunctionType();
3745 if (Value *V = optimizeSPrintFString(CI, B)) {
3746 return V;
3747 }
3748
3749 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
3750
3751 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3752 // point arguments.
3753 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_siprintf) &&
3754 !callHasFloatingPointArgument(CI)) {
3755 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_siprintf,
3756 T: FT, AttributeList: Callee->getAttributes());
3757 CallInst *New = cast<CallInst>(Val: CI->clone());
3758 New->setCalledFunction(SIPrintFFn);
3759 B.Insert(I: New);
3760 return New;
3761 }
3762
3763 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3764 // floating point arguments.
3765 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_sprintf) &&
3766 !callHasFP128Argument(CI)) {
3767 auto SmallSPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_sprintf, T: FT,
3768 AttributeList: Callee->getAttributes());
3769 CallInst *New = cast<CallInst>(Val: CI->clone());
3770 New->setCalledFunction(SmallSPrintFFn);
3771 B.Insert(I: New);
3772 return New;
3773 }
3774
3775 return nullptr;
3776}
3777
3778// Transform an snprintf call CI with the bound N to format the string Str
3779// either to a call to memcpy, or to single character a store, or to nothing,
3780// and fold the result to a constant. A nonnull StrArg refers to the string
3781// argument being formatted. Otherwise the call is one with N < 2 and
3782// the "%c" directive to format a single character.
3783Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3784 StringRef Str, uint64_t N,
3785 IRBuilderBase &B) {
3786 assert(StrArg || (N < 2 && Str.size() == 1));
3787
3788 unsigned IntBits = TLI->getIntSize();
3789 uint64_t IntMax = maxIntN(N: IntBits);
3790 if (Str.size() > IntMax)
3791 // Bail if the string is longer than INT_MAX. POSIX requires
3792 // implementations to set errno to EOVERFLOW in this case, in
3793 // addition to when N is larger than that (checked by the caller).
3794 return nullptr;
3795
3796 Value *StrLen = ConstantInt::get(Ty: CI->getType(), V: Str.size());
3797 if (N == 0)
3798 return StrLen;
3799
3800 // Set to the number of bytes to copy fron StrArg which is also
3801 // the offset of the terinating nul.
3802 uint64_t NCopy;
3803 if (N > Str.size())
3804 // Copy the full string, including the terminating nul (which must
3805 // be present regardless of the bound).
3806 NCopy = Str.size() + 1;
3807 else
3808 NCopy = N - 1;
3809
3810 Value *DstArg = CI->getArgOperand(i: 0);
3811 if (NCopy && StrArg)
3812 // Transform the call to lvm.memcpy(dst, fmt, N).
3813 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst: DstArg, DstAlign: Align(1), Src: StrArg, SrcAlign: Align(1),
3814 Size: TLI->getAsSizeT(V: NCopy, M: *CI->getModule())));
3815
3816 if (N > Str.size())
3817 // Return early when the whole format string, including the final nul,
3818 // has been copied.
3819 return StrLen;
3820
3821 // Otherwise, when truncating the string append a terminating nul.
3822 Type *Int8Ty = B.getInt8Ty();
3823 Value *NulOff = B.getIntN(N: IntBits, C: NCopy);
3824 Value *DstEnd = B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: DstArg, IdxList: NulOff, Name: "endptr");
3825 B.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: 0), Ptr: DstEnd);
3826 return StrLen;
3827}
3828
3829Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3830 IRBuilderBase &B) {
3831 // Check for size
3832 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3833 if (!Size)
3834 return nullptr;
3835
3836 uint64_t N = Size->getZExtValue();
3837 uint64_t IntMax = maxIntN(N: TLI->getIntSize());
3838 if (N > IntMax)
3839 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3840 // to set errno to EOVERFLOW in this case.
3841 return nullptr;
3842
3843 Value *DstArg = CI->getArgOperand(i: 0);
3844 Value *FmtArg = CI->getArgOperand(i: 2);
3845
3846 // Check for a fixed format string.
3847 StringRef FormatStr;
3848 if (!getConstantStringInfo(V: FmtArg, Str&: FormatStr))
3849 return nullptr;
3850
3851 // If we just have a format string (nothing else crazy) transform it.
3852 if (CI->arg_size() == 3) {
3853 if (FormatStr.contains(C: '%'))
3854 // Bail if the format string contains a directive and there are
3855 // no arguments. We could handle "%%" in the future.
3856 return nullptr;
3857
3858 return emitSnPrintfMemCpy(CI, StrArg: FmtArg, Str: FormatStr, N, B);
3859 }
3860
3861 // The remaining optimizations require the format string to be "%s" or "%c"
3862 // and have an extra operand.
3863 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3864 return nullptr;
3865
3866 // Decode the second character of the format string.
3867 if (FormatStr[1] == 'c') {
3868 if (N <= 1) {
3869 // Use an arbitary string of length 1 to transform the call into
3870 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3871 // to one.
3872 StringRef CharStr("*");
3873 return emitSnPrintfMemCpy(CI, StrArg: nullptr, Str: CharStr, N, B);
3874 }
3875
3876 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3877 if (!CI->getArgOperand(i: 3)->getType()->isIntegerTy())
3878 return nullptr;
3879 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 3), DestTy: B.getInt8Ty(), Name: "char");
3880 Value *Ptr = DstArg;
3881 B.CreateStore(Val: V, Ptr);
3882 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3883 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3884 return ConstantInt::get(Ty: CI->getType(), V: 1);
3885 }
3886
3887 if (FormatStr[1] != 's')
3888 return nullptr;
3889
3890 Value *StrArg = CI->getArgOperand(i: 3);
3891 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3892 StringRef Str;
3893 if (!getConstantStringInfo(V: StrArg, Str))
3894 return nullptr;
3895
3896 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3897}
3898
3899Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3900 if (Value *V = optimizeSnPrintFString(CI, B)) {
3901 return V;
3902 }
3903
3904 if (isKnownNonZero(V: CI->getOperand(i_nocapture: 1), Q: DL))
3905 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3906 return nullptr;
3907}
3908
3909Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3910 IRBuilderBase &B) {
3911 optimizeErrorReporting(CI, B, StreamArg: 0);
3912
3913 // All the optimizations depend on the format string.
3914 StringRef FormatStr;
3915 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3916 return nullptr;
3917
3918 // Do not do any of the following transformations if the fprintf return
3919 // value is used, in general the fprintf return value is not compatible
3920 // with fwrite(), fputc() or fputs().
3921 if (!CI->use_empty())
3922 return nullptr;
3923
3924 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3925 if (CI->arg_size() == 2) {
3926 // Could handle %% -> % if we cared.
3927 if (FormatStr.contains(C: '%'))
3928 return nullptr; // We found a format specifier.
3929
3930 return copyFlags(
3931 Old: *CI, New: emitFWrite(Ptr: CI->getArgOperand(i: 1),
3932 Size: TLI->getAsSizeT(V: FormatStr.size(), M: *CI->getModule()),
3933 File: CI->getArgOperand(i: 0), B, DL, TLI));
3934 }
3935
3936 // The remaining optimizations require the format string to be "%s" or "%c"
3937 // and have an extra operand.
3938 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3939 return nullptr;
3940
3941 // Decode the second character of the format string.
3942 if (FormatStr[1] == 'c') {
3943 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3944 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3945 return nullptr;
3946 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3947 Value *V = B.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntTy, /*isSigned*/ true,
3948 Name: "chari");
3949 return copyFlags(Old: *CI, New: emitFPutC(Char: V, File: CI->getArgOperand(i: 0), B, TLI));
3950 }
3951
3952 if (FormatStr[1] == 's') {
3953 // fprintf(F, "%s", str) --> fputs(str, F)
3954 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3955 return nullptr;
3956 return copyFlags(
3957 Old: *CI, New: emitFPutS(Str: CI->getArgOperand(i: 2), File: CI->getArgOperand(i: 0), B, TLI));
3958 }
3959 return nullptr;
3960}
3961
3962Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3963 Module *M = CI->getModule();
3964 Function *Callee = CI->getCalledFunction();
3965 FunctionType *FT = Callee->getFunctionType();
3966 if (Value *V = optimizeFPrintFString(CI, B)) {
3967 return V;
3968 }
3969
3970 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3971 // floating point arguments.
3972 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_fiprintf) &&
3973 !callHasFloatingPointArgument(CI)) {
3974 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_fiprintf,
3975 T: FT, AttributeList: Callee->getAttributes());
3976 CallInst *New = cast<CallInst>(Val: CI->clone());
3977 New->setCalledFunction(FIPrintFFn);
3978 B.Insert(I: New);
3979 return New;
3980 }
3981
3982 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3983 // 128-bit floating point arguments.
3984 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_fprintf) &&
3985 !callHasFP128Argument(CI)) {
3986 auto SmallFPrintFFn =
3987 getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_fprintf, T: FT,
3988 AttributeList: Callee->getAttributes());
3989 CallInst *New = cast<CallInst>(Val: CI->clone());
3990 New->setCalledFunction(SmallFPrintFFn);
3991 B.Insert(I: New);
3992 return New;
3993 }
3994
3995 return nullptr;
3996}
3997
3998Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3999 optimizeErrorReporting(CI, B, StreamArg: 3);
4000
4001 // Get the element size and count.
4002 ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
4003 ConstantInt *CountC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
4004 if (SizeC && CountC) {
4005 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
4006
4007 // If this is writing zero records, remove the call (it's a noop).
4008 if (Bytes == 0)
4009 return ConstantInt::get(Ty: CI->getType(), V: 0);
4010
4011 // If this is writing one byte, turn it into fputc.
4012 // This optimisation is only valid, if the return value is unused.
4013 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
4014 Value *Char = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0), Name: "char");
4015 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
4016 Value *Cast = B.CreateIntCast(V: Char, DestTy: IntTy, /*isSigned*/ true, Name: "chari");
4017 Value *NewCI = emitFPutC(Char: Cast, File: CI->getArgOperand(i: 3), B, TLI);
4018 return NewCI ? ConstantInt::get(Ty: CI->getType(), V: 1) : nullptr;
4019 }
4020 }
4021
4022 return nullptr;
4023}
4024
4025Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
4026 optimizeErrorReporting(CI, B, StreamArg: 1);
4027
4028 // Don't rewrite fputs to fwrite when optimising for size because fwrite
4029 // requires more arguments and thus extra MOVs are required.
4030 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
4031 QueryType: PGSOQueryType::IRPass))
4032 return nullptr;
4033
4034 // We can't optimize if return value is used.
4035 if (!CI->use_empty())
4036 return nullptr;
4037
4038 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
4039 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: 0));
4040 if (!Len)
4041 return nullptr;
4042
4043 // Known to have no uses (see above).
4044 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4045 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4046 return copyFlags(
4047 Old: *CI,
4048 New: emitFWrite(Ptr: CI->getArgOperand(i: 0),
4049 Size: ConstantInt::get(Ty: SizeTTy, V: Len - 1),
4050 File: CI->getArgOperand(i: 1), B, DL, TLI));
4051}
4052
4053Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
4054 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
4055 if (!CI->use_empty())
4056 return nullptr;
4057
4058 // Check for a constant string.
4059 // puts("") -> putchar('\n')
4060 StringRef Str;
4061 if (getConstantStringInfo(V: CI->getArgOperand(i: 0), Str) && Str.empty()) {
4062 // putchar takes an argument of the same type as puts returns, i.e.,
4063 // int, which need not be 32 bits wide.
4064 Type *IntTy = CI->getType();
4065 return copyFlags(Old: *CI, New: emitPutChar(Char: ConstantInt::get(Ty: IntTy, V: '\n'), B, TLI));
4066 }
4067
4068 return nullptr;
4069}
4070
4071Value *LibCallSimplifier::optimizeExit(CallInst *CI) {
4072
4073 // Mark 'exit' as cold if its not exit(0) (success).
4074 const APInt *C;
4075 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
4076 match(V: CI->getArgOperand(i: 0), P: m_APInt(Res&: C)) && !C->isZero()) {
4077 CI->addFnAttr(Kind: Attribute::Cold);
4078 }
4079 return nullptr;
4080}
4081
4082Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
4083 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
4084 return copyFlags(Old: *CI, New: B.CreateMemMove(Dst: CI->getArgOperand(i: 1), DstAlign: Align(1),
4085 Src: CI->getArgOperand(i: 0), SrcAlign: Align(1),
4086 Size: CI->getArgOperand(i: 2)));
4087}
4088
4089bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
4090 SmallString<20> FloatFuncName = FuncName;
4091 FloatFuncName += 'f';
4092 return isLibFuncEmittable(M, TLI, Name: FloatFuncName);
4093}
4094
4095Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
4096 IRBuilderBase &Builder) {
4097 Module *M = CI->getModule();
4098 Function *Callee = CI->getCalledFunction();
4099 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4100
4101 // Check for string/memory library functions.
4102 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4103 // Make sure we never change the calling convention.
4104 assert(
4105 (ignoreCallingConv(Func) ||
4106 TargetLibraryInfoImpl::isCallingConvCCompatible(CI)) &&
4107 "Optimizing string/memory libcall would change the calling convention");
4108 switch (Func) {
4109 case LibFunc_strcat:
4110 return optimizeStrCat(CI, B&: Builder);
4111 case LibFunc_strncat:
4112 return optimizeStrNCat(CI, B&: Builder);
4113 case LibFunc_strchr:
4114 return optimizeStrChr(CI, B&: Builder);
4115 case LibFunc_strrchr:
4116 return optimizeStrRChr(CI, B&: Builder);
4117 case LibFunc_strcmp:
4118 return optimizeStrCmp(CI, B&: Builder);
4119 case LibFunc_strncmp:
4120 return optimizeStrNCmp(CI, B&: Builder);
4121 case LibFunc_strcpy:
4122 return optimizeStrCpy(CI, B&: Builder);
4123 case LibFunc_stpcpy:
4124 return optimizeStpCpy(CI, B&: Builder);
4125 case LibFunc_strlcpy:
4126 return optimizeStrLCpy(CI, B&: Builder);
4127 case LibFunc_stpncpy:
4128 return optimizeStringNCpy(CI, /*RetEnd=*/true, B&: Builder);
4129 case LibFunc_strncpy:
4130 return optimizeStringNCpy(CI, /*RetEnd=*/false, B&: Builder);
4131 case LibFunc_strlen:
4132 return optimizeStrLen(CI, B&: Builder);
4133 case LibFunc_strnlen:
4134 return optimizeStrNLen(CI, B&: Builder);
4135 case LibFunc_strpbrk:
4136 return optimizeStrPBrk(CI, B&: Builder);
4137 case LibFunc_strndup:
4138 return optimizeStrNDup(CI, B&: Builder);
4139 case LibFunc_strtol:
4140 case LibFunc_strtod:
4141 case LibFunc_strtof:
4142 case LibFunc_strtoul:
4143 case LibFunc_strtoll:
4144 case LibFunc_strtold:
4145 case LibFunc_strtoull:
4146 return optimizeStrTo(CI, B&: Builder);
4147 case LibFunc_strspn:
4148 return optimizeStrSpn(CI, B&: Builder);
4149 case LibFunc_strcspn:
4150 return optimizeStrCSpn(CI, B&: Builder);
4151 case LibFunc_strstr:
4152 return optimizeStrStr(CI, B&: Builder);
4153 case LibFunc_memchr:
4154 return optimizeMemChr(CI, B&: Builder);
4155 case LibFunc_memrchr:
4156 return optimizeMemRChr(CI, B&: Builder);
4157 case LibFunc_bcmp:
4158 return optimizeBCmp(CI, B&: Builder);
4159 case LibFunc_memcmp:
4160 return optimizeMemCmp(CI, B&: Builder);
4161 case LibFunc_memcpy:
4162 return optimizeMemCpy(CI, B&: Builder);
4163 case LibFunc_memccpy:
4164 return optimizeMemCCpy(CI, B&: Builder);
4165 case LibFunc_mempcpy:
4166 return optimizeMemPCpy(CI, B&: Builder);
4167 case LibFunc_memmove:
4168 return optimizeMemMove(CI, B&: Builder);
4169 case LibFunc_memset:
4170 return optimizeMemSet(CI, B&: Builder);
4171 case LibFunc_realloc:
4172 return optimizeRealloc(CI, B&: Builder);
4173 case LibFunc_wcslen:
4174 return optimizeWcslen(CI, B&: Builder);
4175 case LibFunc_bcopy:
4176 return optimizeBCopy(CI, B&: Builder);
4177 case LibFunc_Znwm:
4178 case LibFunc_ZnwmRKSt9nothrow_t:
4179 case LibFunc_ZnwmSt11align_val_t:
4180 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
4181 case LibFunc_Znam:
4182 case LibFunc_ZnamRKSt9nothrow_t:
4183 case LibFunc_ZnamSt11align_val_t:
4184 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
4185 case LibFunc_Znwm12__hot_cold_t:
4186 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
4187 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
4188 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4189 case LibFunc_Znam12__hot_cold_t:
4190 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
4191 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
4192 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4193 case LibFunc_size_returning_new:
4194 case LibFunc_size_returning_new_hot_cold:
4195 case LibFunc_size_returning_new_aligned:
4196 case LibFunc_size_returning_new_aligned_hot_cold:
4197 return optimizeNew(CI, B&: Builder, Func);
4198 default:
4199 break;
4200 }
4201 }
4202 return nullptr;
4203}
4204
4205/// Constant folding nan/nanf/nanl.
4206static Value *optimizeNaN(CallInst *CI) {
4207 StringRef CharSeq;
4208 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: CharSeq))
4209 return nullptr;
4210
4211 APInt Fill;
4212 // Treat empty strings as if they were zero.
4213 if (CharSeq.empty())
4214 Fill = APInt(32, 0);
4215 else if (CharSeq.getAsInteger(Radix: 0, Result&: Fill))
4216 return nullptr;
4217
4218 return ConstantFP::getQNaN(Ty: CI->getType(), /*Negative=*/false, Payload: &Fill);
4219}
4220
4221Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
4222 LibFunc Func,
4223 IRBuilderBase &Builder) {
4224 const Module *M = CI->getModule();
4225
4226 // Don't optimize calls that require strict floating point semantics.
4227 if (CI->isStrictFP())
4228 return nullptr;
4229
4230 if (Value *V = optimizeSymmetric(CI, Func, B&: Builder))
4231 return V;
4232
4233 switch (Func) {
4234 case LibFunc_sinpif:
4235 case LibFunc_sinpi:
4236 return optimizeSinCosPi(CI, /*IsSin*/true, B&: Builder);
4237 case LibFunc_cospif:
4238 case LibFunc_cospi:
4239 return optimizeSinCosPi(CI, /*IsSin*/false, B&: Builder);
4240 case LibFunc_sinf:
4241 case LibFunc_sinl:
4242 if (CI->doesNotAccessMemory())
4243 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::sin);
4244 return nullptr;
4245 case LibFunc_cosf:
4246 case LibFunc_cosl:
4247 if (CI->doesNotAccessMemory())
4248 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::cos);
4249 return nullptr;
4250 case LibFunc_powf:
4251 case LibFunc_pow:
4252 case LibFunc_powl:
4253 return optimizePow(Pow: CI, B&: Builder);
4254 case LibFunc_exp2l:
4255 case LibFunc_exp2:
4256 case LibFunc_exp2f:
4257 return optimizeExp2(CI, B&: Builder);
4258 case LibFunc_ldexp:
4259 case LibFunc_ldexpf:
4260 case LibFunc_ldexpl:
4261 case LibFunc_scalbn:
4262 case LibFunc_scalbnf:
4263 case LibFunc_scalbnl:
4264 // LLVM floating-point types have radix 2, so scalbn is equivalent to
4265 // ldexp. Do not replace a libcall that may set errno.
4266 if (CI->doesNotAccessMemory()) {
4267 Value *NewCall =
4268 Builder.CreateLdexp(Src: CI->getArgOperand(i: 0), Exp: CI->getArgOperand(i: 1), FMFSource: CI);
4269 NewCall->takeName(V: CI);
4270 return copyFlags(Old: *CI, New: NewCall);
4271 }
4272 return nullptr;
4273 case LibFunc_fabsf:
4274 case LibFunc_fabs:
4275 case LibFunc_fabsl:
4276 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::fabs);
4277 case LibFunc_sqrtf:
4278 case LibFunc_sqrt:
4279 case LibFunc_sqrtl:
4280 if (Value *V = optimizeSqrt(CI, B&: Builder))
4281 return V;
4282 if (CI->doesNotAccessMemory())
4283 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::sqrt);
4284 return nullptr;
4285 case LibFunc_fmod:
4286 case LibFunc_fmodf:
4287 case LibFunc_fmodl:
4288 return optimizeFMod(CI, B&: Builder);
4289 case LibFunc_logf:
4290 case LibFunc_log:
4291 case LibFunc_logl:
4292 case LibFunc_log10f:
4293 case LibFunc_log10:
4294 case LibFunc_log10l:
4295 case LibFunc_log1pf:
4296 case LibFunc_log1p:
4297 case LibFunc_log1pl:
4298 case LibFunc_log2f:
4299 case LibFunc_log2:
4300 case LibFunc_log2l:
4301 case LibFunc_logbf:
4302 case LibFunc_logb:
4303 case LibFunc_logbl:
4304 return optimizeLog(Log: CI, B&: Builder);
4305 case LibFunc_tan:
4306 case LibFunc_tanf:
4307 case LibFunc_tanl:
4308 case LibFunc_sinh:
4309 case LibFunc_sinhf:
4310 case LibFunc_sinhl:
4311 case LibFunc_asinh:
4312 case LibFunc_asinhf:
4313 case LibFunc_asinhl:
4314 case LibFunc_cosh:
4315 case LibFunc_coshf:
4316 case LibFunc_coshl:
4317 case LibFunc_atanh:
4318 case LibFunc_atanhf:
4319 case LibFunc_atanhl:
4320 return optimizeTrigInversionPairs(CI, B&: Builder);
4321 case LibFunc_ceilf:
4322 case LibFunc_ceil:
4323 case LibFunc_ceill:
4324 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::ceil);
4325 case LibFunc_floorf:
4326 case LibFunc_floor:
4327 case LibFunc_floorl:
4328 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::floor);
4329 case LibFunc_roundf:
4330 case LibFunc_round:
4331 case LibFunc_roundl:
4332 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::round);
4333 case LibFunc_roundevenf:
4334 case LibFunc_roundeven:
4335 case LibFunc_roundevenl:
4336 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::roundeven);
4337 case LibFunc_nearbyintf:
4338 case LibFunc_nearbyint:
4339 case LibFunc_nearbyintl:
4340 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::nearbyint);
4341 case LibFunc_rintf:
4342 case LibFunc_rint:
4343 case LibFunc_rintl:
4344 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::rint);
4345 case LibFunc_truncf:
4346 case LibFunc_trunc:
4347 case LibFunc_truncl:
4348 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::trunc);
4349 case LibFunc_sin:
4350 case LibFunc_cos:
4351 if (UnsafeFPShrink &&
4352 hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4353 if (Value *V = optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true))
4354 return V;
4355 if (CI->doesNotAccessMemory())
4356 return replaceUnaryCall(
4357 CI, B&: Builder, IID: Func == LibFunc_sin ? Intrinsic::sin : Intrinsic::cos);
4358 return nullptr;
4359 case LibFunc_acos:
4360 case LibFunc_acosh:
4361 case LibFunc_asin:
4362 case LibFunc_atan:
4363 case LibFunc_cbrt:
4364 case LibFunc_exp:
4365 case LibFunc_exp10:
4366 case LibFunc_expm1:
4367 case LibFunc_tanh:
4368 if (UnsafeFPShrink && hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4369 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true);
4370 return nullptr;
4371 case LibFunc_copysign:
4372 if (hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4373 if (Value *V = optimizeBinaryDoubleFP(CI, B&: Builder, TLI))
4374 return V;
4375 [[fallthrough]];
4376 case LibFunc_copysignf:
4377 case LibFunc_copysignl:
4378 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::copysign);
4379 case LibFunc_fdim:
4380 case LibFunc_fdimf:
4381 case LibFunc_fdiml:
4382 return optimizeFdim(CI, B&: Builder);
4383 case LibFunc_fminf:
4384 case LibFunc_fmin:
4385 case LibFunc_fminl:
4386 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::minnum);
4387 case LibFunc_fmaxf:
4388 case LibFunc_fmax:
4389 case LibFunc_fmaxl:
4390 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::maxnum);
4391 case LibFunc_fminimum_numf:
4392 case LibFunc_fminimum_num:
4393 case LibFunc_fminimum_numl:
4394 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::minimumnum);
4395 case LibFunc_fmaximum_numf:
4396 case LibFunc_fmaximum_num:
4397 case LibFunc_fmaximum_numl:
4398 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::maximumnum);
4399 case LibFunc_cabs:
4400 case LibFunc_cabsf:
4401 case LibFunc_cabsl:
4402 return optimizeCAbs(CI, B&: Builder);
4403 case LibFunc_remquo:
4404 case LibFunc_remquof:
4405 case LibFunc_remquol:
4406 return optimizeRemquo(CI, B&: Builder);
4407 case LibFunc_nan:
4408 case LibFunc_nanf:
4409 case LibFunc_nanl:
4410 return optimizeNaN(CI);
4411 default:
4412 return nullptr;
4413 }
4414}
4415
4416Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) {
4417 Module *M = CI->getModule();
4418 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
4419
4420 Function *Callee = CI->getCalledFunction();
4421 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4422
4423 // Most calls are neither intrinsics nor known library functions. Bail out
4424 // before doing the (comparatively expensive) set-up below. This also covers
4425 // nobuiltin calls, since operator new calls are always library functions.
4426 if (Func == NotLibFunc && !isa<IntrinsicInst>(Val: CI))
4427 return nullptr;
4428
4429 // TODO: Split out the code below that operates on FP calls so that
4430 // we can all non-FP calls with the StrictFP attribute to be
4431 // optimized.
4432 if (CI->isNoBuiltin()) {
4433 // Optionally update operator new calls.
4434 return maybeOptimizeNoBuiltinOperatorNew(CI, B&: Builder);
4435 }
4436
4437 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4438
4439 SmallVector<OperandBundleDef, 2> OpBundles;
4440 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4441
4442 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4443 Builder.setDefaultOperandBundles(OpBundles);
4444
4445 // Command-line parameter overrides instruction attribute.
4446 // This can't be moved to optimizeFloatingPointLibCall() because it may be
4447 // used by the intrinsic optimizations.
4448 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
4449 UnsafeFPShrink = EnableUnsafeFPShrink;
4450 else if (isa<FPMathOperator>(Val: CI) && CI->isFast())
4451 UnsafeFPShrink = true;
4452
4453 // First, check for intrinsics.
4454 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI)) {
4455 if (!IsCallingConvC)
4456 return nullptr;
4457 // The FP intrinsics have corresponding constrained versions so we don't
4458 // need to check for the StrictFP attribute here.
4459 switch (II->getIntrinsicID()) {
4460 case Intrinsic::pow:
4461 return optimizePow(Pow: CI, B&: Builder);
4462 case Intrinsic::exp2:
4463 return optimizeExp2(CI, B&: Builder);
4464 case Intrinsic::log:
4465 case Intrinsic::log2:
4466 case Intrinsic::log10:
4467 return optimizeLog(Log: CI, B&: Builder);
4468 case Intrinsic::sqrt:
4469 return optimizeSqrt(CI, B&: Builder);
4470 case Intrinsic::memset:
4471 return optimizeMemSet(CI, B&: Builder);
4472 case Intrinsic::memcpy:
4473 return optimizeMemCpy(CI, B&: Builder);
4474 case Intrinsic::memmove:
4475 return optimizeMemMove(CI, B&: Builder);
4476 case Intrinsic::sin:
4477 case Intrinsic::cos:
4478 if (UnsafeFPShrink)
4479 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, /*isPrecise=*/true);
4480 return nullptr;
4481 case Intrinsic::sincos:
4482 if (UnsafeFPShrink)
4483 return optimizeSinCosDoubleFP(CI, B&: Builder);
4484 return nullptr;
4485 default:
4486 return nullptr;
4487 }
4488 }
4489
4490 // Also try to simplify calls to fortified library functions.
4491 if (Value *SimplifiedFortifiedCI =
4492 FortifiedSimplifier.optimizeCall(CI, B&: Builder))
4493 return SimplifiedFortifiedCI;
4494
4495 // Then check for known library functions.
4496 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4497 // We never change the calling convention.
4498 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4499 return nullptr;
4500 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
4501 return V;
4502 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
4503 return V;
4504 switch (Func) {
4505 case LibFunc_ffs:
4506 case LibFunc_ffsl:
4507 case LibFunc_ffsll:
4508 return optimizeFFS(CI, B&: Builder);
4509 case LibFunc_fls:
4510 case LibFunc_flsl:
4511 case LibFunc_flsll:
4512 return optimizeFls(CI, B&: Builder);
4513 case LibFunc_abs:
4514 case LibFunc_labs:
4515 case LibFunc_llabs:
4516 return optimizeAbs(CI, B&: Builder);
4517 case LibFunc_isdigit:
4518 return optimizeIsDigit(CI, B&: Builder);
4519 case LibFunc_isascii:
4520 return optimizeIsAscii(CI, B&: Builder);
4521 case LibFunc_toascii:
4522 return optimizeToAscii(CI, B&: Builder);
4523 case LibFunc_atoi:
4524 case LibFunc_atol:
4525 case LibFunc_atoll:
4526 return optimizeAtoi(CI, B&: Builder);
4527 case LibFunc_strtol:
4528 case LibFunc_strtoll:
4529 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/true);
4530 case LibFunc_strtoul:
4531 case LibFunc_strtoull:
4532 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/false);
4533 case LibFunc_printf:
4534 return optimizePrintF(CI, B&: Builder);
4535 case LibFunc_sprintf:
4536 return optimizeSPrintF(CI, B&: Builder);
4537 case LibFunc_snprintf:
4538 return optimizeSnPrintF(CI, B&: Builder);
4539 case LibFunc_fprintf:
4540 return optimizeFPrintF(CI, B&: Builder);
4541 case LibFunc_fwrite:
4542 return optimizeFWrite(CI, B&: Builder);
4543 case LibFunc_fputs:
4544 return optimizeFPuts(CI, B&: Builder);
4545 case LibFunc_puts:
4546 return optimizePuts(CI, B&: Builder);
4547 case LibFunc_perror:
4548 return optimizeErrorReporting(CI, B&: Builder);
4549 case LibFunc_vfprintf:
4550 case LibFunc_fiprintf:
4551 return optimizeErrorReporting(CI, B&: Builder, StreamArg: 0);
4552 case LibFunc_exit:
4553 case LibFunc_Exit:
4554 return optimizeExit(CI);
4555 default:
4556 return nullptr;
4557 }
4558 }
4559 return nullptr;
4560}
4561
4562LibCallSimplifier::LibCallSimplifier(
4563 const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT,
4564 DomConditionCache *DC, AssumptionCache *AC, OptimizationRemarkEmitter &ORE,
4565 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
4566 function_ref<void(Instruction *, Value *)> Replacer,
4567 function_ref<void(Instruction *)> Eraser)
4568 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), DT(DT), DC(DC), AC(AC),
4569 ORE(ORE), BFI(BFI), PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
4570
4571void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
4572 // Indirect through the replacer used in this instance.
4573 Replacer(I, With);
4574}
4575
4576void LibCallSimplifier::eraseFromParent(Instruction *I) {
4577 Eraser(I);
4578}
4579
4580// TODO:
4581// Additional cases that we need to add to this file:
4582//
4583// cbrt:
4584// * cbrt(expN(X)) -> expN(x/3)
4585// * cbrt(sqrt(x)) -> pow(x,1/6)
4586// * cbrt(cbrt(x)) -> pow(x,1/9)
4587//
4588// exp, expf, expl:
4589// * exp(log(x)) -> x
4590//
4591// log, logf, logl:
4592// * log(exp(x)) -> x
4593// * log(exp(y)) -> y*log(e)
4594// * log(exp10(y)) -> y*log(10)
4595// * log(sqrt(x)) -> 0.5*log(x)
4596//
4597// pow, powf, powl:
4598// * pow(sqrt(x),y) -> pow(x,y*0.5)
4599// * pow(pow(x,y),z)-> pow(x,y*z)
4600//
4601// signbit:
4602// * signbit(cnst) -> cnst'
4603// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
4604//
4605// sqrt, sqrtf, sqrtl:
4606// * sqrt(expN(x)) -> expN(x*0.5)
4607// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
4608// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
4609//
4610
4611//===----------------------------------------------------------------------===//
4612// Fortified Library Call Optimizations
4613//===----------------------------------------------------------------------===//
4614
4615bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
4616 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
4617 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
4618 // If this function takes a flag argument, the implementation may use it to
4619 // perform extra checks. Don't fold into the non-checking variant.
4620 if (FlagOp) {
4621 ConstantInt *Flag = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *FlagOp));
4622 if (!Flag || !Flag->isZero())
4623 return false;
4624 }
4625
4626 if (SizeOp && CI->getArgOperand(i: ObjSizeOp) == CI->getArgOperand(i: *SizeOp))
4627 return true;
4628
4629 if (ConstantInt *ObjSizeCI =
4630 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: ObjSizeOp))) {
4631 if (ObjSizeCI->isMinusOne())
4632 return true;
4633 // If the object size wasn't -1 (unknown), bail out if we were asked to.
4634 if (OnlyLowerUnknownSize)
4635 return false;
4636 if (StrOp) {
4637 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: *StrOp));
4638 // If the length is 0 we don't know how long it is and so we can't
4639 // remove the check.
4640 if (Len)
4641 annotateDereferenceableBytes(CI, ArgNos: *StrOp, DereferenceableBytes: Len);
4642 else
4643 return false;
4644 return ObjSizeCI->getZExtValue() >= Len;
4645 }
4646
4647 if (SizeOp) {
4648 if (ConstantInt *SizeCI =
4649 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *SizeOp)))
4650 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4651 }
4652 }
4653 return false;
4654}
4655
4656Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4657 IRBuilderBase &B) {
4658 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4659 CallInst *NewCI =
4660 B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4661 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4662 mergeAttributesAndFlags(NewCI, Old: *CI);
4663 return CI->getArgOperand(i: 0);
4664 }
4665 return nullptr;
4666}
4667
4668Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4669 IRBuilderBase &B) {
4670 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4671 CallInst *NewCI =
4672 B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4673 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4674 mergeAttributesAndFlags(NewCI, Old: *CI);
4675 return CI->getArgOperand(i: 0);
4676 }
4677 return nullptr;
4678}
4679
4680Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4681 IRBuilderBase &B) {
4682 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4683 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
4684 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val,
4685 Size: CI->getArgOperand(i: 2), Align: Align(1));
4686 mergeAttributesAndFlags(NewCI, Old: *CI);
4687 return CI->getArgOperand(i: 0);
4688 }
4689 return nullptr;
4690}
4691
4692Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4693 IRBuilderBase &B) {
4694 const DataLayout &DL = CI->getDataLayout();
4695 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2))
4696 if (Value *Call = emitMemPCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4697 Len: CI->getArgOperand(i: 2), B, DL, TLI)) {
4698 return mergeAttributesAndFlags(NewCI: cast<CallInst>(Val: Call), Old: *CI);
4699 }
4700 return nullptr;
4701}
4702
4703Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4704 IRBuilderBase &B,
4705 LibFunc Func) {
4706 const DataLayout &DL = CI->getDataLayout();
4707 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1),
4708 *ObjSize = CI->getArgOperand(i: 2);
4709
4710 // __stpcpy_chk(x,x,...) -> x+strlen(x)
4711 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4712 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
4713 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
4714 }
4715
4716 // If a) we don't have any length information, or b) we know this will
4717 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4718 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
4719 // TODO: It might be nice to get a maximum length out of the possible
4720 // string lengths for varying.
4721 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: 1)) {
4722 if (Func == LibFunc_strcpy_chk)
4723 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
4724 else
4725 return copyFlags(Old: *CI, New: emitStpCpy(Dst, Src, B, TLI));
4726 }
4727
4728 if (OnlyLowerUnknownSize)
4729 return nullptr;
4730
4731 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4732 uint64_t Len = GetStringLength(V: Src);
4733 if (Len)
4734 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
4735 else
4736 return nullptr;
4737
4738 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4739 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4740 Value *LenV = ConstantInt::get(Ty: SizeTTy, V: Len);
4741 Value *Ret = emitMemCpyChk(Dst, Src, Len: LenV, ObjSize, B, DL, TLI);
4742 // If the function was an __stpcpy_chk, and we were able to fold it into
4743 // a __memcpy_chk, we still need to return the correct end pointer.
4744 if (Ret && Func == LibFunc_stpcpy_chk)
4745 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst,
4746 IdxList: ConstantInt::get(Ty: SizeTTy, V: Len - 1));
4747 return copyFlags(Old: *CI, New: cast<CallInst>(Val: Ret));
4748}
4749
4750Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4751 IRBuilderBase &B) {
4752 if (isFortifiedCallFoldable(CI, ObjSizeOp: 1, SizeOp: std::nullopt, StrOp: 0))
4753 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B,
4754 DL: CI->getDataLayout(), TLI));
4755 return nullptr;
4756}
4757
4758Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4759 IRBuilderBase &B,
4760 LibFunc Func) {
4761 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4762 if (Func == LibFunc_strncpy_chk)
4763 return copyFlags(Old: *CI,
4764 New: emitStrNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4765 Len: CI->getArgOperand(i: 2), B, TLI));
4766 else
4767 return copyFlags(Old: *CI,
4768 New: emitStpNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4769 Len: CI->getArgOperand(i: 2), B, TLI));
4770 }
4771
4772 return nullptr;
4773}
4774
4775Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4776 IRBuilderBase &B) {
4777 if (isFortifiedCallFoldable(CI, ObjSizeOp: 4, SizeOp: 3))
4778 return copyFlags(
4779 Old: *CI, New: emitMemCCpy(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
4780 Val: CI->getArgOperand(i: 2), Len: CI->getArgOperand(i: 3), B, TLI));
4781
4782 return nullptr;
4783}
4784
4785Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4786 IRBuilderBase &B) {
4787 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2)) {
4788 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 5));
4789 return copyFlags(Old: *CI,
4790 New: emitSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4791 Fmt: CI->getArgOperand(i: 4), Args: VariadicArgs, B, TLI));
4792 }
4793
4794 return nullptr;
4795}
4796
4797Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4798 IRBuilderBase &B) {
4799 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1)) {
4800 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 4));
4801 return copyFlags(Old: *CI,
4802 New: emitSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4803 VariadicArgs, B, TLI));
4804 }
4805
4806 return nullptr;
4807}
4808
4809Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4810 IRBuilderBase &B) {
4811 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2))
4812 return copyFlags(
4813 Old: *CI, New: emitStrCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1), B, TLI));
4814
4815 return nullptr;
4816}
4817
4818Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4819 IRBuilderBase &B) {
4820 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4821 return copyFlags(Old: *CI,
4822 New: emitStrLCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4823 Size: CI->getArgOperand(i: 2), B, TLI));
4824
4825 return nullptr;
4826}
4827
4828Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4829 IRBuilderBase &B) {
4830 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4831 return copyFlags(Old: *CI,
4832 New: emitStrNCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4833 Size: CI->getArgOperand(i: 2), B, TLI));
4834
4835 return nullptr;
4836}
4837
4838Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4839 IRBuilderBase &B) {
4840 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4841 return copyFlags(Old: *CI,
4842 New: emitStrLCpy(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4843 Size: CI->getArgOperand(i: 2), B, TLI));
4844
4845 return nullptr;
4846}
4847
4848Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4849 IRBuilderBase &B) {
4850 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2))
4851 return copyFlags(
4852 Old: *CI, New: emitVSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4853 Fmt: CI->getArgOperand(i: 4), VAList: CI->getArgOperand(i: 5), B, TLI));
4854
4855 return nullptr;
4856}
4857
4858Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4859 IRBuilderBase &B) {
4860 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1))
4861 return copyFlags(Old: *CI,
4862 New: emitVSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4863 VAList: CI->getArgOperand(i: 4), B, TLI));
4864
4865 return nullptr;
4866}
4867
4868Value *FortifiedLibCallSimplifier::optimizeCall(CallInst *CI,
4869 IRBuilderBase &Builder) {
4870 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4871 // Some clang users checked for _chk libcall availability using:
4872 // __has_builtin(__builtin___memcpy_chk)
4873 // When compiling with -fno-builtin, this is always true.
4874 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4875 // end up with fortified libcalls, which isn't acceptable in a freestanding
4876 // environment which only provides their non-fortified counterparts.
4877 //
4878 // Until we change clang and/or teach external users to check for availability
4879 // differently, disregard the "nobuiltin" attribute and TLI::has.
4880 //
4881 // PR23093.
4882
4883 Function *Callee = CI->getCalledFunction();
4884 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4885
4886 SmallVector<OperandBundleDef, 2> OpBundles;
4887 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4888
4889 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4890 Builder.setDefaultOperandBundles(OpBundles);
4891
4892 // First, check that this is a known library functions and that the prototype
4893 // is correct.
4894 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4895 if (Func == NotLibFunc)
4896 return nullptr;
4897
4898 // We never change the calling convention.
4899 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4900 return nullptr;
4901
4902 switch (Func) {
4903 case LibFunc_memcpy_chk:
4904 return optimizeMemCpyChk(CI, B&: Builder);
4905 case LibFunc_mempcpy_chk:
4906 return optimizeMemPCpyChk(CI, B&: Builder);
4907 case LibFunc_memmove_chk:
4908 return optimizeMemMoveChk(CI, B&: Builder);
4909 case LibFunc_memset_chk:
4910 return optimizeMemSetChk(CI, B&: Builder);
4911 case LibFunc_stpcpy_chk:
4912 case LibFunc_strcpy_chk:
4913 return optimizeStrpCpyChk(CI, B&: Builder, Func);
4914 case LibFunc_strlen_chk:
4915 return optimizeStrLenChk(CI, B&: Builder);
4916 case LibFunc_stpncpy_chk:
4917 case LibFunc_strncpy_chk:
4918 return optimizeStrpNCpyChk(CI, B&: Builder, Func);
4919 case LibFunc_memccpy_chk:
4920 return optimizeMemCCpyChk(CI, B&: Builder);
4921 case LibFunc_snprintf_chk:
4922 return optimizeSNPrintfChk(CI, B&: Builder);
4923 case LibFunc_sprintf_chk:
4924 return optimizeSPrintfChk(CI, B&: Builder);
4925 case LibFunc_strcat_chk:
4926 return optimizeStrCatChk(CI, B&: Builder);
4927 case LibFunc_strlcat_chk:
4928 return optimizeStrLCat(CI, B&: Builder);
4929 case LibFunc_strncat_chk:
4930 return optimizeStrNCatChk(CI, B&: Builder);
4931 case LibFunc_strlcpy_chk:
4932 return optimizeStrLCpyChk(CI, B&: Builder);
4933 case LibFunc_vsnprintf_chk:
4934 return optimizeVSNPrintfChk(CI, B&: Builder);
4935 case LibFunc_vsprintf_chk:
4936 return optimizeVSPrintfChk(CI, B&: Builder);
4937 default:
4938 break;
4939 }
4940 return nullptr;
4941}
4942
4943FortifiedLibCallSimplifier::FortifiedLibCallSimplifier(
4944 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4945 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
4946