1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/SmallString.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/Analysis/ConstantFolding.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/Utils/Local.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/AttributeMask.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/MDBuilder.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/PatternMatch.h"
34#include "llvm/IR/ProfDataUtils.h"
35#include "llvm/Support/Casting.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Support/KnownBits.h"
38#include "llvm/Support/KnownFPClass.h"
39#include "llvm/Support/MathExtras.h"
40#include "llvm/TargetParser/Triple.h"
41#include "llvm/Transforms/Utils/BuildLibCalls.h"
42#include "llvm/Transforms/Utils/Local.h"
43#include "llvm/Transforms/Utils/SizeOpts.h"
44
45#include <cmath>
46
47using namespace llvm;
48using namespace PatternMatch;
49
50#define DEBUG_TYPE "simplify-lib-calls"
51
52static cl::opt<bool>
53 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
54 cl::init(Val: false),
55 cl::desc("Enable unsafe double to float "
56 "shrinking for math lib calls"));
57
58// Enable conversion of operator new calls with a MemProf hot or cold hint
59// to an operator new call that takes a hot/cold hint. Off by default since
60// not all allocators currently support this extension.
61static cl::opt<bool>
62 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(Val: false),
63 cl::desc("Enable hot/cold operator new library calls"));
64enum class OptimizeExistingHotColdNewKind {
65 None,
66 Cold,
67 Always,
68};
69static cl::opt<OptimizeExistingHotColdNewKind> OptimizeExistingHotColdNew(
70 "optimize-existing-hot-cold-new", cl::Hidden,
71 cl::desc(
72 "Enable optimization of existing hot/cold operator new library calls"),
73 cl::values(
74 clEnumValN(
75 OptimizeExistingHotColdNewKind::None, "none",
76 "Do not optimize existing hot/cold operator new library calls"),
77 clEnumValN(OptimizeExistingHotColdNewKind::Cold, "cold",
78 "Only optimize existing hot/cold operator new library calls "
79 "if determined to be cold"),
80 clEnumValN(
81 OptimizeExistingHotColdNewKind::Always, "always",
82 "Always optimize existing hot/cold operator new library calls"),
83 clEnumValN(
84 OptimizeExistingHotColdNewKind::Always, "",
85 "Always optimize existing hot/cold operator new library calls")),
86 cl::init(Val: OptimizeExistingHotColdNewKind::None), cl::ValueOptional);
87static cl::opt<bool> OptimizeNoBuiltinHotColdNew(
88 "optimize-nobuiltin-hot-cold-new-new", cl::Hidden, cl::init(Val: false),
89 cl::desc("Enable transformation of nobuiltin operator new library calls"));
90static cl::opt<bool> MinExistingHotColdNewHint(
91 "min-existing-hot-cold-new-hint", cl::Hidden, cl::init(Val: false),
92 cl::desc("Take the minimum of compiler hint and existing hint when "
93 "optimizing existing hot/cold operator new library calls"));
94
95namespace llvm {
96extern cl::opt<bool> ProfcheckDisableMetadataFixes;
97} // namespace llvm
98
99namespace {
100
101// Specialized parser to ensure the hint is an 8 bit value (we can't specify
102// uint8_t to opt<> as that is interpreted to mean that we are passing a char
103// option with a specific set of values.
104struct HotColdHintParser : public cl::parser<unsigned> {
105 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
106
107 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
108 if (Arg.getAsInteger(Radix: 0, Result&: Value))
109 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
110
111 if (Value > 255)
112 return O.error(Message: "'" + Arg + "' value must be in the range [0, 255]!");
113
114 return false;
115 }
116};
117
118} // end anonymous namespace
119
120// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
121// and 255 is the hottest. Default to 1 value away from the coldest and hottest
122// hints, so that the compiler hinted allocations are slightly less strong than
123// manually inserted hints at the two extremes.
124static cl::opt<unsigned, false, HotColdHintParser> ColdNewHintValue(
125 "cold-new-hint-value", cl::Hidden, cl::init(Val: 1),
126 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
127static cl::opt<unsigned, false, HotColdHintParser>
128 NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(Val: 128),
129 cl::desc("Value to pass to hot/cold operator new for "
130 "notcold (warm) allocation"));
131static cl::opt<unsigned, false, HotColdHintParser> HotNewHintValue(
132 "hot-new-hint-value", cl::Hidden, cl::init(Val: 254),
133 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
134static cl::opt<unsigned, false, HotColdHintParser> AmbiguousNewHintValue(
135 "ambiguous-new-hint-value", cl::Hidden, cl::init(Val: 222),
136 cl::desc(
137 "Value to pass to hot/cold operator new for ambiguous allocation"));
138
139//===----------------------------------------------------------------------===//
140// Helper Functions
141//===----------------------------------------------------------------------===//
142
143static bool ignoreCallingConv(LibFunc Func) {
144 return Func == LibFunc_abs || Func == LibFunc_labs ||
145 Func == LibFunc_llabs || Func == LibFunc_strlen;
146}
147
148/// Return true if it is only used in equality comparisons with With.
149static bool isOnlyUsedInEqualityComparison(Value *V, Value *With) {
150 for (User *U : V->users()) {
151 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
152 if (IC->isEquality() && IC->getOperand(i_nocapture: 1) == With)
153 continue;
154 // Unknown instruction.
155 return false;
156 }
157 return true;
158}
159
160static bool callHasFloatingPointArgument(const CallInst *CI) {
161 return any_of(Range: CI->operands(), P: [](const Use &OI) {
162 return OI->getType()->isFloatingPointTy();
163 });
164}
165
166static bool callHasFP128Argument(const CallInst *CI) {
167 return any_of(Range: CI->operands(), P: [](const Use &OI) {
168 return OI->getType()->isFP128Ty();
169 });
170}
171
172// Convert the entire string Str representing an integer in Base, up to
173// the terminating nul if present, to a constant according to the rules
174// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
175// return the result, otherwise null.
176// The function assumes the string is encoded in ASCII and carefully
177// avoids converting sequences (including "") that the corresponding
178// library call might fail and set errno for.
179static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
180 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
181 if (Base < 2 || Base > 36)
182 if (Base != 0)
183 // Fail for an invalid base (required by POSIX).
184 return nullptr;
185
186 // Current offset into the original string to reflect in EndPtr.
187 size_t Offset = 0;
188 // Strip leading whitespace.
189 for ( ; Offset != Str.size(); ++Offset)
190 if (!isSpace(C: (unsigned char)Str[Offset])) {
191 Str = Str.substr(Start: Offset);
192 break;
193 }
194
195 if (Str.empty())
196 // Fail for empty subject sequences (POSIX allows but doesn't require
197 // strtol[l]/strtoul[l] to fail with EINVAL).
198 return nullptr;
199
200 // Strip but remember the sign.
201 bool Negate = Str[0] == '-';
202 if (Str[0] == '-' || Str[0] == '+') {
203 Str = Str.drop_front();
204 if (Str.empty())
205 // Fail for a sign with nothing after it.
206 return nullptr;
207 ++Offset;
208 }
209
210 // Set Max to the absolute value of the minimum (for signed), or
211 // to the maximum (for unsigned) value representable in the type.
212 Type *RetTy = CI->getType();
213 unsigned NBits = RetTy->getPrimitiveSizeInBits();
214 uint64_t Max = AsSigned && Negate ? 1 : 0;
215 Max += AsSigned ? maxIntN(N: NBits) : maxUIntN(N: NBits);
216
217 // Autodetect Base if it's zero and consume the "0x" prefix.
218 if (Str.size() > 1) {
219 if (Str[0] == '0') {
220 if (toUpper(x: (unsigned char)Str[1]) == 'X') {
221 if (Str.size() == 2 || (Base && Base != 16))
222 // Fail if Base doesn't allow the "0x" prefix or for the prefix
223 // alone that implementations like BSD set errno to EINVAL for.
224 return nullptr;
225
226 Str = Str.drop_front(N: 2);
227 Offset += 2;
228 Base = 16;
229 }
230 else if (Base == 0)
231 Base = 8;
232 } else if (Base == 0)
233 Base = 10;
234 }
235 else if (Base == 0)
236 Base = 10;
237
238 // Convert the rest of the subject sequence, not including the sign,
239 // to its uint64_t representation (this assumes the source character
240 // set is ASCII).
241 uint64_t Result = 0;
242 for (unsigned i = 0; i != Str.size(); ++i) {
243 unsigned char DigVal = Str[i];
244 if (isDigit(C: DigVal))
245 DigVal = DigVal - '0';
246 else {
247 DigVal = toUpper(x: DigVal);
248 if (isAlpha(C: DigVal))
249 DigVal = DigVal - 'A' + 10;
250 else
251 return nullptr;
252 }
253
254 if (DigVal >= Base)
255 // Fail if the digit is not valid in the Base.
256 return nullptr;
257
258 // Add the digit and fail if the result is not representable in
259 // the (unsigned form of the) destination type.
260 bool VFlow;
261 Result = SaturatingMultiplyAdd(X: Result, Y: Base, A: (uint64_t)DigVal, ResultOverflowed: &VFlow);
262 if (VFlow || Result > Max)
263 return nullptr;
264 }
265
266 if (EndPtr) {
267 // Store the pointer to the end.
268 Value *Off = B.getInt64(C: Offset + Str.size());
269 Value *StrBeg = CI->getArgOperand(i: 0);
270 Value *StrEnd = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: StrBeg, IdxList: Off, Name: "endptr");
271 B.CreateStore(Val: StrEnd, Ptr: EndPtr);
272 }
273
274 if (Negate) {
275 // Unsigned negation doesn't overflow.
276 Result = -Result;
277 // For unsigned numbers, discard sign bits.
278 if (!AsSigned)
279 Result &= maxUIntN(N: NBits);
280 }
281
282 return ConstantInt::get(Ty: RetTy, V: Result, IsSigned: AsSigned);
283}
284
285static bool isOnlyUsedInComparisonWithZero(Value *V) {
286 for (User *U : V->users()) {
287 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
288 if (Constant *C = dyn_cast<Constant>(Val: IC->getOperand(i_nocapture: 1)))
289 if (C->isNullValue())
290 continue;
291 // Unknown instruction.
292 return false;
293 }
294 return true;
295}
296
297static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
298 const SimplifyQuery &SQ) {
299 if (!isOnlyUsedInComparisonWithZero(V: CI))
300 return false;
301
302 if (!isDereferenceablePointer(V: Str, Size: APInt(64, Len), Q: SQ))
303 return false;
304
305 if (CI->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemory))
306 return false;
307
308 return true;
309}
310
311static void annotateDereferenceableBytes(CallInst *CI,
312 ArrayRef<unsigned> ArgNos,
313 uint64_t DereferenceableBytes) {
314 const Function *F = CI->getCaller();
315 if (!F)
316 return;
317 for (unsigned ArgNo : ArgNos) {
318 uint64_t DerefBytes = DereferenceableBytes;
319 unsigned AS = CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
320 if (!llvm::NullPointerIsDefined(F, AS) ||
321 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
322 DerefBytes = std::max(a: CI->getParamDereferenceableOrNullBytes(i: ArgNo),
323 b: DereferenceableBytes);
324
325 if (CI->getParamDereferenceableBytes(i: ArgNo) < DerefBytes) {
326 CI->removeParamAttr(ArgNo, Kind: Attribute::Dereferenceable);
327 if (!llvm::NullPointerIsDefined(F, AS) ||
328 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
329 CI->removeParamAttr(ArgNo, Kind: Attribute::DereferenceableOrNull);
330 CI->addParamAttr(ArgNo, Attr: Attribute::getWithDereferenceableBytes(
331 Context&: CI->getContext(), Bytes: DerefBytes));
332 }
333 }
334}
335
336static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI,
337 ArrayRef<unsigned> ArgNos) {
338 Function *F = CI->getCaller();
339 if (!F)
340 return;
341
342 for (unsigned ArgNo : ArgNos) {
343 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NoUndef))
344 CI->addParamAttr(ArgNo, Kind: Attribute::NoUndef);
345
346 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull)) {
347 unsigned AS =
348 CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
349 if (llvm::NullPointerIsDefined(F, AS))
350 continue;
351 CI->addParamAttr(ArgNo, Kind: Attribute::NonNull);
352 }
353
354 annotateDereferenceableBytes(CI, ArgNos: ArgNo, DereferenceableBytes: 1);
355 }
356}
357
358static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> ArgNos,
359 Value *Size, const DataLayout &DL) {
360 if (ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size)) {
361 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
362 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: LenC->getZExtValue());
363 } else if (isKnownNonZero(V: Size, Q: DL)) {
364 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
365 uint64_t X, Y;
366 uint64_t DerefMin = 1;
367 if (match(V: Size, P: m_Select(C: m_Value(), L: m_ConstantInt(V&: X), R: m_ConstantInt(V&: Y)))) {
368 DerefMin = std::min(a: X, b: Y);
369 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: DerefMin);
370 }
371 }
372}
373
374// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
375// easier chaining. Calls to emit* and B.createCall should probably be wrapped
376// in this function when New is created to replace Old. Callers should take
377// care to check Old.isMustTailCall() if they aren't replacing Old directly
378// with New.
379static Value *copyFlags(const CallInst &Old, Value *New) {
380 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
381 assert(!Old.isNoTailCall() && "do not copy notail call flags");
382 if (auto *NewCI = dyn_cast_or_null<CallInst>(Val: New))
383 NewCI->setTailCallKind(Old.getTailCallKind());
384 return New;
385}
386
387static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
388 NewCI->setAttributes(AttributeList::get(
389 C&: NewCI->getContext(), Attrs: {NewCI->getAttributes(), Old.getAttributes()}));
390 NewCI->removeRetAttrs(AttrsToRemove: AttributeFuncs::typeIncompatible(
391 Ty: NewCI->getType(), AS: NewCI->getRetAttributes()));
392 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
393 NewCI->removeParamAttrs(
394 ArgNo: I, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewCI->getArgOperand(i: I)->getType(),
395 AS: NewCI->getParamAttributes(ArgNo: I)));
396
397 return copyFlags(Old, New: NewCI);
398}
399
400// Helper to avoid truncating the length if size_t is 32-bits.
401static StringRef substr(StringRef Str, uint64_t Len) {
402 return Len >= Str.size() ? Str : Str.substr(Start: 0, N: Len);
403}
404
405//===----------------------------------------------------------------------===//
406// String and Memory Library Call Optimizations
407//===----------------------------------------------------------------------===//
408
409Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
410 // Extract some information from the instruction
411 Value *Dst = CI->getArgOperand(i: 0);
412 Value *Src = CI->getArgOperand(i: 1);
413 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
414
415 // See if we can get the length of the input string.
416 uint64_t Len = GetStringLength(V: Src);
417 if (Len)
418 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
419 else
420 return nullptr;
421 --Len; // Unbias length.
422
423 // Handle the simple, do-nothing case: strcat(x, "") -> x
424 if (Len == 0)
425 return Dst;
426
427 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len, B));
428}
429
430Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
431 IRBuilderBase &B) {
432 // We need to find the end of the destination string. That's where the
433 // memory is to be moved to. We just generate a call to strlen.
434 Value *DstLen = emitStrLen(Ptr: Dst, B, DL, TLI);
435 if (!DstLen)
436 return nullptr;
437
438 // Now that we have the destination's length, we must index into the
439 // destination's pointer to get the actual memcpy destination (end of
440 // the string .. we're concatenating).
441 Value *CpyDst = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: DstLen, Name: "endptr");
442
443 // We have enough information to now generate the memcpy call to do the
444 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
445 B.CreateMemCpy(Dst: CpyDst, DstAlign: Align(1), Src, SrcAlign: Align(1),
446 Size: TLI->getAsSizeT(V: Len + 1, M: *B.getModule()));
447 return Dst;
448}
449
450Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
451 // Extract some information from the instruction.
452 Value *Dst = CI->getArgOperand(i: 0);
453 Value *Src = CI->getArgOperand(i: 1);
454 Value *Size = CI->getArgOperand(i: 2);
455 uint64_t Len;
456 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
457 if (isKnownNonZero(V: Size, Q: DL))
458 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
459
460 // We don't do anything if length is not constant.
461 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size);
462 if (LengthArg) {
463 Len = LengthArg->getZExtValue();
464 // strncat(x, c, 0) -> x
465 if (!Len)
466 return Dst;
467 } else {
468 return nullptr;
469 }
470
471 // See if we can get the length of the input string.
472 uint64_t SrcLen = GetStringLength(V: Src);
473 if (SrcLen) {
474 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
475 --SrcLen; // Unbias length.
476 } else {
477 return nullptr;
478 }
479
480 // strncat(x, "", c) -> x
481 if (SrcLen == 0)
482 return Dst;
483
484 // We don't optimize this case.
485 if (Len < SrcLen)
486 return nullptr;
487
488 // strncat(x, s, c) -> strcat(x, s)
489 // s is constant so the strcat can be optimized further.
490 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len: SrcLen, B));
491}
492
493// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
494// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
495// is that either S is dereferenceable or the value of N is nonzero.
496static Value* memChrToCharCompare(CallInst *CI, Value *NBytes,
497 IRBuilderBase &B, const DataLayout &DL)
498{
499 Value *Src = CI->getArgOperand(i: 0);
500 Value *CharVal = CI->getArgOperand(i: 1);
501
502 // Fold memchr(A, C, N) == A to N && *A == C.
503 Type *CharTy = B.getInt8Ty();
504 Value *Char0 = B.CreateLoad(Ty: CharTy, Ptr: Src);
505 CharVal = B.CreateTrunc(V: CharVal, DestTy: CharTy);
506 Value *Cmp = B.CreateICmpEQ(LHS: Char0, RHS: CharVal, Name: "char0cmp");
507
508 if (NBytes) {
509 Value *Zero = ConstantInt::get(Ty: NBytes->getType(), V: 0);
510 Value *And = B.CreateICmpNE(LHS: NBytes, RHS: Zero);
511 Cmp = B.CreateLogicalAnd(Cond1: And, Cond2: Cmp);
512 // The and above is based on the byte count and the query, neither of which
513 // we know without value profiling, so mark the profile as unknown.
514 if (auto *SI = dyn_cast<SelectInst>(Val: Cmp))
515 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE);
516 }
517
518 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
519 return B.CreateSelect(C: Cmp, True: Src, False: NullPtr);
520}
521
522Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
523 Value *SrcStr = CI->getArgOperand(i: 0);
524 Value *CharVal = CI->getArgOperand(i: 1);
525 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
526
527 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
528 return memChrToCharCompare(CI, NBytes: nullptr, B, DL);
529
530 // If the second operand is non-constant, see if we can compute the length
531 // of the input string and turn this into memchr.
532 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
533 if (!CharC) {
534 uint64_t Len = GetStringLength(V: SrcStr);
535 if (Len)
536 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len);
537 else
538 return nullptr;
539
540 Function *Callee = CI->getCalledFunction();
541 FunctionType *FT = Callee->getFunctionType();
542 unsigned IntBits = TLI->getIntSize();
543 if (!FT->getParamType(i: 1)->isIntegerTy(BitWidth: IntBits)) // memchr needs 'int'.
544 return nullptr;
545
546 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
547 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
548 return copyFlags(Old: *CI,
549 New: emitMemChr(Ptr: SrcStr, Val: CharVal, // include nul.
550 Len: ConstantInt::get(Ty: SizeTTy, V: Len), B,
551 DL, TLI));
552 }
553
554 if (CharC->isZero()) {
555 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
556 if (isOnlyUsedInEqualityComparison(V: CI, With: NullPtr))
557 // Pre-empt the transformation to strlen below and fold
558 // strchr(A, '\0') == null to false.
559 return B.CreateIntToPtr(V: B.getTrue(), DestTy: CI->getType());
560 }
561
562 // Otherwise, the character is a constant, see if the first argument is
563 // a string literal. If so, we can constant fold.
564 StringRef Str;
565 if (!getConstantStringInfo(V: SrcStr, Str)) {
566 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
567 if (Value *StrLen = emitStrLen(Ptr: SrcStr, B, DL, TLI))
568 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: StrLen, Name: "strchr");
569 return nullptr;
570 }
571
572 // Compute the offset, make sure to handle the case when we're searching for
573 // zero (a weird way to spell strlen).
574 size_t I = (0xFF & CharC->getSExtValue()) == 0
575 ? Str.size()
576 : Str.find(C: CharC->getSExtValue());
577 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
578 return Constant::getNullValue(Ty: CI->getType());
579
580 // strchr(s+n,c) -> gep(s+n+i,c)
581 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: I), Name: "strchr");
582}
583
584Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
585 Value *SrcStr = CI->getArgOperand(i: 0);
586 Value *CharVal = CI->getArgOperand(i: 1);
587 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
588 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
589
590 StringRef Str;
591 if (!getConstantStringInfo(V: SrcStr, Str)) {
592 // strrchr(s, 0) -> strchr(s, 0)
593 if (CharC && CharC->isZero())
594 return copyFlags(Old: *CI, New: emitStrChr(Ptr: SrcStr, C: '\0', B, TLI));
595 return nullptr;
596 }
597
598 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
599 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
600
601 // Try to expand strrchr to the memrchr nonstandard extension if it's
602 // available, or simply fail otherwise.
603 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
604 Value *Size = ConstantInt::get(Ty: SizeTTy, V: NBytes);
605 return copyFlags(Old: *CI, New: emitMemRChr(Ptr: SrcStr, Val: CharVal, Len: Size, B, DL, TLI));
606}
607
608Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
609 Value *Str1P = CI->getArgOperand(i: 0), *Str2P = CI->getArgOperand(i: 1);
610 if (Str1P == Str2P) // strcmp(x,x) -> 0
611 return ConstantInt::get(Ty: CI->getType(), V: 0);
612
613 StringRef Str1, Str2;
614 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
615 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
616
617 // strcmp(x, y) -> cnst (if both x and y are constant strings)
618 if (HasStr1 && HasStr2)
619 return ConstantInt::getSigned(Ty: CI->getType(),
620 V: std::clamp(val: Str1.compare(RHS: Str2), lo: -1, hi: 1));
621
622 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
623 return B.CreateNeg(V: B.CreateZExt(
624 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
625
626 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
627 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
628 DestTy: CI->getType());
629
630 // strcmp(P, "x") -> memcmp(P, "x", 2)
631 uint64_t Len1 = GetStringLength(V: Str1P);
632 if (Len1)
633 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
634 uint64_t Len2 = GetStringLength(V: Str2P);
635 if (Len2)
636 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
637
638 if (Len1 && Len2) {
639 return copyFlags(
640 Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
641 Len: TLI->getAsSizeT(V: std::min(a: Len1, b: Len2), M: *CI->getModule()),
642 B, DL, TLI));
643 }
644
645 // strcmp to memcmp
646 SimplifyQuery SQ(DL, TLI, DT, AC, CI);
647 if (!HasStr1 && HasStr2) {
648 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ))
649 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
650 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
651 B, DL, TLI));
652 } else if (HasStr1 && !HasStr2) {
653 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ))
654 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
655 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
656 B, DL, TLI));
657 }
658
659 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
660 return nullptr;
661}
662
663// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
664// arrays LHS and RHS and nonconstant Size.
665static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
666 Value *Size, bool StrNCmp,
667 IRBuilderBase &B, const DataLayout &DL);
668
669Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
670 Value *Str1P = CI->getArgOperand(i: 0);
671 Value *Str2P = CI->getArgOperand(i: 1);
672 Value *Size = CI->getArgOperand(i: 2);
673 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
674 return ConstantInt::get(Ty: CI->getType(), V: 0);
675
676 if (isKnownNonZero(V: Size, Q: DL))
677 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
678 // Get the length argument if it is constant.
679 uint64_t Length;
680 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size))
681 Length = LengthArg->getZExtValue();
682 else
683 return optimizeMemCmpVarSize(CI, LHS: Str1P, RHS: Str2P, Size, StrNCmp: true, B, DL);
684
685 if (Length == 0) // strncmp(x,y,0) -> 0
686 return ConstantInt::get(Ty: CI->getType(), V: 0);
687
688 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
689 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P, Len: Size, B, DL, TLI));
690
691 StringRef Str1, Str2;
692 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
693 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
694
695 // strncmp(x, y) -> cnst (if both x and y are constant strings)
696 if (HasStr1 && HasStr2) {
697 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
698 StringRef SubStr1 = substr(Str: Str1, Len: Length);
699 StringRef SubStr2 = substr(Str: Str2, Len: Length);
700 return ConstantInt::getSigned(Ty: CI->getType(),
701 V: std::clamp(val: SubStr1.compare(RHS: SubStr2), lo: -1, hi: 1));
702 }
703
704 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
705 return B.CreateNeg(V: B.CreateZExt(
706 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
707
708 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
709 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
710 DestTy: CI->getType());
711
712 uint64_t Len1 = GetStringLength(V: Str1P);
713 if (Len1)
714 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
715 uint64_t Len2 = GetStringLength(V: Str2P);
716 if (Len2)
717 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
718
719 // strncmp to memcmp
720 if (!HasStr1 && HasStr2) {
721 Len2 = std::min(a: Len2, b: Length);
722 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ: DL))
723 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
724 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
725 B, DL, TLI));
726 } else if (HasStr1 && !HasStr2) {
727 Len1 = std::min(a: Len1, b: Length);
728 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ: DL))
729 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
730 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
731 B, DL, TLI));
732 }
733
734 return nullptr;
735}
736
737Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
738 Value *Src = CI->getArgOperand(i: 0);
739 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
740 uint64_t SrcLen = GetStringLength(V: Src);
741 if (SrcLen && Size) {
742 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: SrcLen);
743 if (SrcLen <= Size->getZExtValue() + 1)
744 return copyFlags(Old: *CI, New: emitStrDup(Ptr: Src, B, TLI));
745 }
746
747 return nullptr;
748}
749
750Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
751 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
752 if (Dst == Src) // strcpy(x,x) -> x
753 return Src;
754
755 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
756 // See if we can get the length of the input string.
757 uint64_t Len = GetStringLength(V: Src);
758 if (Len)
759 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
760 else
761 return nullptr;
762
763 // We have enough information to now generate the memcpy call to do the
764 // copy for us. Make a memcpy to copy the nul byte with align = 1.
765 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
766 Size: TLI->getAsSizeT(V: Len, M: *CI->getModule()));
767 mergeAttributesAndFlags(NewCI, Old: *CI);
768 return Dst;
769}
770
771Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
772 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
773
774 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
775 if (CI->use_empty())
776 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
777
778 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
779 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
780 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
781 }
782
783 // See if we can get the length of the input string.
784 uint64_t Len = GetStringLength(V: Src);
785 if (Len)
786 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
787 else
788 return nullptr;
789
790 Value *LenV = TLI->getAsSizeT(V: Len, M: *CI->getModule());
791 Value *DstEnd = B.CreateInBoundsGEP(
792 Ty: B.getInt8Ty(), Ptr: Dst, IdxList: TLI->getAsSizeT(V: Len - 1, M: *CI->getModule()));
793
794 // We have enough information to now generate the memcpy call to do the
795 // copy for us. Make a memcpy to copy the nul byte with align = 1.
796 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: LenV);
797 mergeAttributesAndFlags(NewCI, Old: *CI);
798 return DstEnd;
799}
800
801// Optimize a call to size_t strlcpy(char*, const char*, size_t).
802
803Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
804 Value *Size = CI->getArgOperand(i: 2);
805 if (isKnownNonZero(V: Size, Q: DL))
806 // Like snprintf, the function stores into the destination only when
807 // the size argument is nonzero.
808 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
809 // The function reads the source argument regardless of Size (it returns
810 // its length).
811 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
812
813 uint64_t NBytes;
814 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
815 NBytes = SizeC->getZExtValue();
816 else
817 return nullptr;
818
819 Value *Dst = CI->getArgOperand(i: 0);
820 Value *Src = CI->getArgOperand(i: 1);
821 if (NBytes <= 1) {
822 if (NBytes == 1)
823 // For a call to strlcpy(D, S, 1) first store a nul in *D.
824 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
825
826 // Transform strlcpy(D, S, 0) to a call to strlen(S).
827 return copyFlags(Old: *CI, New: emitStrLen(Ptr: Src, B, DL, TLI));
828 }
829
830 // Try to determine the length of the source, substituting its size
831 // when it's not nul-terminated (as it's required to be) to avoid
832 // reading past its end.
833 StringRef Str;
834 if (!getConstantStringInfo(V: Src, Str, /*TrimAtNul=*/false))
835 return nullptr;
836
837 uint64_t SrcLen = Str.find(C: '\0');
838 // Set if the terminating nul should be copied by the call to memcpy
839 // below.
840 bool NulTerm = SrcLen < NBytes;
841
842 if (NulTerm)
843 // Overwrite NBytes with the number of bytes to copy, including
844 // the terminating nul.
845 NBytes = SrcLen + 1;
846 else {
847 // Set the length of the source for the function to return to its
848 // size, and cap NBytes at the same.
849 SrcLen = std::min(a: SrcLen, b: uint64_t(Str.size()));
850 NBytes = std::min(a: NBytes - 1, b: SrcLen);
851 }
852
853 if (SrcLen == 0) {
854 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
855 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
856 return ConstantInt::get(Ty: CI->getType(), V: 0);
857 }
858
859 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
860 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
861 // D[N' - 1] if necessary.
862 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
863 Size: TLI->getAsSizeT(V: NBytes, M: *CI->getModule()));
864 mergeAttributesAndFlags(NewCI, Old: *CI);
865
866 if (!NulTerm) {
867 Value *EndOff = ConstantInt::get(Ty: CI->getType(), V: NBytes);
868 Value *EndPtr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: EndOff);
869 B.CreateStore(Val: B.getInt8(C: 0), Ptr: EndPtr);
870 }
871
872 // Like snprintf, strlcpy returns the number of nonzero bytes that would
873 // have been copied if the bound had been sufficiently big (which in this
874 // case is strlen(Src)).
875 return ConstantInt::get(Ty: CI->getType(), V: SrcLen);
876}
877
878// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
879// otherwise.
880Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
881 IRBuilderBase &B) {
882 Value *Dst = CI->getArgOperand(i: 0);
883 Value *Src = CI->getArgOperand(i: 1);
884 Value *Size = CI->getArgOperand(i: 2);
885
886 if (isKnownNonZero(V: Size, Q: DL)) {
887 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
888 // only when N is nonzero.
889 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
890 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
891 }
892
893 // If the "bound" argument is known set N to it. Otherwise set it to
894 // UINT64_MAX and handle it later.
895 uint64_t N = UINT64_MAX;
896 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
897 N = SizeC->getZExtValue();
898
899 if (N == 0)
900 // Fold st{p,r}ncpy(D, S, 0) to D.
901 return Dst;
902
903 if (N == 1) {
904 Type *CharTy = B.getInt8Ty();
905 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "stxncpy.char0");
906 B.CreateStore(Val: CharVal, Ptr: Dst);
907 if (!RetEnd)
908 // Transform strncpy(D, S, 1) to return (*D = *S), D.
909 return Dst;
910
911 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
912 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
913 Value *Cmp = B.CreateICmpEQ(LHS: CharVal, RHS: ZeroChar, Name: "stpncpy.char0cmp");
914
915 Value *Off1 = B.getInt32(C: 1);
916 Value *EndPtr = B.CreateInBoundsGEP(Ty: CharTy, Ptr: Dst, IdxList: Off1, Name: "stpncpy.end");
917 return B.CreateSelect(C: Cmp, True: Dst, False: EndPtr, Name: "stpncpy.sel");
918 }
919
920 // If the length of the input string is known set SrcLen to it.
921 uint64_t SrcLen = GetStringLength(V: Src);
922 if (SrcLen)
923 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
924 else
925 return nullptr;
926
927 --SrcLen; // Unbias length.
928
929 if (SrcLen == 0) {
930 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
931 Align MemSetAlign =
932 CI->getAttributes().getParamAttrs(ArgNo: 0).getAlignment().valueOrOne();
933 CallInst *NewCI = B.CreateMemSet(Ptr: Dst, Val: B.getInt8(C: '\0'), Size, Align: MemSetAlign);
934 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(ArgNo: 0));
935 NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
936 C&: CI->getContext(), ArgNo: 0, B: ArgAttrs));
937 copyFlags(Old: *CI, New: NewCI);
938 return Dst;
939 }
940
941 if (N > SrcLen + 1) {
942 if (N > 128)
943 // Bail if N is large or unknown.
944 return nullptr;
945
946 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
947 StringRef Str;
948 if (!getConstantStringInfo(V: Src, Str))
949 return nullptr;
950 std::string SrcStr = Str.str();
951 // Create a bigger, nul-padded array with the same length, SrcLen,
952 // as the original string.
953 SrcStr.resize(n: N, c: '\0');
954 Src = B.CreateGlobalString(Str: SrcStr, Name: "str", /*AddressSpace=*/0,
955 /*M=*/nullptr, /*AddNull=*/false);
956 }
957
958 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
959 // S and N are constant.
960 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
961 Size: TLI->getAsSizeT(V: N, M: *CI->getModule()));
962 mergeAttributesAndFlags(NewCI, Old: *CI);
963 if (!RetEnd)
964 return Dst;
965
966 // stpncpy(D, S, N) returns the address of the first null in D if it writes
967 // one, otherwise D + N.
968 Value *Off = B.getInt64(C: std::min(a: SrcLen, b: N));
969 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: Off, Name: "endptr");
970}
971
972Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
973 unsigned CharSize,
974 Value *Bound) {
975 Value *Src = CI->getArgOperand(i: 0);
976 Type *CharTy = B.getIntNTy(N: CharSize);
977
978 if (isOnlyUsedInZeroEqualityComparison(CtxI: CI) &&
979 (!Bound || isKnownNonZero(V: Bound, Q: DL))) {
980 // Fold strlen:
981 // strlen(x) != 0 --> *x != 0
982 // strlen(x) == 0 --> *x == 0
983 // and likewise strnlen with constant N > 0:
984 // strnlen(x, N) != 0 --> *x != 0
985 // strnlen(x, N) == 0 --> *x == 0
986 return B.CreateZExt(V: B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "char0"),
987 DestTy: CI->getType());
988 }
989
990 if (Bound) {
991 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Val: Bound)) {
992 if (BoundCst->isZero())
993 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
994 return ConstantInt::get(Ty: CI->getType(), V: 0);
995
996 if (BoundCst->isOne()) {
997 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
998 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "strnlen.char0");
999 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
1000 Value *Cmp = B.CreateICmpNE(LHS: CharVal, RHS: ZeroChar, Name: "strnlen.char0cmp");
1001 return B.CreateZExt(V: Cmp, DestTy: CI->getType());
1002 }
1003 }
1004 }
1005
1006 if (uint64_t Len = GetStringLength(V: Src, CharSize)) {
1007 Value *LenC = ConstantInt::get(Ty: CI->getType(), V: Len - 1);
1008 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
1009 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
1010 if (Bound)
1011 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: LenC, RHS: Bound);
1012 return LenC;
1013 }
1014
1015 if (Bound)
1016 // Punt for strnlen for now.
1017 return nullptr;
1018
1019 // If s is a constant pointer pointing to a string literal, we can fold
1020 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
1021 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
1022 // We only try to simplify strlen when the pointer s points to an array
1023 // of CharSize elements. Otherwise, we would need to scale the offset x before
1024 // doing the subtraction. This will make the optimization more complex, and
1025 // it's not very useful because calling strlen for a pointer of other types is
1026 // very uncommon.
1027 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: Src)) {
1028 unsigned BW = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1029 SmallMapVector<Value *, APInt, 4> VarOffsets;
1030 APInt ConstOffset(BW, 0);
1031 assert(CharSize % 8 == 0 && "Expected a multiple of 8 sized CharSize");
1032 // Check the gep is a single variable offset.
1033 if (!GEP->collectOffset(DL, BitWidth: BW, VariableOffsets&: VarOffsets, ConstantOffset&: ConstOffset) ||
1034 VarOffsets.size() != 1 || ConstOffset != 0 ||
1035 VarOffsets.begin()->second != CharSize / 8)
1036 return nullptr;
1037
1038 ConstantDataArraySlice Slice;
1039 if (getConstantDataArrayInfo(V: GEP->getOperand(i_nocapture: 0), Slice, ElementSize: CharSize)) {
1040 uint64_t NullTermIdx;
1041 if (Slice.Array == nullptr) {
1042 NullTermIdx = 0;
1043 } else {
1044 NullTermIdx = ~((uint64_t)0);
1045 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
1046 if (Slice.Array->getElementAsInteger(i: I + Slice.Offset) == 0) {
1047 NullTermIdx = I;
1048 break;
1049 }
1050 }
1051 // If the string does not have '\0', leave it to strlen to compute
1052 // its length.
1053 if (NullTermIdx == ~((uint64_t)0))
1054 return nullptr;
1055 }
1056
1057 Value *Offset = VarOffsets.begin()->first;
1058 KnownBits Known = computeKnownBits(V: Offset, DL, AC: nullptr, CtxI: CI, DT: nullptr);
1059
1060 // If Offset is not provably in the range [0, NullTermIdx], we can still
1061 // optimize if we can prove that the program has undefined behavior when
1062 // Offset is outside that range. That is the case when GEP->getOperand(0)
1063 // is a pointer to an object whose memory extent is NullTermIdx+1.
1064 if ((Known.isNonNegative() && Known.getMaxValue().ule(RHS: NullTermIdx)) ||
1065 (isa<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)) &&
1066 NullTermIdx == Slice.Length - 1)) {
1067 Offset = B.CreateSExtOrTrunc(V: Offset, DestTy: CI->getType());
1068 return B.CreateSub(LHS: ConstantInt::get(Ty: CI->getType(), V: NullTermIdx),
1069 RHS: Offset);
1070 }
1071 }
1072 }
1073
1074 // strlen(x?"foo":"bars") --> x ? 3 : 4
1075 if (SelectInst *SI = dyn_cast<SelectInst>(Val: Src)) {
1076 uint64_t LenTrue = GetStringLength(V: SI->getTrueValue(), CharSize);
1077 uint64_t LenFalse = GetStringLength(V: SI->getFalseValue(), CharSize);
1078 if (LenTrue && LenFalse) {
1079 ORE.emit(RemarkBuilder: [&]() {
1080 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1081 << "folded strlen(select) to select of constants";
1082 });
1083 return B.CreateSelect(C: SI->getCondition(),
1084 True: ConstantInt::get(Ty: CI->getType(), V: LenTrue - 1),
1085 False: ConstantInt::get(Ty: CI->getType(), V: LenFalse - 1), Name: "",
1086 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : SI);
1087 }
1088 }
1089
1090 return nullptr;
1091}
1092
1093Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1094 if (Value *V = optimizeStringLength(CI, B, CharSize: 8))
1095 return V;
1096 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1097 return nullptr;
1098}
1099
1100Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1101 Value *Bound = CI->getArgOperand(i: 1);
1102 if (Value *V = optimizeStringLength(CI, B, CharSize: 8, Bound))
1103 return V;
1104
1105 if (isKnownNonZero(V: Bound, Q: DL))
1106 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1107 return nullptr;
1108}
1109
1110Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1111 Module &M = *CI->getModule();
1112 unsigned WCharSize = TLI->getWCharSize(M) * 8;
1113 // We cannot perform this optimization without wchar_size metadata.
1114 if (WCharSize == 0)
1115 return nullptr;
1116
1117 return optimizeStringLength(CI, B, CharSize: WCharSize);
1118}
1119
1120Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1121 StringRef S1, S2;
1122 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1123 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1124
1125 // strpbrk(s, "") -> nullptr
1126 // strpbrk("", s) -> nullptr
1127 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1128 return Constant::getNullValue(Ty: CI->getType());
1129
1130 // Constant folding.
1131 if (HasS1 && HasS2) {
1132 size_t I = S1.find_first_of(Chars: S2);
1133 if (I == StringRef::npos) // No match.
1134 return Constant::getNullValue(Ty: CI->getType());
1135
1136 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1137 IdxList: B.getInt64(C: I), Name: "strpbrk");
1138 }
1139
1140 // strpbrk(s, "a") -> strchr(s, 'a')
1141 if (HasS2 && S2.size() == 1)
1142 return copyFlags(Old: *CI, New: emitStrChr(Ptr: CI->getArgOperand(i: 0), C: S2[0], B, TLI));
1143
1144 return nullptr;
1145}
1146
1147Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1148 Value *EndPtr = CI->getArgOperand(i: 1);
1149 if (isa<ConstantPointerNull>(Val: EndPtr)) {
1150 // With a null EndPtr, this function won't capture the main argument.
1151 // It would be readonly too, except that it still may write to errno.
1152 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
1153 CI: CaptureInfo::none()));
1154 }
1155
1156 return nullptr;
1157}
1158
1159Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1160 StringRef S1, S2;
1161 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1162 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1163
1164 // strspn(s, "") -> 0
1165 // strspn("", s) -> 0
1166 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1167 return Constant::getNullValue(Ty: CI->getType());
1168
1169 // Constant folding.
1170 if (HasS1 && HasS2) {
1171 size_t Pos = S1.find_first_not_of(Chars: S2);
1172 if (Pos == StringRef::npos)
1173 Pos = S1.size();
1174 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1175 }
1176
1177 return nullptr;
1178}
1179
1180Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1181 StringRef S1, S2;
1182 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1183 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1184
1185 // strcspn("", s) -> 0
1186 if (HasS1 && S1.empty())
1187 return Constant::getNullValue(Ty: CI->getType());
1188
1189 // Constant folding.
1190 if (HasS1 && HasS2) {
1191 size_t Pos = S1.find_first_of(Chars: S2);
1192 if (Pos == StringRef::npos)
1193 Pos = S1.size();
1194 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1195 }
1196
1197 // strcspn(s, "") -> strlen(s)
1198 if (HasS2 && S2.empty())
1199 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B, DL, TLI));
1200
1201 return nullptr;
1202}
1203
1204Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1205 // fold strstr(x, x) -> x.
1206 if (CI->getArgOperand(i: 0) == CI->getArgOperand(i: 1))
1207 return CI->getArgOperand(i: 0);
1208
1209 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1210 if (isOnlyUsedInEqualityComparison(V: CI, With: CI->getArgOperand(i: 0))) {
1211 Value *StrLen = emitStrLen(Ptr: CI->getArgOperand(i: 1), B, DL, TLI);
1212 if (!StrLen)
1213 return nullptr;
1214 Value *StrNCmp = emitStrNCmp(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
1215 Len: StrLen, B, DL, TLI);
1216 if (!StrNCmp)
1217 return nullptr;
1218 for (User *U : llvm::make_early_inc_range(Range: CI->users())) {
1219 ICmpInst *Old = cast<ICmpInst>(Val: U);
1220 Value *Cmp =
1221 B.CreateICmp(P: Old->getPredicate(), LHS: StrNCmp,
1222 RHS: ConstantInt::getNullValue(Ty: StrNCmp->getType()), Name: "cmp");
1223 replaceAllUsesWith(I: Old, With: Cmp);
1224 }
1225 return CI;
1226 }
1227
1228 // See if either input string is a constant string.
1229 StringRef SearchStr, ToFindStr;
1230 bool HasStr1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: SearchStr);
1231 bool HasStr2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: ToFindStr);
1232
1233 // fold strstr(x, "") -> x.
1234 if (HasStr2 && ToFindStr.empty())
1235 return CI->getArgOperand(i: 0);
1236
1237 // If both strings are known, constant fold it.
1238 if (HasStr1 && HasStr2) {
1239 size_t Offset = SearchStr.find(Str: ToFindStr);
1240
1241 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1242 return Constant::getNullValue(Ty: CI->getType());
1243
1244 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1245 return B.CreateConstInBoundsGEP1_64(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1246 Idx0: Offset, Name: "strstr");
1247 }
1248
1249 // fold strstr(x, "y") -> strchr(x, 'y').
1250 if (HasStr2 && ToFindStr.size() == 1) {
1251 return emitStrChr(Ptr: CI->getArgOperand(i: 0), C: ToFindStr[0], B, TLI);
1252 }
1253
1254 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
1255 return nullptr;
1256}
1257
1258Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1259 Value *SrcStr = CI->getArgOperand(i: 0);
1260 Value *Size = CI->getArgOperand(i: 2);
1261 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1262 Value *CharVal = CI->getArgOperand(i: 1);
1263 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1264 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1265
1266 if (LenC) {
1267 if (LenC->isZero())
1268 // Fold memrchr(x, y, 0) --> null.
1269 return NullPtr;
1270
1271 if (LenC->isOne()) {
1272 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1273 // constant or otherwise.
1274 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memrchr.char0");
1275 // Slice off the character's high end bits.
1276 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1277 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memrchr.char0cmp");
1278 return B.CreateSelectWithUnknownProfile(C: Cmp, True: SrcStr, False: NullPtr, DEBUG_TYPE,
1279 Name: "memrchr.sel");
1280 }
1281 }
1282
1283 StringRef Str;
1284 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1285 return nullptr;
1286
1287 if (Str.size() == 0)
1288 // If the array is empty fold memrchr(A, C, N) to null for any value
1289 // of C and N on the basis that the only valid value of N is zero
1290 // (otherwise the call is undefined).
1291 return NullPtr;
1292
1293 uint64_t EndOff = UINT64_MAX;
1294 if (LenC) {
1295 EndOff = LenC->getZExtValue();
1296 if (Str.size() < EndOff)
1297 // Punt out-of-bounds accesses to sanitizers and/or libc.
1298 return nullptr;
1299 }
1300
1301 if (ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal)) {
1302 // Fold memrchr(S, C, N) for a constant C.
1303 size_t Pos = Str.rfind(C: CharC->getZExtValue(), From: EndOff);
1304 if (Pos == StringRef::npos)
1305 // When the character is not in the source array fold the result
1306 // to null regardless of Size.
1307 return NullPtr;
1308
1309 if (LenC)
1310 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1311 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos));
1312
1313 if (Str.find(C: Str[Pos]) == Pos) {
1314 // When there is just a single occurrence of C in S, i.e., the one
1315 // in Str[Pos], fold
1316 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1317 // for nonconstant N.
1318 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1319 Name: "memrchr.cmp");
1320 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr,
1321 IdxList: B.getInt64(C: Pos), Name: "memrchr.ptr_plus");
1322 return B.CreateSelectWithUnknownProfile(C: Cmp, True: NullPtr, False: SrcPlus, DEBUG_TYPE,
1323 Name: "memrchr.sel");
1324 }
1325 }
1326
1327 // Truncate the string to search at most EndOff characters.
1328 Str = Str.substr(Start: 0, N: EndOff);
1329 if (Str.find_first_not_of(C: Str[0]) != StringRef::npos)
1330 return nullptr;
1331
1332 // If the source array consists of all equal characters, then for any
1333 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1334 // N != 0 && *S == C ? S + N - 1 : null
1335 Type *SizeTy = Size->getType();
1336 Type *Int8Ty = B.getInt8Ty();
1337 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1338 // Slice off the sought character's high end bits.
1339 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1340 Value *CEqS0 =
1341 B.CreateICmpEQ(LHS: ConstantInt::get(Ty: Int8Ty, V: (unsigned char)Str[0]), RHS: CharVal);
1342 Value *And = B.CreateLogicalAnd(Cond1: NNeZ, Cond2: CEqS0);
1343 if (auto *AndSI = dyn_cast<SelectInst>(Val: And))
1344 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *AndSI, DEBUG_TYPE);
1345 Value *SizeM1 = B.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
1346 Value *SrcPlus =
1347 B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: SrcStr, IdxList: SizeM1, Name: "memrchr.ptr_plus");
1348 return B.CreateSelectWithUnknownProfile(C: And, True: SrcPlus, False: NullPtr, DEBUG_TYPE,
1349 Name: "memrchr.sel");
1350}
1351
1352Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1353 Value *SrcStr = CI->getArgOperand(i: 0);
1354 Value *Size = CI->getArgOperand(i: 2);
1355
1356 if (isKnownNonZero(V: Size, Q: DL)) {
1357 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1358 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1359 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1360 }
1361
1362 Value *CharVal = CI->getArgOperand(i: 1);
1363 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
1364 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1365 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1366
1367 // memchr(x, y, 0) -> null
1368 if (LenC) {
1369 if (LenC->isZero())
1370 return NullPtr;
1371
1372 if (LenC->isOne()) {
1373 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1374 // constant or otherwise.
1375 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memchr.char0");
1376 // Slice off the character's high end bits.
1377 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1378 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memchr.char0cmp");
1379 // The condition depends on the value of the string being equal to the
1380 // query, neither of which we know without value profiling, so mark the
1381 // profile unknown.
1382 return B.CreateSelectWithUnknownProfile(C: Cmp, True: SrcStr, False: NullPtr, DEBUG_TYPE,
1383 Name: "memchr.sel");
1384 }
1385 }
1386
1387 StringRef Str;
1388 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1389 return nullptr;
1390
1391 if (CharC) {
1392 size_t Pos = Str.find(C: CharC->getZExtValue());
1393 if (Pos == StringRef::npos)
1394 // When the character is not in the source array fold the result
1395 // to null regardless of Size.
1396 return NullPtr;
1397
1398 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1399 // When the constant Size is less than or equal to the character
1400 // position also fold the result to null.
1401 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1402 Name: "memchr.cmp");
1403 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos),
1404 Name: "memchr.ptr");
1405 // The condition is dependent upon the value of n, which we cannot infer
1406 // without value profiling, so mark the profile unknown.
1407 return B.CreateSelectWithUnknownProfile(C: Cmp, True: NullPtr, False: SrcPlus, DEBUG_TYPE);
1408 }
1409
1410 if (Str.size() == 0)
1411 // If the array is empty fold memchr(A, C, N) to null for any value
1412 // of C and N on the basis that the only valid value of N is zero
1413 // (otherwise the call is undefined).
1414 return NullPtr;
1415
1416 if (LenC)
1417 Str = substr(Str, Len: LenC->getZExtValue());
1418
1419 size_t Pos = Str.find_first_not_of(C: Str[0]);
1420 if (Pos == StringRef::npos
1421 || Str.find_first_not_of(C: Str[Pos], From: Pos) == StringRef::npos) {
1422 // If the source array consists of at most two consecutive sequences
1423 // of the same characters, then for any C and N (whether in bounds or
1424 // not), fold memchr(S, C, N) to
1425 // N != 0 && *S == C ? S : null
1426 // or for the two sequences to:
1427 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1428 // ^Sel2 ^Sel1 are denoted above.
1429 // The latter makes it also possible to fold strchr() calls with strings
1430 // of the same characters.
1431 Type *SizeTy = Size->getType();
1432 Type *Int8Ty = B.getInt8Ty();
1433
1434 // Slice off the sought character's high end bits.
1435 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1436
1437 Value *Sel1 = NullPtr;
1438 if (Pos != StringRef::npos) {
1439 // Handle two consecutive sequences of the same characters.
1440 Value *PosVal = ConstantInt::get(Ty: SizeTy, V: Pos);
1441 Value *StrPos = ConstantInt::get(Ty: Int8Ty, V: (unsigned char)Str[Pos]);
1442 Value *CEqSPos = B.CreateICmpEQ(LHS: CharVal, RHS: StrPos);
1443 Value *NGtPos = B.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: Size, RHS: PosVal);
1444 Value *And = B.CreateAnd(LHS: CEqSPos, RHS: NGtPos);
1445 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: PosVal);
1446 // The condition depends on the value of the query and size, neither of
1447 // which we know without value profiling, so mark the profile unknown.
1448 Sel1 = B.CreateSelectWithUnknownProfile(C: And, True: SrcPlus, False: NullPtr, DEBUG_TYPE,
1449 Name: "memchr.sel1");
1450 }
1451
1452 Value *Str0 = ConstantInt::get(Ty: Int8Ty, V: (unsigned char)Str[0]);
1453 Value *CEqS0 = B.CreateICmpEQ(LHS: Str0, RHS: CharVal);
1454 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1455 Value *And = B.CreateAnd(LHS: NNeZ, RHS: CEqS0);
1456 // The condition depends on the value of the query and size, neither of
1457 // which we know without value profiling, so mark the profile unknown.
1458 return B.CreateSelectWithUnknownProfile(C: And, True: SrcStr, False: Sel1, DEBUG_TYPE,
1459 Name: "memchr.sel2");
1460 }
1461
1462 if (!LenC) {
1463 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1464 // S is dereferenceable so it's safe to load from it and fold
1465 // memchr(S, C, N) == S to N && *S == C for any C and N.
1466 // TODO: This is safe even for nonconstant S.
1467 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1468
1469 // From now on we need a constant length and constant array.
1470 return nullptr;
1471 }
1472
1473 bool OptForSize = llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
1474 QueryType: PGSOQueryType::IRPass);
1475
1476 // If the char is variable but the input str and length are not we can turn
1477 // this memchr call into a simple bit field test. Of course this only works
1478 // when the return value is only checked against null.
1479 //
1480 // It would be really nice to reuse switch lowering here but we can't change
1481 // the CFG at this point.
1482 //
1483 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1484 // != 0
1485 // after bounds check.
1486 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CtxI: CI))
1487 return nullptr;
1488
1489 unsigned char Max =
1490 *std::max_element(first: reinterpret_cast<const unsigned char *>(Str.begin()),
1491 last: reinterpret_cast<const unsigned char *>(Str.end()));
1492
1493 // Make sure the bit field we're about to create fits in a register on the
1494 // target.
1495 // FIXME: On a 64 bit architecture this prevents us from using the
1496 // interesting range of alpha ascii chars. We could do better by emitting
1497 // two bitfields or shifting the range by 64 if no lower chars are used.
1498 if (!DL.fitsInLegalInteger(Width: Max + 1)) {
1499 // Build chain of ORs
1500 // Transform:
1501 // memchr("abcd", C, 4) != nullptr
1502 // to:
1503 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1504 std::string SortedStr = Str.str();
1505 llvm::sort(C&: SortedStr);
1506 // Compute the number of of non-contiguous ranges.
1507 unsigned NonContRanges = 1;
1508 for (size_t i = 1; i < SortedStr.size(); ++i) {
1509 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1510 NonContRanges++;
1511 }
1512 }
1513
1514 // Restrict this optimization to profitable cases with one or two range
1515 // checks.
1516 if (NonContRanges > 2)
1517 return nullptr;
1518
1519 // Slice off the character's high end bits.
1520 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1521
1522 SmallVector<Value *> CharCompares;
1523 for (unsigned char C : SortedStr)
1524 CharCompares.push_back(Elt: B.CreateICmpEQ(LHS: CharVal, RHS: B.getInt8(C)));
1525
1526 return B.CreateIntToPtr(V: B.CreateOr(Ops: CharCompares), DestTy: CI->getType());
1527 }
1528
1529 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1530 // creating unnecessary illegal types.
1531 unsigned char Width = NextPowerOf2(A: std::max(a: (unsigned char)7, b: Max));
1532
1533 // Now build the bit field.
1534 APInt Bitfield(Width, 0);
1535 for (char C : Str)
1536 Bitfield.setBit((unsigned char)C);
1537 Value *BitfieldC = B.getInt(AI: Bitfield);
1538
1539 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1540 Value *C = B.CreateZExtOrTrunc(V: CharVal, DestTy: BitfieldC->getType());
1541 C = B.CreateAnd(LHS: C, RHS: B.getIntN(N: Width, C: 0xFF));
1542
1543 // First check that the bit field access is within bounds.
1544 Value *Bounds = B.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: C, RHS: B.getIntN(N: Width, C: Width),
1545 Name: "memchr.bounds");
1546
1547 // Create code that checks if the given bit is set in the field.
1548 Value *Shl = B.CreateShl(LHS: B.getIntN(N: Width, C: 1ULL), RHS: C);
1549 Value *Bits = B.CreateIsNotNull(Arg: B.CreateAnd(LHS: Shl, RHS: BitfieldC), Name: "memchr.bits");
1550
1551 // Finally merge both checks and cast to pointer type. The inttoptr
1552 // implicitly zexts the i1 to intptr type.
1553 Value *Memchr = B.CreateLogicalAnd(Cond1: Bounds, Cond2: Bits, Name: "memchr");
1554 // We construct an and between the value of the memory and the bytes to search
1555 // for. We cannot infer how often this would be true without value profiling
1556 // for the query, so mark the profile unknown.
1557 if (auto *SI = dyn_cast<SelectInst>(Val: Memchr))
1558 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE);
1559 return B.CreateIntToPtr(V: Memchr, DestTy: CI->getType());
1560}
1561
1562// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1563// arrays LHS and RHS and nonconstant Size.
1564static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
1565 Value *Size, bool StrNCmp,
1566 IRBuilderBase &B, const DataLayout &DL) {
1567 if (LHS == RHS) // memcmp(s,s,x) -> 0
1568 return Constant::getNullValue(Ty: CI->getType());
1569
1570 StringRef LStr, RStr;
1571 if (!getConstantStringInfo(V: LHS, Str&: LStr, /*TrimAtNul=*/false) ||
1572 !getConstantStringInfo(V: RHS, Str&: RStr, /*TrimAtNul=*/false))
1573 return nullptr;
1574
1575 // If the contents of both constant arrays are known, fold a call to
1576 // memcmp(A, B, N) to
1577 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1578 // where Pos is the first mismatch between A and B, determined below.
1579
1580 uint64_t Pos = 0;
1581 Value *Zero = ConstantInt::get(Ty: CI->getType(), V: 0);
1582 for (uint64_t MinSize = std::min(a: LStr.size(), b: RStr.size()); ; ++Pos) {
1583 if (Pos == MinSize ||
1584 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1585 // One array is a leading part of the other of equal or greater
1586 // size, or for strncmp, the arrays are equal strings.
1587 // Fold the result to zero. Size is assumed to be in bounds, since
1588 // otherwise the call would be undefined.
1589 return Zero;
1590 }
1591
1592 if (LStr[Pos] != RStr[Pos])
1593 break;
1594 }
1595
1596 // Normalize the result.
1597 typedef unsigned char UChar;
1598 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1599 Value *MaxSize = ConstantInt::get(Ty: Size->getType(), V: Pos);
1600 Value *Cmp = B.CreateICmp(P: ICmpInst::ICMP_ULE, LHS: Size, RHS: MaxSize);
1601 Value *Res = ConstantInt::getSigned(Ty: CI->getType(), V: IRes);
1602 return B.CreateSelect(C: Cmp, True: Zero, False: Res);
1603}
1604
1605// Optimize a memcmp call CI with constant size Len.
1606static Value *optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS,
1607 uint64_t Len, IRBuilderBase &B,
1608 const DataLayout &DL) {
1609 if (Len == 0) // memcmp(s1,s2,0) -> 0
1610 return Constant::getNullValue(Ty: CI->getType());
1611
1612 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1613 if (Len == 1) {
1614 Value *LHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: LHS, Name: "lhsc"),
1615 DestTy: CI->getType(), Name: "lhsv");
1616 Value *RHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: RHS, Name: "rhsc"),
1617 DestTy: CI->getType(), Name: "rhsv");
1618 return B.CreateSub(LHS: LHSV, RHS: RHSV, Name: "chardiff");
1619 }
1620
1621 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1622 // TODO: The case where both inputs are constants does not need to be limited
1623 // to legal integers or equality comparison. See block below this.
1624 if (DL.isLegalInteger(Width: Len * 8) && isOnlyUsedInZeroEqualityComparison(CtxI: CI)) {
1625 IntegerType *IntType = IntegerType::get(C&: CI->getContext(), NumBits: Len * 8);
1626 Align PrefAlignment = DL.getPrefTypeAlign(Ty: IntType);
1627
1628 // First, see if we can fold either argument to a constant.
1629 Value *LHSV = nullptr;
1630 if (auto *LHSC = dyn_cast<Constant>(Val: LHS))
1631 LHSV = ConstantFoldLoadFromConstPtr(C: LHSC, Ty: IntType, DL);
1632
1633 Value *RHSV = nullptr;
1634 if (auto *RHSC = dyn_cast<Constant>(Val: RHS))
1635 RHSV = ConstantFoldLoadFromConstPtr(C: RHSC, Ty: IntType, DL);
1636
1637 // Don't generate unaligned loads. If either source is constant data,
1638 // alignment doesn't matter for that source because there is no load.
1639 if ((LHSV || getKnownAlignment(V: LHS, DL, CtxI: CI) >= PrefAlignment) &&
1640 (RHSV || getKnownAlignment(V: RHS, DL, CtxI: CI) >= PrefAlignment)) {
1641 if (!LHSV)
1642 LHSV = B.CreateLoad(Ty: IntType, Ptr: LHS, Name: "lhsv");
1643 if (!RHSV)
1644 RHSV = B.CreateLoad(Ty: IntType, Ptr: RHS, Name: "rhsv");
1645 return B.CreateZExt(V: B.CreateICmpNE(LHS: LHSV, RHS: RHSV), DestTy: CI->getType(), Name: "memcmp");
1646 }
1647 }
1648
1649 return nullptr;
1650}
1651
1652// Most simplifications for memcmp also apply to bcmp.
1653Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1654 IRBuilderBase &B) {
1655 Value *LHS = CI->getArgOperand(i: 0), *RHS = CI->getArgOperand(i: 1);
1656 Value *Size = CI->getArgOperand(i: 2);
1657
1658 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1659
1660 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, StrNCmp: false, B, DL))
1661 return Res;
1662
1663 // Handle constant Size.
1664 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1665 if (!LenC)
1666 return nullptr;
1667
1668 return optimizeMemCmpConstantSize(CI, LHS, RHS, Len: LenC->getZExtValue(), B, DL);
1669}
1670
1671Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1672 Module *M = CI->getModule();
1673 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1674 return V;
1675
1676 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1677 // bcmp can be more efficient than memcmp because it only has to know that
1678 // there is a difference, not how different one is to the other.
1679 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_bcmp) &&
1680 isOnlyUsedInZeroEqualityComparison(CtxI: CI)) {
1681 Value *LHS = CI->getArgOperand(i: 0);
1682 Value *RHS = CI->getArgOperand(i: 1);
1683 Value *Size = CI->getArgOperand(i: 2);
1684 return copyFlags(Old: *CI, New: emitBCmp(Ptr1: LHS, Ptr2: RHS, Len: Size, B, DL, TLI));
1685 }
1686
1687 return nullptr;
1688}
1689
1690Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1691 return optimizeMemCmpBCmpCommon(CI, B);
1692}
1693
1694Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1695 Value *Size = CI->getArgOperand(i: 2);
1696 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1697 if (isa<IntrinsicInst>(Val: CI))
1698 return nullptr;
1699
1700 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1701 CallInst *NewCI = B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1702 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1703 mergeAttributesAndFlags(NewCI, Old: *CI);
1704 return CI->getArgOperand(i: 0);
1705}
1706
1707Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1708 Value *Dst = CI->getArgOperand(i: 0);
1709 Value *Src = CI->getArgOperand(i: 1);
1710 ConstantInt *StopChar = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
1711 ConstantInt *N = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
1712 StringRef SrcStr;
1713 if (CI->use_empty() && Dst == Src)
1714 return Dst;
1715 // memccpy(d, s, c, 0) -> nullptr
1716 if (N) {
1717 if (N->isNullValue())
1718 return Constant::getNullValue(Ty: CI->getType());
1719 if (!getConstantStringInfo(V: Src, Str&: SrcStr, /*TrimAtNul=*/false) ||
1720 // TODO: Handle zeroinitializer.
1721 !StopChar)
1722 return nullptr;
1723 } else {
1724 return nullptr;
1725 }
1726
1727 // Wrap arg 'c' of type int to char
1728 size_t Pos = SrcStr.find(C: StopChar->getSExtValue() & 0xFF);
1729 if (Pos == StringRef::npos) {
1730 if (N->getZExtValue() <= SrcStr.size()) {
1731 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
1732 Size: CI->getArgOperand(i: 3)));
1733 return Constant::getNullValue(Ty: CI->getType());
1734 }
1735 return nullptr;
1736 }
1737
1738 Value *NewN =
1739 ConstantInt::get(Ty: N->getType(), V: std::min(a: uint64_t(Pos + 1), b: N->getZExtValue()));
1740 // memccpy -> llvm.memcpy
1741 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: NewN));
1742 return Pos + 1 <= N->getZExtValue()
1743 ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: NewN)
1744 : Constant::getNullValue(Ty: CI->getType());
1745}
1746
1747Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1748 Value *Dst = CI->getArgOperand(i: 0);
1749 Value *N = CI->getArgOperand(i: 2);
1750 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1751 CallInst *NewCI =
1752 B.CreateMemCpy(Dst, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size: N);
1753 // Propagate attributes, but memcpy has no return value, so make sure that
1754 // any return attributes are compliant.
1755 // TODO: Attach return value attributes to the 1st operand to preserve them?
1756 mergeAttributesAndFlags(NewCI, Old: *CI);
1757 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: N);
1758}
1759
1760Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1761 Value *Size = CI->getArgOperand(i: 2);
1762 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1763 if (isa<IntrinsicInst>(Val: CI))
1764 return nullptr;
1765
1766 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1767 CallInst *NewCI = B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1768 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1769 mergeAttributesAndFlags(NewCI, Old: *CI);
1770 return CI->getArgOperand(i: 0);
1771}
1772
1773Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1774 Value *Size = CI->getArgOperand(i: 2);
1775 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1776 if (isa<IntrinsicInst>(Val: CI))
1777 return nullptr;
1778
1779 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1780 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
1781 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val, Size, Align: Align(1));
1782 mergeAttributesAndFlags(NewCI, Old: *CI);
1783 return CI->getArgOperand(i: 0);
1784}
1785
1786Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1787 if (isa<ConstantPointerNull>(Val: CI->getArgOperand(i: 0))) {
1788 Value *Malloc = emitMalloc(Num: CI->getArgOperand(i: 1), B, DL, TLI);
1789 if (auto *MallocCI = dyn_cast_or_null<CallInst>(Val: Malloc))
1790 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_alloc_token))
1791 MallocCI->setMetadata(KindID: LLVMContext::MD_alloc_token, Node: MD);
1792 return copyFlags(Old: *CI, New: Malloc);
1793 }
1794
1795 return nullptr;
1796}
1797
1798// Optionally allow optimization of nobuiltin calls to operator new and its
1799// variants.
1800Value *LibCallSimplifier::maybeOptimizeNoBuiltinOperatorNew(CallInst *CI,
1801 IRBuilderBase &B) {
1802 if (!OptimizeHotColdNew)
1803 return nullptr;
1804 Function *Callee = CI->getCalledFunction();
1805 if (!Callee)
1806 return nullptr;
1807 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
1808 if (Func == NotLibFunc)
1809 return nullptr;
1810 switch (Func) {
1811 case LibFunc_Znwm:
1812 case LibFunc_ZnwmRKSt9nothrow_t:
1813 case LibFunc_ZnwmSt11align_val_t:
1814 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1815 case LibFunc_Znam:
1816 case LibFunc_ZnamRKSt9nothrow_t:
1817 case LibFunc_ZnamSt11align_val_t:
1818 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1819 case LibFunc_size_returning_new:
1820 case LibFunc_size_returning_new_aligned:
1821 // By default normal operator new calls (not already passing a hot_cold_t
1822 // parameter) are not mutated if the call is not marked builtin. Optionally
1823 // enable that in cases where it is known to be safe.
1824 if (!OptimizeNoBuiltinHotColdNew)
1825 return nullptr;
1826 break;
1827 case LibFunc_Znwm12__hot_cold_t:
1828 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1829 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1830 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1831 case LibFunc_Znam12__hot_cold_t:
1832 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1833 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1834 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1835 case LibFunc_size_returning_new_hot_cold:
1836 case LibFunc_size_returning_new_aligned_hot_cold:
1837 // If the nobuiltin call already passes a hot_cold_t parameter, allow update
1838 // of that parameter when enabled.
1839 if (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::None)
1840 return nullptr;
1841 break;
1842 default:
1843 return nullptr;
1844 }
1845 return optimizeNew(CI, B, Func);
1846}
1847
1848// When enabled, replace operator new() calls marked with a hot or cold memprof
1849// attribute with an operator new() call that takes a __hot_cold_t parameter.
1850// Currently this is supported by the open source version of tcmalloc, see:
1851// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1852Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1853 LibFunc &Func) {
1854 if (!OptimizeHotColdNew)
1855 return nullptr;
1856
1857 uint8_t HotCold;
1858 bool IsCold = false;
1859 if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold") {
1860 HotCold = ColdNewHintValue;
1861 IsCold = true;
1862 } else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1863 "notcold")
1864 HotCold = NotColdNewHintValue;
1865 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "hot")
1866 HotCold = HotNewHintValue;
1867 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1868 "ambiguous")
1869 HotCold = AmbiguousNewHintValue;
1870 else
1871 return nullptr;
1872
1873 bool ShouldOptimizeExistingHotColdNew =
1874 OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Always ||
1875 (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Cold &&
1876 IsCold);
1877
1878 Value *HotColdVal = B.getInt8(C: HotCold);
1879 auto getHotColdHintForExisting = [&](uint8_t HotCold) -> Value * {
1880 // If not taking the minimum, simply use the compiler hint value.
1881 if (!MinExistingHotColdNewHint)
1882 return HotColdVal;
1883 Value *ExistingHint = CI->getArgOperand(i: CI->arg_size() - 1);
1884 if (ExistingHint->getType() != B.getInt8Ty())
1885 ExistingHint = B.CreateTruncOrBitCast(V: ExistingHint, DestTy: B.getInt8Ty());
1886 // Emit a umin intrinsic to take the minimum of the existing hint and the
1887 // compiler hint. When the existing hint is a compile-time constant, the
1888 // IRBuilder folder will automatically constant-fold this into a constant.
1889 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: ExistingHint, RHS: HotColdVal);
1890 };
1891
1892 // For calls that already pass a hot/cold hint, only update the hint if
1893 // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
1894 // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
1895 // Note that in cases where we decide it is "notcold", it might be slightly
1896 // better to replace the hinted call with a non hinted call, to avoid the
1897 // extra parameter and the if condition check of the hint value in the
1898 // allocator. This can be considered in the future.
1899 Value *NewCall = nullptr;
1900 switch (Func) {
1901 case LibFunc_Znwm12__hot_cold_t:
1902 if (ShouldOptimizeExistingHotColdNew)
1903 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1904 NewFunc: LibFunc_Znwm12__hot_cold_t,
1905 HotCold: getHotColdHintForExisting(HotCold));
1906 break;
1907 case LibFunc_Znwm:
1908 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1909 NewFunc: LibFunc_Znwm12__hot_cold_t, HotCold: HotColdVal);
1910 break;
1911 case LibFunc_Znam12__hot_cold_t:
1912 if (ShouldOptimizeExistingHotColdNew)
1913 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1914 NewFunc: LibFunc_Znam12__hot_cold_t,
1915 HotCold: getHotColdHintForExisting(HotCold));
1916 break;
1917 case LibFunc_Znam:
1918 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1919 NewFunc: LibFunc_Znam12__hot_cold_t, HotCold: HotColdVal);
1920 break;
1921 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1922 if (ShouldOptimizeExistingHotColdNew)
1923 NewCall =
1924 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1925 TLI, NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
1926 HotCold: getHotColdHintForExisting(HotCold));
1927 break;
1928 case LibFunc_ZnwmRKSt9nothrow_t:
1929 NewCall = emitHotColdNewNoThrow(
1930 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1931 NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1932 break;
1933 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1934 if (ShouldOptimizeExistingHotColdNew)
1935 NewCall =
1936 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1937 TLI, NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
1938 HotCold: getHotColdHintForExisting(HotCold));
1939 break;
1940 case LibFunc_ZnamRKSt9nothrow_t:
1941 NewCall = emitHotColdNewNoThrow(
1942 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1943 NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1944 break;
1945 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1946 if (ShouldOptimizeExistingHotColdNew)
1947 NewCall =
1948 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1949 TLI, NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
1950 HotCold: getHotColdHintForExisting(HotCold));
1951 break;
1952 case LibFunc_ZnwmSt11align_val_t:
1953 NewCall = emitHotColdNewAligned(
1954 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1955 NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1956 break;
1957 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1958 if (ShouldOptimizeExistingHotColdNew)
1959 NewCall =
1960 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1961 TLI, NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t,
1962 HotCold: getHotColdHintForExisting(HotCold));
1963 break;
1964 case LibFunc_ZnamSt11align_val_t:
1965 NewCall = emitHotColdNewAligned(
1966 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1967 NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1968 break;
1969 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1970 if (ShouldOptimizeExistingHotColdNew)
1971 NewCall = emitHotColdNewAlignedNoThrow(
1972 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1973 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1974 HotCold: getHotColdHintForExisting(HotCold));
1975 break;
1976 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1977 NewCall = emitHotColdNewAlignedNoThrow(
1978 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1979 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1980 HotCold: HotColdVal);
1981 break;
1982 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1983 if (ShouldOptimizeExistingHotColdNew)
1984 NewCall = emitHotColdNewAlignedNoThrow(
1985 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1986 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1987 HotCold: getHotColdHintForExisting(HotCold));
1988 break;
1989 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1990 NewCall = emitHotColdNewAlignedNoThrow(
1991 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1992 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1993 HotCold: HotColdVal);
1994 break;
1995 case LibFunc_size_returning_new:
1996 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
1997 NewFunc: LibFunc_size_returning_new_hot_cold,
1998 HotCold: HotColdVal);
1999 break;
2000 case LibFunc_size_returning_new_hot_cold:
2001 if (ShouldOptimizeExistingHotColdNew)
2002 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
2003 NewFunc: LibFunc_size_returning_new_hot_cold,
2004 HotCold: getHotColdHintForExisting(HotCold));
2005 break;
2006 case LibFunc_size_returning_new_aligned:
2007 NewCall = emitHotColdSizeReturningNewAligned(
2008 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
2009 NewFunc: LibFunc_size_returning_new_aligned_hot_cold, HotCold: HotColdVal);
2010 break;
2011 case LibFunc_size_returning_new_aligned_hot_cold:
2012 if (ShouldOptimizeExistingHotColdNew)
2013 NewCall = emitHotColdSizeReturningNewAligned(
2014 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
2015 NewFunc: LibFunc_size_returning_new_aligned_hot_cold,
2016 HotCold: getHotColdHintForExisting(HotCold));
2017 break;
2018 default:
2019 return nullptr;
2020 }
2021
2022 if (auto *NewCI = dyn_cast_or_null<Instruction>(Val: NewCall))
2023 NewCI->copyMetadata(SrcInst: *CI);
2024
2025 return NewCall;
2026}
2027
2028//===----------------------------------------------------------------------===//
2029// Math Library Optimizations
2030//===----------------------------------------------------------------------===//
2031
2032/// Preserve the accuracy requirement of \p Old on the replacement \p New.
2033static void copyFPMath(const CallInst &Old, Value *New) {
2034 if (auto *NewI = dyn_cast<Instruction>(Val: New))
2035 if (MDNode *MD = Old.getMetadata(KindID: LLVMContext::MD_fpmath))
2036 NewI->setMetadata(KindID: LLVMContext::MD_fpmath, Node: MD);
2037}
2038
2039// Replace a libcall \p CI with a call to intrinsic \p IID
2040static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
2041 Intrinsic::ID IID) {
2042 Value *NewCall = B.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0), FMFSource: CI);
2043 NewCall->takeName(V: CI);
2044 copyFPMath(Old: *CI, New: NewCall);
2045 return copyFlags(Old: *CI, New: NewCall);
2046}
2047
2048static Value *replaceBinaryCall(CallInst *CI, IRBuilderBase &B,
2049 Intrinsic::ID IID) {
2050 Value *NewCall = B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2051 RHS: CI->getArgOperand(i: 1), FMFSource: CI);
2052 NewCall->takeName(V: CI);
2053 copyFPMath(Old: *CI, New: NewCall);
2054 return copyFlags(Old: *CI, New: NewCall);
2055}
2056
2057/// Return a variant of Val with float type.
2058/// Currently this works in two cases: If Val is an FPExtension of a float
2059/// value to something bigger, simply return the operand.
2060/// If Val is a ConstantFP but can be converted to a float ConstantFP without
2061/// loss of precision do so.
2062static Value *valueHasFloatPrecision(Value *Val) {
2063 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
2064 Value *Op = Cast->getOperand(i_nocapture: 0);
2065 if (Op->getType()->isFloatTy())
2066 return Op;
2067 }
2068 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
2069 APFloat F = Const->getValueAPF();
2070 bool losesInfo;
2071 (void)F.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
2072 losesInfo: &losesInfo);
2073 if (!losesInfo)
2074 return ConstantFP::get(Context&: Const->getContext(), V: F);
2075 }
2076 return nullptr;
2077}
2078
2079/// Shrink double -> float functions.
2080static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
2081 bool isBinary, const TargetLibraryInfo *TLI,
2082 bool isPrecise = false) {
2083 Function *CalleeFn = CI->getCalledFunction();
2084 if (!CI->getType()->isDoubleTy() || !CalleeFn)
2085 return nullptr;
2086
2087 // If not all the uses of the function are converted to float, then bail out.
2088 // This matters if the precision of the result is more important than the
2089 // precision of the arguments.
2090 if (isPrecise)
2091 for (User *U : CI->users()) {
2092 FPTruncInst *Cast = dyn_cast<FPTruncInst>(Val: U);
2093 if (!Cast || !Cast->getType()->isFloatTy())
2094 return nullptr;
2095 }
2096
2097 // If this is something like 'g((double) float)', convert to 'gf(float)'.
2098 Value *V[2];
2099 V[0] = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2100 V[1] = isBinary ? valueHasFloatPrecision(Val: CI->getArgOperand(i: 1)) : nullptr;
2101 if (!V[0] || (isBinary && !V[1]))
2102 return nullptr;
2103
2104 // If call isn't an intrinsic, check that it isn't within a function with the
2105 // same name as the float version of this call, otherwise the result is an
2106 // infinite loop. For example, from MinGW-w64:
2107 //
2108 // float expf(float val) { return (float) exp((double) val); }
2109 StringRef CalleeName = CalleeFn->getName();
2110 bool IsIntrinsic = CalleeFn->isIntrinsic();
2111 if (!IsIntrinsic) {
2112 StringRef CallerName = CI->getFunction()->getName();
2113 if (CallerName.ends_with(Suffix: 'f') &&
2114 CallerName.size() == (CalleeName.size() + 1) &&
2115 CallerName.starts_with(Prefix: CalleeName))
2116 return nullptr;
2117 }
2118
2119 // Propagate the math semantics from the current function to the new function.
2120 IRBuilderBase::FastMathFlagGuard Guard(B);
2121 B.setFastMathFlags(CI->getFastMathFlags());
2122
2123 // g((double) float) -> (double) gf(float)
2124 Value *R;
2125 if (IsIntrinsic) {
2126 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
2127 R = isBinary ? B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V)
2128 : B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V[0]);
2129 } else {
2130 AttributeList CallsiteAttrs = CI->getAttributes();
2131 R = isBinary
2132 ? emitBinaryFloatFnCall(Op1: V[0], Op2: V[1], TLI, Name: CalleeName, B,
2133 Attrs: CallsiteAttrs)
2134 : emitUnaryFloatFnCall(Op: V[0], TLI, Name: CalleeName, B, Attrs: CallsiteAttrs);
2135 }
2136 return B.CreateFPExt(V: R, DestTy: B.getDoubleTy());
2137}
2138
2139/// Shrink double -> float for unary functions.
2140static Value *optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2141 const TargetLibraryInfo *TLI,
2142 bool isPrecise = false) {
2143 return optimizeDoubleFP(CI, B, isBinary: false, TLI, isPrecise);
2144}
2145
2146/// Shrink double -> float for binary functions.
2147static Value *optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2148 const TargetLibraryInfo *TLI,
2149 bool isPrecise = false) {
2150 return optimizeDoubleFP(CI, B, isBinary: true, TLI, isPrecise);
2151}
2152
2153/// Shrink double -> float for llvm.sincos.
2154static Value *optimizeSinCosDoubleFP(CallInst *CI, IRBuilderBase &B) {
2155 auto *RetTy = dyn_cast<StructType>(Val: CI->getType());
2156 if (!RetTy || RetTy->getNumElements() != 2 ||
2157 !RetTy->getElementType(N: 0)->getScalarType()->isDoubleTy())
2158 return nullptr;
2159
2160 Value *X = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2161 if (!X)
2162 if (auto *Ext = dyn_cast<FPExtInst>(Val: CI->getArgOperand(i: 0)))
2163 if (Ext->getOperand(i_nocapture: 0)->getType()->getScalarType()->isFloatTy())
2164 X = Ext->getOperand(i_nocapture: 0);
2165 if (!X)
2166 return nullptr;
2167
2168 for (User *U : CI->users()) {
2169 auto *EV = dyn_cast<ExtractValueInst>(Val: U);
2170 if (!EV)
2171 return nullptr;
2172 for (User *EVU : EV->users()) {
2173 auto *Cast = dyn_cast<FPTruncInst>(Val: EVU);
2174 if (!Cast || !Cast->getType()->getScalarType()->isFloatTy())
2175 return nullptr;
2176 }
2177 }
2178
2179 IRBuilderBase::FastMathFlagGuard Guard(B);
2180 B.setFastMathFlags(CI->getFastMathFlags());
2181
2182 Value *NewCall = B.CreateIntrinsic(ID: Intrinsic::sincos, OverloadTypes: X->getType(), Args: X);
2183 cast<Instruction>(Val: NewCall)->setMetadata(
2184 KindID: LLVMContext::MD_fpmath, Node: CI->getMetadata(KindID: LLVMContext::MD_fpmath));
2185 Value *Res = PoisonValue::get(T: RetTy);
2186 for (unsigned I = 0; I != 2; ++I) {
2187 Value *Ext = B.CreateFPExt(V: B.CreateExtractValue(Agg: NewCall, Idxs: I),
2188 DestTy: RetTy->getElementType(N: I));
2189 Res = B.CreateInsertValue(Agg: Res, Val: Ext, Idxs: I);
2190 }
2191 return Res;
2192}
2193
2194// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
2195Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
2196 Value *Real, *Imag;
2197
2198 if (CI->arg_size() == 1) {
2199
2200 if (!CI->isFast())
2201 return nullptr;
2202
2203 Value *Op = CI->getArgOperand(i: 0);
2204 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
2205
2206 Real = B.CreateExtractValue(Agg: Op, Idxs: 0, Name: "real");
2207 Imag = B.CreateExtractValue(Agg: Op, Idxs: 1, Name: "imag");
2208
2209 } else {
2210 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
2211
2212 Real = CI->getArgOperand(i: 0);
2213 Imag = CI->getArgOperand(i: 1);
2214
2215 // if real or imaginary part is zero, simplify to abs(cimag(z))
2216 // or abs(creal(z))
2217 Value *AbsOp = nullptr;
2218 if (ConstantFP *ConstReal = dyn_cast<ConstantFP>(Val: Real)) {
2219 if (ConstReal->isZero())
2220 AbsOp = Imag;
2221
2222 } else if (ConstantFP *ConstImag = dyn_cast<ConstantFP>(Val: Imag)) {
2223 if (ConstImag->isZero())
2224 AbsOp = Real;
2225 }
2226
2227 if (AbsOp)
2228 return copyFlags(Old: *CI, New: B.CreateFAbs(V: AbsOp, FMFSource: CI, Name: "cabs"));
2229
2230 if (!CI->isFast())
2231 return nullptr;
2232 }
2233
2234 // Propagate fast-math flags from the existing call to new instructions.
2235 Value *RealReal = B.CreateFMulFMF(L: Real, R: Real, FMFSource: CI);
2236 Value *ImagImag = B.CreateFMulFMF(L: Imag, R: Imag, FMFSource: CI);
2237 return copyFlags(
2238 Old: *CI, New: B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt,
2239 Op: B.CreateFAddFMF(L: RealReal, R: ImagImag, FMFSource: CI), FMFSource: CI,
2240 Name: "cabs"));
2241}
2242
2243// Return a properly extended integer (DstWidth bits wide) if the operation is
2244// an itofp.
2245static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
2246 if (isa<SIToFPInst>(Val: I2F) || isa<UIToFPInst>(Val: I2F)) {
2247 Value *Op = cast<Instruction>(Val: I2F)->getOperand(i: 0);
2248 // Make sure that the exponent fits inside an "int" of size DstWidth,
2249 // thus avoiding any range issues that FP has not.
2250 unsigned BitWidth = Op->getType()->getScalarSizeInBits();
2251 if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(Val: I2F))) {
2252 Type *IntTy = Op->getType()->getWithNewBitWidth(NewBitWidth: DstWidth);
2253 return isa<SIToFPInst>(Val: I2F) ? B.CreateSExt(V: Op, DestTy: IntTy)
2254 : B.CreateZExt(V: Op, DestTy: IntTy);
2255 }
2256 }
2257
2258 return nullptr;
2259}
2260
2261/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
2262/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
2263/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
2264Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
2265 Module *M = Pow->getModule();
2266 Value *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2267 Type *Ty = Pow->getType();
2268 bool Ignored;
2269
2270 // Evaluate special cases related to a nested function as the base.
2271
2272 // pow(exp(x), y) -> exp(x * y)
2273 // pow(exp2(x), y) -> exp2(x * y)
2274 // If exp{,2}() is used only once, it is better to fold two transcendental
2275 // math functions into one. If used again, exp{,2}() would still have to be
2276 // called with the original argument, then keep both original transcendental
2277 // functions. However, this transformation is only safe with fully relaxed
2278 // math semantics, since, besides rounding differences, it changes overflow
2279 // and underflow behavior quite dramatically. For example:
2280 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
2281 // Whereas:
2282 // exp(1000 * 0.001) = exp(1)
2283 // TODO: Loosen the requirement for fully relaxed math semantics.
2284 // TODO: Handle exp10() when more targets have it available.
2285 CallInst *BaseFn = dyn_cast<CallInst>(Val: Base);
2286 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
2287 Function *CalleeFn = BaseFn->getCalledFunction();
2288 LibFunc LibFn =
2289 CalleeFn ? TLI->getLibFunc(funcName: CalleeFn->getName()) : NotLibFunc;
2290 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFn)) {
2291 StringRef ExpName;
2292 Intrinsic::ID ID;
2293 Value *ExpFn;
2294 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2295
2296 switch (LibFn) {
2297 default:
2298 return nullptr;
2299 case LibFunc_expf:
2300 case LibFunc_exp:
2301 case LibFunc_expl:
2302 ExpName = TLI->getName(F: LibFunc_exp);
2303 ID = Intrinsic::exp;
2304 LibFnFloat = LibFunc_expf;
2305 LibFnDouble = LibFunc_exp;
2306 LibFnLongDouble = LibFunc_expl;
2307 break;
2308 case LibFunc_exp2f:
2309 case LibFunc_exp2:
2310 case LibFunc_exp2l:
2311 ExpName = TLI->getName(F: LibFunc_exp2);
2312 ID = Intrinsic::exp2;
2313 LibFnFloat = LibFunc_exp2f;
2314 LibFnDouble = LibFunc_exp2;
2315 LibFnLongDouble = LibFunc_exp2l;
2316 break;
2317 }
2318
2319 // Create new exp{,2}() with the product as its argument.
2320 Value *FMul = B.CreateFMul(L: BaseFn->getArgOperand(i: 0), R: Expo, Name: "mul");
2321 ExpFn = BaseFn->doesNotAccessMemory()
2322 ? B.CreateUnaryIntrinsic(ID, Op: FMul, FMFSource: nullptr, Name: ExpName)
2323 : emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFnDouble, FloatFn: LibFnFloat,
2324 LongDoubleFn: LibFnLongDouble, B,
2325 Attrs: BaseFn->getAttributes());
2326
2327 // Since the new exp{,2}() is different from the original one, dead code
2328 // elimination cannot be trusted to remove it, since it may have side
2329 // effects (e.g., errno). When the only consumer for the original
2330 // exp{,2}() is pow(), then it has to be explicitly erased.
2331 substituteInParent(I: BaseFn, With: ExpFn);
2332 return ExpFn;
2333 }
2334 }
2335
2336 // Evaluate special cases related to a constant base.
2337
2338 const APFloat *BaseF;
2339 if (!match(V: Base, P: m_APFloat(Res&: BaseF)))
2340 return nullptr;
2341
2342 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2343
2344 const bool UseIntrinsic = Pow->doesNotAccessMemory();
2345
2346 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2347 if ((UseIntrinsic || !Ty->isVectorTy()) && BaseF->isExactlyValue(V: 2.0) &&
2348 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo)) &&
2349 (UseIntrinsic ||
2350 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2351
2352 // TODO: Shouldn't really need to depend on getIntToFPVal for intrinsic. Can
2353 // just directly use the original integer type.
2354 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize())) {
2355 Constant *One = ConstantFP::get(Ty, V: 1.0);
2356
2357 if (UseIntrinsic) {
2358 return copyFlags(Old: *Pow, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2359 OverloadTypes: {Ty, ExpoI->getType()},
2360 Args: {One, ExpoI}, FMFSource: Pow, Name: "exp2"));
2361 }
2362
2363 return copyFlags(Old: *Pow, New: emitBinaryFloatFnCall(
2364 Op1: One, Op2: ExpoI, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2365 LongDoubleFn: LibFunc_ldexpl, B, Attrs: NoAttrs));
2366 }
2367 }
2368
2369 // pow(2.0 ** n, x) -> exp2(n * x)
2370 if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f, LongDoubleFn: LibFunc_exp2l)) {
2371 APFloat BaseR = APFloat(1.0);
2372 BaseR.convert(ToSemantics: BaseF->getSemantics(), RM: APFloat::rmTowardZero, losesInfo: &Ignored);
2373 BaseR = BaseR / *BaseF;
2374 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2375 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2376 APSInt NI(64, false);
2377 if ((IsInteger || IsReciprocal) &&
2378 NF->convertToInteger(Result&: NI, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2379 APFloat::opOK &&
2380 NI > 1 && NI.isPowerOf2()) {
2381 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2382 Value *FMul = B.CreateFMul(L: Expo, R: ConstantFP::get(Ty, V: N), Name: "mul");
2383 if (Pow->doesNotAccessMemory())
2384 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2385 FMFSource: nullptr, Name: "exp2"));
2386 else
2387 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2388 FloatFn: LibFunc_exp2f,
2389 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2390 }
2391 }
2392
2393 // pow(10.0, x) -> exp10(x)
2394 if (BaseF->isExactlyValue(V: 10.0) &&
2395 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp10, FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l)) {
2396
2397 if (Pow->doesNotAccessMemory()) {
2398 return B.CreateIntrinsic(ID: Intrinsic::exp10, OverloadTypes: {Ty}, Args: {Expo}, FMFSource: Pow, Name: "exp10", OpBundles: {},
2399 SetFn: [Pow](CallInst *CI) { CI->copyIRFlags(V: Pow); });
2400 }
2401
2402 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: Expo, TLI, DoubleFn: LibFunc_exp10,
2403 FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l,
2404 B, Attrs: NoAttrs));
2405 }
2406
2407 // pow(x, y) -> exp2(log2(x) * y)
2408 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2409 !BaseF->isNegative()) {
2410 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2411 // Luckily optimizePow has already handled the x == 1 case.
2412 assert(!match(Base, m_FPOne()) &&
2413 "pow(1.0, y) should have been simplified earlier!");
2414
2415 Value *Log = nullptr;
2416 if (Ty->isFloatTy())
2417 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToFloat()));
2418 else if (Ty->isDoubleTy())
2419 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToDouble()));
2420
2421 if (Log) {
2422 Value *FMul = B.CreateFMul(L: Log, R: Expo, Name: "mul");
2423 if (Pow->doesNotAccessMemory())
2424 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2425 FMFSource: nullptr, Name: "exp2"));
2426 else if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f,
2427 LongDoubleFn: LibFunc_exp2l))
2428 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2429 FloatFn: LibFunc_exp2f,
2430 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2431 }
2432 }
2433
2434 return nullptr;
2435}
2436
2437static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2438 Module *M, IRBuilderBase &B,
2439 const TargetLibraryInfo *TLI) {
2440 // If errno is never set, then use the intrinsic for sqrt().
2441 if (NoErrno)
2442 return B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: V, FMFSource: nullptr, Name: "sqrt");
2443
2444 // Otherwise, use the libcall for sqrt().
2445 if (hasFloatFn(M, TLI, Ty: V->getType(), DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2446 LongDoubleFn: LibFunc_sqrtl))
2447 // TODO: We also should check that the target can in fact lower the sqrt()
2448 // libcall. We currently have no way to ask this question, so we ask if
2449 // the target has a sqrt() libcall, which is not exactly the same.
2450 return emitUnaryFloatFnCall(Op: V, TLI, DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2451 LongDoubleFn: LibFunc_sqrtl, B, Attrs);
2452
2453 return nullptr;
2454}
2455
2456/// Use square root in place of pow(x, +/-0.5).
2457Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2458 Value *Sqrt, *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2459 Module *Mod = Pow->getModule();
2460 Type *Ty = Pow->getType();
2461
2462 const APFloat *ExpoF;
2463 if (!match(V: Expo, P: m_APFloat(Res&: ExpoF)) ||
2464 (!ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)))
2465 return nullptr;
2466
2467 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2468 // so that requires fast-math-flags (afn or reassoc).
2469 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2470 return nullptr;
2471
2472 // If we have a pow() library call (accesses memory) and we can't guarantee
2473 // that the base is not an infinity, give up:
2474 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2475 // errno), but sqrt(-Inf) is required by various standards to set errno.
2476 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2477 !isKnownNeverInfinity(
2478 V: Base, SQ: SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
2479 return nullptr;
2480
2481 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M: Mod, B,
2482 TLI);
2483 if (!Sqrt)
2484 return nullptr;
2485
2486 // Handle signed zero base by expanding to fabs(sqrt(x)).
2487 if (!Pow->hasNoSignedZeros())
2488 Sqrt = B.CreateFAbs(V: Sqrt, FMFSource: nullptr, Name: "abs");
2489
2490 Sqrt = copyFlags(Old: *Pow, New: Sqrt);
2491
2492 // Handle non finite base by expanding to
2493 // (x == -infinity ? +infinity : sqrt(x)).
2494 if (!Pow->hasNoInfs()) {
2495 Value *PosInf = ConstantFP::getInfinity(Ty),
2496 *NegInf = ConstantFP::getInfinity(Ty, Negative: true);
2497 Value *FCmp = B.CreateFCmpOEQ(LHS: Base, RHS: NegInf, Name: "isinf");
2498 Sqrt = B.CreateSelect(C: FCmp, True: PosInf, False: Sqrt);
2499 // We assume that the case where x == -infinity is unlikely, so we assign
2500 // unlikely branch weights to that arm of the select.
2501 if (!ProfcheckDisableMetadataFixes) {
2502 if (auto *SqrtSI = dyn_cast<SelectInst>(Val: Sqrt))
2503 setBranchWeights(
2504 I&: *SqrtSI,
2505 Weights: {MDBuilder::kUnlikelyBranchWeight, MDBuilder::kLikelyBranchWeight},
2506 /*IsExpected=*/false);
2507 }
2508 }
2509
2510 // If the exponent is negative, then get the reciprocal.
2511 if (ExpoF->isNegative())
2512 Sqrt = B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Sqrt, Name: "reciprocal");
2513
2514 return Sqrt;
2515}
2516
2517static Value *createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M,
2518 IRBuilderBase &B) {
2519 Value *Args[] = {Base, Expo};
2520 Type *Types[] = {Base->getType(), Expo->getType()};
2521 return B.CreateIntrinsic(ID: Intrinsic::powi, OverloadTypes: Types, Args);
2522}
2523
2524Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2525 Value *Base = Pow->getArgOperand(i: 0);
2526 Value *Expo = Pow->getArgOperand(i: 1);
2527 Function *Callee = Pow->getCalledFunction();
2528 StringRef Name = Callee->getName();
2529 Type *Ty = Pow->getType();
2530 Module *M = Pow->getModule();
2531 bool AllowApprox = Pow->hasApproxFunc();
2532 bool Ignored;
2533
2534 // Propagate the math semantics from the call to any created instructions.
2535 IRBuilderBase::FastMathFlagGuard Guard(B);
2536 B.setFastMathFlags(Pow->getFastMathFlags());
2537 // Evaluate special cases related to the base.
2538
2539 // pow(1.0, x) -> 1.0
2540 if (match(V: Base, P: m_FPOne()))
2541 return Base;
2542
2543 if (Value *Exp = replacePowWithExp(Pow, B))
2544 return Exp;
2545
2546 // Evaluate special cases related to the exponent.
2547
2548 // pow(x, -1.0) -> 1.0 / x
2549 if (match(V: Expo, P: m_SpecificFP(V: -1.0)))
2550 return B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Base, Name: "reciprocal");
2551
2552 // pow(x, +/-0.0) -> 1.0
2553 if (match(V: Expo, P: m_AnyZeroFP()))
2554 return ConstantFP::get(Ty, V: 1.0);
2555
2556 // pow(x, 1.0) -> x
2557 if (match(V: Expo, P: m_FPOne()))
2558 return Base;
2559
2560 // pow(x, 2.0) -> x * x
2561 if (match(V: Expo, P: m_SpecificFP(V: 2.0)) && Pow->doesNotAccessMemory())
2562 return B.CreateFMul(L: Base, R: Base, Name: "square");
2563
2564 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2565 return Sqrt;
2566
2567 // If we can approximate pow:
2568 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2569 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2570 const APFloat *ExpoF;
2571 if (AllowApprox && match(V: Expo, P: m_APFloat(Res&: ExpoF)) &&
2572 !ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)) {
2573 APFloat ExpoA(abs(X: *ExpoF));
2574 APFloat ExpoI(*ExpoF);
2575 Value *Sqrt = nullptr;
2576 if (!ExpoA.isInteger()) {
2577 APFloat Expo2 = ExpoA;
2578 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2579 // is no floating point exception and the result is an integer, then
2580 // ExpoA == integer + 0.5
2581 if (Expo2.add(RHS: ExpoA, RM: APFloat::rmNearestTiesToEven) != APFloat::opOK)
2582 return nullptr;
2583
2584 if (!Expo2.isInteger())
2585 return nullptr;
2586
2587 if (ExpoI.roundToIntegral(RM: APFloat::rmTowardNegative) !=
2588 APFloat::opInexact)
2589 return nullptr;
2590 if (!ExpoI.isInteger())
2591 return nullptr;
2592 ExpoF = &ExpoI;
2593
2594 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M,
2595 B, TLI);
2596 if (!Sqrt)
2597 return nullptr;
2598 }
2599
2600 // 0.5 fraction is now optionally handled.
2601 // Do pow -> powi for remaining integer exponent
2602 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2603 if (ExpoF->isInteger() &&
2604 ExpoF->convertToInteger(Result&: IntExpo, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2605 APFloat::opOK) {
2606 Value *PowI = copyFlags(
2607 Old: *Pow,
2608 New: createPowWithIntegerExponent(
2609 Base, Expo: ConstantInt::get(Ty: B.getIntNTy(N: TLI->getIntSize()), V: IntExpo),
2610 M, B));
2611
2612 if (PowI && Sqrt)
2613 return B.CreateFMul(L: PowI, R: Sqrt);
2614
2615 return PowI;
2616 }
2617 }
2618
2619 // powf(x, itofp(y)) -> powi(x, y)
2620 // The powi exponent must be a scalar integer, so a vector y is not usable.
2621 if (AllowApprox && !Expo->getType()->isVectorTy() &&
2622 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo))) {
2623 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize()))
2624 return copyFlags(Old: *Pow, New: createPowWithIntegerExponent(Base, Expo: ExpoI, M, B));
2625 }
2626
2627 // Shrink pow() to powf() if the arguments are single precision,
2628 // unless the result is expected to be double precision.
2629 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_pow) &&
2630 hasFloatVersion(M, FuncName: Name)) {
2631 if (Value *Shrunk = optimizeBinaryDoubleFP(CI: Pow, B, TLI, isPrecise: true))
2632 return Shrunk;
2633 }
2634
2635 return nullptr;
2636}
2637
2638Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2639 Module *M = CI->getModule();
2640 Function *Callee = CI->getCalledFunction();
2641 StringRef Name = Callee->getName();
2642 Value *Ret = nullptr;
2643 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_exp2) &&
2644 hasFloatVersion(M, FuncName: Name))
2645 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2646
2647 // If we have an llvm.exp2 intrinsic, emit the llvm.ldexp intrinsic. If we
2648 // have the libcall, emit the libcall.
2649 //
2650 // TODO: In principle we should be able to just always use the intrinsic for
2651 // any doesNotAccessMemory callsite.
2652
2653 const bool UseIntrinsic = Callee->isIntrinsic();
2654 // Bail out for vectors because the code below only expects scalars.
2655 Type *Ty = CI->getType();
2656 if (!UseIntrinsic && Ty->isVectorTy())
2657 return Ret;
2658
2659 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2660 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2661 Value *Op = CI->getArgOperand(i: 0);
2662 if ((isa<SIToFPInst>(Val: Op) || isa<UIToFPInst>(Val: Op)) &&
2663 (UseIntrinsic ||
2664 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2665 if (Value *Exp = getIntToFPVal(I2F: Op, B, DstWidth: TLI->getIntSize())) {
2666 Constant *One = ConstantFP::get(Ty, V: 1.0);
2667
2668 if (UseIntrinsic) {
2669 return copyFlags(Old: *CI, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2670 OverloadTypes: {Ty, Exp->getType()},
2671 Args: {One, Exp}, FMFSource: CI));
2672 }
2673
2674 IRBuilderBase::FastMathFlagGuard Guard(B);
2675 B.setFastMathFlags(CI->getFastMathFlags());
2676 return copyFlags(Old: *CI, New: emitBinaryFloatFnCall(
2677 Op1: One, Op2: Exp, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2678 LongDoubleFn: LibFunc_ldexpl, B, Attrs: AttributeList()));
2679 }
2680 }
2681
2682 return Ret;
2683}
2684
2685Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B,
2686 Intrinsic::ID IID) {
2687 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2688 // the intrinsics for improved optimization (for example, vectorization).
2689 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2690 // From the C standard draft WG14/N1256:
2691 // "Ideally, fmax would be sensitive to the sign of zero, for example
2692 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2693 // might be impractical."
2694 FastMathFlags FMF = CI->getFastMathFlags();
2695 FMF.setNoSignedZeros();
2696 return copyFlags(Old: *CI, New: B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2697 RHS: CI->getArgOperand(i: 1), FMFSource: FMF));
2698}
2699
2700Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2701 Function *LogFn = Log->getCalledFunction();
2702 StringRef LogNm = LogFn->getName();
2703 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2704 Module *Mod = Log->getModule();
2705 Type *Ty = Log->getType();
2706
2707 if (UnsafeFPShrink && (LogFn->isIntrinsic() || hasFloatVersion(M: Mod, FuncName: LogNm)))
2708 if (Value *Ret = optimizeUnaryDoubleFP(CI: Log, B, TLI, isPrecise: true))
2709 return Ret;
2710
2711 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2712
2713 // This is only applicable to log(), log2(), log10().
2714 LogLb = TLI->getLibFunc(funcName: LogNm);
2715 if (LogLb != NotLibFunc) {
2716 switch (LogLb) {
2717 case LibFunc_logf:
2718 LogID = Intrinsic::log;
2719 ExpLb = LibFunc_expf;
2720 Exp2Lb = LibFunc_exp2f;
2721 Exp10Lb = LibFunc_exp10f;
2722 PowLb = LibFunc_powf;
2723 break;
2724 case LibFunc_log:
2725 LogID = Intrinsic::log;
2726 ExpLb = LibFunc_exp;
2727 Exp2Lb = LibFunc_exp2;
2728 Exp10Lb = LibFunc_exp10;
2729 PowLb = LibFunc_pow;
2730 break;
2731 case LibFunc_logl:
2732 LogID = Intrinsic::log;
2733 ExpLb = LibFunc_expl;
2734 Exp2Lb = LibFunc_exp2l;
2735 Exp10Lb = LibFunc_exp10l;
2736 PowLb = LibFunc_powl;
2737 break;
2738 case LibFunc_log2f:
2739 LogID = Intrinsic::log2;
2740 ExpLb = LibFunc_expf;
2741 Exp2Lb = LibFunc_exp2f;
2742 Exp10Lb = LibFunc_exp10f;
2743 PowLb = LibFunc_powf;
2744 break;
2745 case LibFunc_log2:
2746 LogID = Intrinsic::log2;
2747 ExpLb = LibFunc_exp;
2748 Exp2Lb = LibFunc_exp2;
2749 Exp10Lb = LibFunc_exp10;
2750 PowLb = LibFunc_pow;
2751 break;
2752 case LibFunc_log2l:
2753 LogID = Intrinsic::log2;
2754 ExpLb = LibFunc_expl;
2755 Exp2Lb = LibFunc_exp2l;
2756 Exp10Lb = LibFunc_exp10l;
2757 PowLb = LibFunc_powl;
2758 break;
2759 case LibFunc_log10f:
2760 LogID = Intrinsic::log10;
2761 ExpLb = LibFunc_expf;
2762 Exp2Lb = LibFunc_exp2f;
2763 Exp10Lb = LibFunc_exp10f;
2764 PowLb = LibFunc_powf;
2765 break;
2766 case LibFunc_log10:
2767 LogID = Intrinsic::log10;
2768 ExpLb = LibFunc_exp;
2769 Exp2Lb = LibFunc_exp2;
2770 Exp10Lb = LibFunc_exp10;
2771 PowLb = LibFunc_pow;
2772 break;
2773 case LibFunc_log10l:
2774 LogID = Intrinsic::log10;
2775 ExpLb = LibFunc_expl;
2776 Exp2Lb = LibFunc_exp2l;
2777 Exp10Lb = LibFunc_exp10l;
2778 PowLb = LibFunc_powl;
2779 break;
2780 default:
2781 return nullptr;
2782 }
2783
2784 // Convert libcall to intrinsic if the value is known > 0.
2785 bool IsKnownNoErrno = Log->hasNoNaNs() && Log->hasNoInfs();
2786 if (!IsKnownNoErrno) {
2787 SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
2788 KnownFPClass Known = computeKnownFPClass(
2789 V: Log->getOperand(i_nocapture: 0),
2790 InterestedClasses: KnownFPClass::OrderedLessThanZeroMask | fcSubnormal, SQ);
2791 Function *F = Log->getParent()->getParent();
2792 const fltSemantics &FltSem = Ty->getScalarType()->getFltSemantics();
2793 IsKnownNoErrno =
2794 Known.cannotBeOrderedLessThanZero() &&
2795 Known.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
2796 }
2797 if (IsKnownNoErrno) {
2798 Value *NewLog = B.CreateUnaryIntrinsic(ID: LogID, Op: Log->getArgOperand(i: 0), FMFSource: Log);
2799 if (auto *I = dyn_cast<Instruction>(Val: NewLog)) {
2800 I->copyMetadata(SrcInst: *Log);
2801 return copyFlags(Old: *Log, New: I);
2802 }
2803 return NewLog;
2804 }
2805 } else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2806 LogID == Intrinsic::log10) {
2807 if (Ty->getScalarType()->isFloatTy()) {
2808 ExpLb = LibFunc_expf;
2809 Exp2Lb = LibFunc_exp2f;
2810 Exp10Lb = LibFunc_exp10f;
2811 PowLb = LibFunc_powf;
2812 } else if (Ty->getScalarType()->isDoubleTy()) {
2813 ExpLb = LibFunc_exp;
2814 Exp2Lb = LibFunc_exp2;
2815 Exp10Lb = LibFunc_exp10;
2816 PowLb = LibFunc_pow;
2817 } else
2818 return nullptr;
2819 } else
2820 return nullptr;
2821
2822 // The earlier call must also be 'fast' in order to do these transforms.
2823 CallInst *Arg = dyn_cast<CallInst>(Val: Log->getArgOperand(i: 0));
2824 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2825 return nullptr;
2826
2827 IRBuilderBase::FastMathFlagGuard Guard(B);
2828 B.setFastMathFlags(FastMathFlags::getFast());
2829
2830 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2831 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2832
2833 // log(pow(x,y)) -> y*log(x)
2834 AttributeList NoAttrs;
2835 if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2836 Value *LogX =
2837 Log->doesNotAccessMemory()
2838 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Arg->getOperand(i_nocapture: 0), FMFSource: nullptr, Name: "log")
2839 : emitUnaryFloatFnCall(Op: Arg->getOperand(i_nocapture: 0), TLI, Name: LogNm, B, Attrs: NoAttrs);
2840 Value *Y = Arg->getArgOperand(i: 1);
2841 // Cast exponent to FP if integer.
2842 if (ArgID == Intrinsic::powi)
2843 Y = B.CreateSIToFP(V: Y, DestTy: Ty, Name: "cast");
2844 Value *MulY = B.CreateFMul(L: Y, R: LogX, Name: "mul");
2845 // Since pow() may have side effects, e.g. errno,
2846 // dead code elimination may not be trusted to remove it.
2847 substituteInParent(I: Arg, With: MulY);
2848 return MulY;
2849 }
2850
2851 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2852 // TODO: There is no exp10() intrinsic yet.
2853 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2854 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
2855 Constant *Eul;
2856 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2857 // FIXME: Add more precise value of e for long double.
2858 Eul = ConstantFP::get(Ty: Log->getType(), V: numbers::e);
2859 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2860 Eul = ConstantFP::get(Ty: Log->getType(), V: 2.0);
2861 else
2862 Eul = ConstantFP::get(Ty: Log->getType(), V: 10.0);
2863 Value *LogE = Log->doesNotAccessMemory()
2864 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Eul, FMFSource: nullptr, Name: "log")
2865 : emitUnaryFloatFnCall(Op: Eul, TLI, Name: LogNm, B, Attrs: NoAttrs);
2866 Value *MulY = B.CreateFMul(L: Arg->getArgOperand(i: 0), R: LogE, Name: "mul");
2867 // Since exp() may have side effects, e.g. errno,
2868 // dead code elimination may not be trusted to remove it.
2869 substituteInParent(I: Arg, With: MulY);
2870 return MulY;
2871 }
2872
2873 return nullptr;
2874}
2875
2876// sqrt(exp(X)) -> exp(X * 0.5)
2877Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2878 if (!CI->hasAllowReassoc())
2879 return nullptr;
2880
2881 Function *SqrtFn = CI->getCalledFunction();
2882 CallInst *Arg = dyn_cast<CallInst>(Val: CI->getArgOperand(i: 0));
2883 if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2884 return nullptr;
2885 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2886 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2887
2888 LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2889
2890 SqrtLb = TLI->getLibFunc(funcName: SqrtFn->getName());
2891 if (SqrtLb != NotLibFunc)
2892 switch (SqrtLb) {
2893 case LibFunc_sqrtf:
2894 ExpLb = LibFunc_expf;
2895 Exp2Lb = LibFunc_exp2f;
2896 Exp10Lb = LibFunc_exp10f;
2897 break;
2898 case LibFunc_sqrt:
2899 ExpLb = LibFunc_exp;
2900 Exp2Lb = LibFunc_exp2;
2901 Exp10Lb = LibFunc_exp10;
2902 break;
2903 case LibFunc_sqrtl:
2904 ExpLb = LibFunc_expl;
2905 Exp2Lb = LibFunc_exp2l;
2906 Exp10Lb = LibFunc_exp10l;
2907 break;
2908 default:
2909 return nullptr;
2910 }
2911 else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2912 if (CI->getType()->getScalarType()->isFloatTy()) {
2913 ExpLb = LibFunc_expf;
2914 Exp2Lb = LibFunc_exp2f;
2915 Exp10Lb = LibFunc_exp10f;
2916 } else if (CI->getType()->getScalarType()->isDoubleTy()) {
2917 ExpLb = LibFunc_exp;
2918 Exp2Lb = LibFunc_exp2;
2919 Exp10Lb = LibFunc_exp10;
2920 } else
2921 return nullptr;
2922 } else
2923 return nullptr;
2924
2925 if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2926 ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2927 return nullptr;
2928
2929 IRBuilderBase::InsertPointGuard Guard(B);
2930 B.SetInsertPoint(Arg);
2931 auto *ExpOperand = Arg->getOperand(i_nocapture: 0);
2932 auto *FMul =
2933 B.CreateFMulFMF(L: ExpOperand, R: ConstantFP::get(Ty: ExpOperand->getType(), V: 0.5),
2934 FMFSource: CI, Name: "merged.sqrt");
2935
2936 Arg->setOperand(i_nocapture: 0, Val_nocapture: FMul);
2937 return Arg;
2938}
2939
2940Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2941 Module *M = CI->getModule();
2942 Function *Callee = CI->getCalledFunction();
2943 Value *Ret = nullptr;
2944 // TODO: Once we have a way (other than checking for the existince of the
2945 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2946 // condition below.
2947 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_sqrtf) &&
2948 (Callee->getName() == "sqrt" ||
2949 Callee->getIntrinsicID() == Intrinsic::sqrt))
2950 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2951
2952 if (Value *Opt = mergeSqrtToExp(CI, B))
2953 return Opt;
2954
2955 if (!CI->isFast())
2956 return Ret;
2957
2958 Instruction *I = dyn_cast<Instruction>(Val: CI->getArgOperand(i: 0));
2959 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2960 return Ret;
2961
2962 // We're looking for a repeated factor in a multiplication tree,
2963 // so we can do this fold: sqrt(x * x) -> fabs(x);
2964 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2965 Value *Op0 = I->getOperand(i: 0);
2966 Value *Op1 = I->getOperand(i: 1);
2967 Value *RepeatOp = nullptr;
2968 Value *OtherOp = nullptr;
2969 if (Op0 == Op1) {
2970 // Simple match: the operands of the multiply are identical.
2971 RepeatOp = Op0;
2972 } else {
2973 // Look for a more complicated pattern: one of the operands is itself
2974 // a multiply, so search for a common factor in that multiply.
2975 // Note: We don't bother looking any deeper than this first level or for
2976 // variations of this pattern because instcombine's visitFMUL and/or the
2977 // reassociation pass should give us this form.
2978 Value *MulOp;
2979 if (match(V: Op0, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2980 cast<Instruction>(Val: Op0)->isFast()) {
2981 // Pattern: sqrt((x * x) * z)
2982 RepeatOp = MulOp;
2983 OtherOp = Op1;
2984 } else if (match(V: Op1, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2985 cast<Instruction>(Val: Op1)->isFast()) {
2986 // Pattern: sqrt(z * (x * x))
2987 RepeatOp = MulOp;
2988 OtherOp = Op0;
2989 }
2990 }
2991 if (!RepeatOp)
2992 return Ret;
2993
2994 // Fast math flags for any created instructions should match the sqrt
2995 // and multiply.
2996
2997 // If we found a repeated factor, hoist it out of the square root and
2998 // replace it with the fabs of that factor.
2999 Value *FabsCall = B.CreateFAbs(V: RepeatOp, FMFSource: I, Name: "fabs");
3000 if (OtherOp) {
3001 // If we found a non-repeated factor, we still need to get its square
3002 // root. We then multiply that by the value that was simplified out
3003 // of the square root calculation.
3004 Value *SqrtCall =
3005 B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: OtherOp, FMFSource: I, Name: "sqrt");
3006 return copyFlags(Old: *CI, New: B.CreateFMulFMF(L: FabsCall, R: SqrtCall, FMFSource: I));
3007 }
3008 return copyFlags(Old: *CI, New: FabsCall);
3009}
3010
3011Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
3012
3013 // fmod(x,y) sets errno if y == 0 or x == +/-inf. frem does not set errno,
3014 // so the fold is valid only when we can prove fmod wouldn't either.
3015 bool IsNoErrno = CI->hasNoNaNs();
3016 if (!IsNoErrno) {
3017 SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
3018 KnownFPClass Known0 = computeKnownFPClass(V: CI->getOperand(i_nocapture: 0), InterestedClasses: fcInf, SQ);
3019 if (Known0.isKnownNeverInfinity()) {
3020 KnownFPClass Known1 =
3021 computeKnownFPClass(V: CI->getOperand(i_nocapture: 1), InterestedClasses: fcZero | fcSubnormal, SQ);
3022 Function *F = CI->getParent()->getParent();
3023 const fltSemantics &FltSem =
3024 CI->getType()->getScalarType()->getFltSemantics();
3025 IsNoErrno = Known1.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
3026 }
3027 }
3028
3029 if (IsNoErrno)
3030 return B.CreateFRemFMF(L: CI->getOperand(i_nocapture: 0), R: CI->getOperand(i_nocapture: 1), FMFSource: CI);
3031 return nullptr;
3032}
3033
3034Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
3035 IRBuilderBase &B) {
3036 Module *M = CI->getModule();
3037 Function *Callee = CI->getCalledFunction();
3038 Value *Ret = nullptr;
3039 StringRef Name = Callee->getName();
3040 if (UnsafeFPShrink &&
3041 (Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
3042 Name == "asinh") &&
3043 hasFloatVersion(M, FuncName: Name))
3044 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
3045
3046 Value *Op1 = CI->getArgOperand(i: 0);
3047 auto *OpC = dyn_cast<CallInst>(Val: Op1);
3048 if (!OpC)
3049 return Ret;
3050
3051 // Both calls must be 'fast' in order to remove them.
3052 if (!CI->isFast() || !OpC->isFast())
3053 return Ret;
3054
3055 // tan(atan(x)) -> x
3056 // atanh(tanh(x)) -> x
3057 // sinh(asinh(x)) -> x
3058 // asinh(sinh(x)) -> x
3059 // cosh(acosh(x)) -> x
3060 Function *F = OpC->getCalledFunction();
3061 LibFunc Func = F ? TLI->getLibFunc(funcName: F->getName()) : NotLibFunc;
3062 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3063 LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
3064 .Case(S: "tan", Value: LibFunc_atan)
3065 .Case(S: "atanh", Value: LibFunc_tanh)
3066 .Case(S: "sinh", Value: LibFunc_asinh)
3067 .Case(S: "cosh", Value: LibFunc_acosh)
3068 .Case(S: "tanf", Value: LibFunc_atanf)
3069 .Case(S: "atanhf", Value: LibFunc_tanhf)
3070 .Case(S: "sinhf", Value: LibFunc_asinhf)
3071 .Case(S: "coshf", Value: LibFunc_acoshf)
3072 .Case(S: "tanl", Value: LibFunc_atanl)
3073 .Case(S: "atanhl", Value: LibFunc_tanhl)
3074 .Case(S: "sinhl", Value: LibFunc_asinhl)
3075 .Case(S: "coshl", Value: LibFunc_acoshl)
3076 .Case(S: "asinh", Value: LibFunc_sinh)
3077 .Case(S: "asinhf", Value: LibFunc_sinhf)
3078 .Case(S: "asinhl", Value: LibFunc_sinhl)
3079 .Default(Value: NotLibFunc); // Used as error value
3080 if (Func == inverseFunc)
3081 Ret = OpC->getArgOperand(i: 0);
3082 }
3083 return Ret;
3084}
3085
3086static bool isTrigLibCall(CallInst *CI) {
3087 // We can only hope to do anything useful if we can ignore things like errno
3088 // and floating-point exceptions.
3089 // We already checked the prototype.
3090 return CI->doesNotThrow() && CI->doesNotAccessMemory();
3091}
3092
3093static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
3094 bool UseFloat, Value *&Sin, Value *&Cos,
3095 Value *&SinCos, const TargetLibraryInfo *TLI) {
3096 Module *M = OrigCallee->getParent();
3097 Type *ArgTy = Arg->getType();
3098 Type *ResTy;
3099 StringRef Name;
3100
3101 Triple T(OrigCallee->getParent()->getTargetTriple());
3102 if (UseFloat) {
3103 Name = "__sincospif_stret";
3104
3105 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
3106 // x86_64 can't use {float, float} since that would be returned in both
3107 // xmm0 and xmm1, which isn't what a real struct would do.
3108 ResTy = T.getArch() == Triple::x86_64
3109 ? static_cast<Type *>(FixedVectorType::get(ElementType: ArgTy, NumElts: 2))
3110 : static_cast<Type *>(StructType::get(elt1: ArgTy, elts: ArgTy));
3111 } else {
3112 Name = "__sincospi_stret";
3113 ResTy = StructType::get(elt1: ArgTy, elts: ArgTy);
3114 }
3115
3116 if (!isLibFuncEmittable(M, TLI, Name))
3117 return false;
3118 LibFunc TheLibFunc = TLI->getLibFunc(funcName: Name);
3119 FunctionCallee Callee = getOrInsertLibFunc(
3120 M, TLI: *TLI, TheLibFunc, AttributeList: OrigCallee->getAttributes(), RetTy: ResTy, Args: ArgTy);
3121
3122 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
3123 // If the argument is an instruction, it must dominate all uses so put our
3124 // sincos call there.
3125 B.SetInsertPoint(++ArgInst->getIterator());
3126 } else {
3127 // Otherwise (e.g. for a constant) the beginning of the function is as
3128 // good a place as any.
3129 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
3130 B.SetInsertPoint(EntryBB.begin());
3131 }
3132
3133 SinCos = B.CreateCall(Callee, Args: Arg, Name: "sincospi");
3134
3135 if (SinCos->getType()->isStructTy()) {
3136 Sin = B.CreateExtractValue(Agg: SinCos, Idxs: 0, Name: "sinpi");
3137 Cos = B.CreateExtractValue(Agg: SinCos, Idxs: 1, Name: "cospi");
3138 } else {
3139 Sin = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{0}, Name: "sinpi");
3140 Cos = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{1}, Name: "cospi");
3141 }
3142
3143 return true;
3144}
3145
3146/// Flushing a denormal to +0.0 breaks f(-x) = -f(x) for odd f.
3147static bool mayFlushDenormalsToPositiveZero(const CallInst *CI) {
3148 DenormalMode Mode = CI->getFunction()->getDenormalMode(
3149 FPType: CI->getType()->getScalarType()->getFltSemantics());
3150 return Mode.inputsMayBePositiveZero() || Mode.outputsMayBePositiveZero();
3151}
3152
3153static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
3154 IRBuilderBase &B) {
3155 Value *X;
3156 Value *Src = CI->getArgOperand(i: 0);
3157
3158 if (match(V: Src, P: m_OneUse(SubPattern: m_FNeg(X: m_Value(V&: X)))) &&
3159 (IsEven || !mayFlushDenormalsToPositiveZero(CI))) {
3160 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3161 auto *CallInst = copyFlags(Old: *CI, New: Call);
3162 if (IsEven) {
3163 // Even function: f(-x) = f(x)
3164 return CallInst;
3165 }
3166 // Odd function: f(-x) = -f(x)
3167 return B.CreateFNegFMF(V: CallInst, FMFSource: CI);
3168 }
3169
3170 // Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
3171 if (IsEven && (match(V: Src, P: m_FAbs(Op0: m_Value(V&: X))) ||
3172 match(V: Src, P: m_CopySign(Op0: m_Value(V&: X), Op1: m_Value())))) {
3173 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3174 return copyFlags(Old: *CI, New: Call);
3175 }
3176
3177 return nullptr;
3178}
3179
3180Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
3181 IRBuilderBase &B) {
3182 switch (Func) {
3183 case LibFunc_cos:
3184 case LibFunc_cosf:
3185 case LibFunc_cosl:
3186
3187 case LibFunc_cosh:
3188 case LibFunc_coshf:
3189 case LibFunc_coshl:
3190 return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
3191
3192 case LibFunc_cbrt:
3193 case LibFunc_cbrtf:
3194 case LibFunc_cbrtl:
3195
3196 case LibFunc_sin:
3197 case LibFunc_sinf:
3198 case LibFunc_sinl:
3199
3200 case LibFunc_sinh:
3201 case LibFunc_sinhf:
3202 case LibFunc_sinhl:
3203
3204 case LibFunc_asin:
3205 case LibFunc_asinf:
3206 case LibFunc_asinl:
3207
3208 case LibFunc_asinh:
3209 case LibFunc_asinhf:
3210 case LibFunc_asinhl:
3211
3212 case LibFunc_tan:
3213 case LibFunc_tanf:
3214 case LibFunc_tanl:
3215
3216 case LibFunc_tanh:
3217 case LibFunc_tanhf:
3218 case LibFunc_tanhl:
3219
3220 case LibFunc_atan:
3221 case LibFunc_atanf:
3222 case LibFunc_atanl:
3223
3224 case LibFunc_erf:
3225 case LibFunc_erff:
3226 case LibFunc_erfl:
3227 return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
3228
3229 default:
3230 return nullptr;
3231 }
3232}
3233
3234Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
3235 // Make sure the prototype is as expected, otherwise the rest of the
3236 // function is probably invalid and likely to abort.
3237 if (!isTrigLibCall(CI))
3238 return nullptr;
3239
3240 Value *Arg = CI->getArgOperand(i: 0);
3241 if (isa<ConstantData>(Val: Arg))
3242 return nullptr;
3243
3244 SmallVector<CallInst *, 1> SinCalls;
3245 SmallVector<CallInst *, 1> CosCalls;
3246 SmallVector<CallInst *, 1> SinCosCalls;
3247
3248 bool IsFloat = Arg->getType()->isFloatTy();
3249
3250 // Look for all compatible sinpi, cospi and sincospi calls with the same
3251 // argument. If there are enough (in some sense) we can make the
3252 // substitution.
3253 Function *F = CI->getFunction();
3254 for (User *U : Arg->users())
3255 classifyArgUse(Val: U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
3256
3257 // It's only worthwhile if both sinpi and cospi are actually used.
3258 if (SinCalls.empty() || CosCalls.empty())
3259 return nullptr;
3260
3261 Value *Sin, *Cos, *SinCos;
3262 if (!insertSinCosCall(B, OrigCallee: CI->getCalledFunction(), Arg, UseFloat: IsFloat, Sin, Cos,
3263 SinCos, TLI))
3264 return nullptr;
3265
3266 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
3267 Value *Res) {
3268 for (CallInst *C : Calls)
3269 replaceAllUsesWith(I: C, With: Res);
3270 };
3271
3272 replaceTrigInsts(SinCalls, Sin);
3273 replaceTrigInsts(CosCalls, Cos);
3274 replaceTrigInsts(SinCosCalls, SinCos);
3275
3276 return IsSin ? Sin : Cos;
3277}
3278
3279void LibCallSimplifier::classifyArgUse(
3280 Value *Val, Function *F, bool IsFloat,
3281 SmallVectorImpl<CallInst *> &SinCalls,
3282 SmallVectorImpl<CallInst *> &CosCalls,
3283 SmallVectorImpl<CallInst *> &SinCosCalls) {
3284 auto *CI = dyn_cast<CallInst>(Val);
3285 if (!CI || CI->use_empty())
3286 return;
3287
3288 // Don't consider calls in other functions.
3289 if (CI->getFunction() != F)
3290 return;
3291
3292 Module *M = CI->getModule();
3293 Function *Callee = CI->getCalledFunction();
3294 LibFunc Func = Callee ? TLI->getLibFunc(FDecl: *Callee) : NotLibFunc;
3295 if (!isLibFuncEmittable(M, TLI, TheLibFunc: Func) || !isTrigLibCall(CI))
3296 return;
3297
3298 if (IsFloat) {
3299 if (Func == LibFunc_sinpif)
3300 SinCalls.push_back(Elt: CI);
3301 else if (Func == LibFunc_cospif)
3302 CosCalls.push_back(Elt: CI);
3303 else if (Func == LibFunc_sincospif_stret)
3304 SinCosCalls.push_back(Elt: CI);
3305 } else {
3306 if (Func == LibFunc_sinpi)
3307 SinCalls.push_back(Elt: CI);
3308 else if (Func == LibFunc_cospi)
3309 CosCalls.push_back(Elt: CI);
3310 else if (Func == LibFunc_sincospi_stret)
3311 SinCosCalls.push_back(Elt: CI);
3312 }
3313}
3314
3315/// Constant folds remquo
3316Value *LibCallSimplifier::optimizeRemquo(CallInst *CI, IRBuilderBase &B) {
3317 const APFloat *X, *Y;
3318 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3319 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3320 return nullptr;
3321
3322 APFloat::opStatus Status;
3323 APFloat Quot = *X;
3324 Status = Quot.divide(RHS: *Y, RM: APFloat::rmNearestTiesToEven);
3325 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3326 return nullptr;
3327 APFloat Rem = *X;
3328 if (Rem.remainder(RHS: *Y) != APFloat::opOK)
3329 return nullptr;
3330
3331 // TODO: We can only keep at least the three of the last bits of x/y
3332 unsigned IntBW = TLI->getIntSize();
3333 APSInt QuotInt(IntBW, /*isUnsigned=*/false);
3334 bool IsExact;
3335 Status =
3336 Quot.convertToInteger(Result&: QuotInt, RM: APFloat::rmNearestTiesToEven, IsExact: &IsExact);
3337 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3338 return nullptr;
3339
3340 B.CreateAlignedStore(
3341 Val: ConstantInt::getSigned(Ty: B.getIntNTy(N: IntBW), V: QuotInt.getExtValue()),
3342 Ptr: CI->getArgOperand(i: 2), Align: CI->getParamAlign(ArgNo: 2));
3343 return ConstantFP::get(Ty: CI->getType(), V: Rem);
3344}
3345
3346/// Constant folds fdim
3347Value *LibCallSimplifier::optimizeFdim(CallInst *CI, IRBuilderBase &B) {
3348 // Cannot perform the fold unless the call has attribute memory(none)
3349 if (!CI->doesNotAccessMemory())
3350 return nullptr;
3351
3352 // TODO : Handle undef values
3353 // Propagate poison if any
3354 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 0)))
3355 return CI->getArgOperand(i: 0);
3356 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 1)))
3357 return CI->getArgOperand(i: 1);
3358
3359 const APFloat *X, *Y;
3360 // Check if both values are constants
3361 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3362 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3363 return nullptr;
3364
3365 // C99 fdim(x, y) = (x > y) ? x - y : +0.
3366 if (X->compare(RHS: *Y) != APFloat::cmpGreaterThan && !X->isNaN() && !Y->isNaN())
3367 return ConstantFP::getZero(Ty: CI->getType());
3368 APFloat Difference = *X;
3369 Difference.subtract(RHS: *Y, RM: RoundingMode::NearestTiesToEven);
3370 return ConstantFP::get(Ty: CI->getType(), V: Difference);
3371}
3372
3373//===----------------------------------------------------------------------===//
3374// Integer Library Call Optimizations
3375//===----------------------------------------------------------------------===//
3376
3377Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
3378 // All variants of ffs return int which need not be 32 bits wide.
3379 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
3380 Type *RetType = CI->getType();
3381 Value *Op = CI->getArgOperand(i: 0);
3382 Type *ArgType = Op->getType();
3383 Value *V = B.CreateIntrinsic(ID: Intrinsic::cttz, OverloadTypes: {ArgType}, Args: {Op, B.getTrue()},
3384 FMFSource: nullptr, Name: "cttz");
3385 V = B.CreateAdd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 1));
3386 V = B.CreateIntCast(V, DestTy: RetType, isSigned: false);
3387
3388 Value *Cond = B.CreateICmpNE(LHS: Op, RHS: Constant::getNullValue(Ty: ArgType));
3389 Value *S = B.CreateSelect(C: Cond, True: V, False: ConstantInt::get(Ty: RetType, V: 0));
3390 if (ProfcheckDisableMetadataFixes)
3391 return S;
3392 if (auto *SI = dyn_cast<SelectInst>(Val: S))
3393 setBranchWeights(
3394 I&: *SI, Weights: {MDBuilder::kLikelyBranchWeight, MDBuilder::kUnlikelyBranchWeight},
3395 /*IsExpected=*/false);
3396 return S;
3397}
3398
3399Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
3400 // All variants of fls return int which need not be 32 bits wide.
3401 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
3402 Value *Op = CI->getArgOperand(i: 0);
3403 Type *ArgType = Op->getType();
3404 Value *V = B.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {ArgType}, Args: {Op, B.getFalse()},
3405 FMFSource: nullptr, Name: "ctlz");
3406 V = B.CreateSub(LHS: ConstantInt::get(Ty: V->getType(), V: ArgType->getIntegerBitWidth()),
3407 RHS: V);
3408 return B.CreateIntCast(V, DestTy: CI->getType(), isSigned: false);
3409}
3410
3411Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
3412 // abs(x) -> x <s 0 ? -x : x
3413 // The negation has 'nsw' because abs of INT_MIN is undefined.
3414 Value *X = CI->getArgOperand(i: 0);
3415 Value *IsNeg = B.CreateIsNeg(Arg: X);
3416 Value *NegX = B.CreateNSWNeg(V: X, Name: "neg");
3417 return B.CreateSelect(C: IsNeg, True: NegX, False: X);
3418}
3419
3420Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
3421 // isdigit(c) -> (c-'0') <u 10
3422 Value *Op = CI->getArgOperand(i: 0);
3423 Type *ArgType = Op->getType();
3424 Op = B.CreateSub(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: '0'), Name: "isdigittmp");
3425 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 10), Name: "isdigit");
3426 return B.CreateZExt(V: Op, DestTy: CI->getType());
3427}
3428
3429Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
3430 // isascii(c) -> c <u 128
3431 Value *Op = CI->getArgOperand(i: 0);
3432 Type *ArgType = Op->getType();
3433 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 128), Name: "isascii");
3434 return B.CreateZExt(V: Op, DestTy: CI->getType());
3435}
3436
3437Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
3438 // toascii(c) -> c & 0x7f
3439 return B.CreateAnd(LHS: CI->getArgOperand(i: 0),
3440 RHS: ConstantInt::get(Ty: CI->getType(), V: 0x7F));
3441}
3442
3443// Fold calls to atoi, atol, and atoll.
3444Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
3445 StringRef Str;
3446 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3447 return nullptr;
3448
3449 return convertStrToInt(CI, Str, EndPtr: nullptr, Base: 10, /*AsSigned=*/true, B);
3450}
3451
3452// Fold calls to strtol, strtoll, strtoul, and strtoull.
3453Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
3454 bool AsSigned) {
3455 Value *EndPtr = CI->getArgOperand(i: 1);
3456 if (isa<ConstantPointerNull>(Val: EndPtr)) {
3457 // With a null EndPtr, this function won't capture the main argument.
3458 // It would be readonly too, except that it still may write to errno.
3459 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
3460 CI: CaptureInfo::none()));
3461 EndPtr = nullptr;
3462 } else if (!isKnownNonZero(V: EndPtr, Q: DL))
3463 return nullptr;
3464
3465 StringRef Str;
3466 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3467 return nullptr;
3468
3469 if (ConstantInt *CInt = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2))) {
3470 return convertStrToInt(CI, Str, EndPtr, Base: CInt->getSExtValue(), AsSigned, B);
3471 }
3472
3473 return nullptr;
3474}
3475
3476//===----------------------------------------------------------------------===//
3477// Formatting and IO Library Call Optimizations
3478//===----------------------------------------------------------------------===//
3479
3480static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
3481
3482Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3483 int StreamArg) {
3484 Function *Callee = CI->getCalledFunction();
3485 // Error reporting calls should be cold, mark them as such.
3486 // This applies even to non-builtin calls: it is only a hint and applies to
3487 // functions that the frontend might not understand as builtins.
3488
3489 // This heuristic was suggested in:
3490 // Improving Static Branch Prediction in a Compiler
3491 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3492 // Proceedings of PACT'98, Oct. 1998, IEEE
3493 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
3494 isReportingError(Callee, CI, StreamArg)) {
3495 CI->addFnAttr(Kind: Attribute::Cold);
3496 }
3497
3498 return nullptr;
3499}
3500
3501static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3502 if (!Callee || !Callee->isDeclaration())
3503 return false;
3504
3505 if (StreamArg < 0)
3506 return true;
3507
3508 // These functions might be considered cold, but only if their stream
3509 // argument is stderr.
3510
3511 if (StreamArg >= (int)CI->arg_size())
3512 return false;
3513 LoadInst *LI = dyn_cast<LoadInst>(Val: CI->getArgOperand(i: StreamArg));
3514 if (!LI)
3515 return false;
3516 GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: LI->getPointerOperand());
3517 if (!GV || !GV->isDeclaration())
3518 return false;
3519 return GV->getName() == "stderr";
3520}
3521
3522Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3523 // Check for a fixed format string.
3524 StringRef FormatStr;
3525 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: FormatStr))
3526 return nullptr;
3527
3528 // Empty format string -> noop.
3529 if (FormatStr.empty()) // Tolerate printf's declared void.
3530 return CI->use_empty() ? (Value *)CI : ConstantInt::get(Ty: CI->getType(), V: 0);
3531
3532 // Do not do any of the following transformations if the printf return value
3533 // is used, in general the printf return value is not compatible with either
3534 // putchar() or puts().
3535 if (!CI->use_empty())
3536 return nullptr;
3537
3538 Type *IntTy = CI->getType();
3539 // printf("x") -> putchar('x'), even for "%" and "%%".
3540 if (FormatStr.size() == 1 || FormatStr == "%%") {
3541 // Convert the character to unsigned char before passing it to putchar
3542 // to avoid host-specific sign extension in the IR. Putchar converts
3543 // it to unsigned char regardless.
3544 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)FormatStr[0]);
3545 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3546 }
3547
3548 // Try to remove call or emit putchar/puts.
3549 if (FormatStr == "%s" && CI->arg_size() > 1) {
3550 StringRef OperandStr;
3551 if (!getConstantStringInfo(V: CI->getOperand(i_nocapture: 1), Str&: OperandStr))
3552 return nullptr;
3553 // printf("%s", "") --> NOP
3554 if (OperandStr.empty())
3555 return (Value *)CI;
3556 // printf("%s", "a") --> putchar('a')
3557 if (OperandStr.size() == 1) {
3558 // Convert the character to unsigned char before passing it to putchar
3559 // to avoid host-specific sign extension in the IR. Putchar converts
3560 // it to unsigned char regardless.
3561 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)OperandStr[0]);
3562 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3563 }
3564 // printf("%s", str"\n") --> puts(str)
3565 if (OperandStr.back() == '\n') {
3566 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3567 return nullptr;
3568 OperandStr = OperandStr.drop_back();
3569 Value *GV = B.CreateGlobalString(Str: OperandStr, Name: "str");
3570 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3571 }
3572 return nullptr;
3573 }
3574
3575 // printf("foo\n") --> puts("foo")
3576 if (FormatStr.back() == '\n' &&
3577 !FormatStr.contains(C: '%')) { // No format characters.
3578 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3579 return nullptr;
3580 // Create a string literal with no \n on it. We expect the constant merge
3581 // pass to be run after this pass, to merge duplicate strings.
3582 FormatStr = FormatStr.drop_back();
3583 Value *GV = B.CreateGlobalString(Str: FormatStr, Name: "str");
3584 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3585 }
3586
3587 // Optimize specific format strings.
3588 // printf("%c", chr) --> putchar(chr)
3589 if (FormatStr == "%c" && CI->arg_size() > 1 &&
3590 CI->getArgOperand(i: 1)->getType()->isIntegerTy()) {
3591 // Convert the argument to the type expected by putchar, i.e., int, which
3592 // need not be 32 bits wide but which is the same as printf's return type.
3593 Value *IntChar = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: IntTy, isSigned: false);
3594 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3595 }
3596
3597 // printf("%s\n", str) --> puts(str)
3598 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3599 CI->getArgOperand(i: 1)->getType()->isPointerTy())
3600 return copyFlags(Old: *CI, New: emitPutS(Str: CI->getArgOperand(i: 1), B, TLI));
3601 return nullptr;
3602}
3603
3604Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3605
3606 Module *M = CI->getModule();
3607 Function *Callee = CI->getCalledFunction();
3608 FunctionType *FT = Callee->getFunctionType();
3609 if (Value *V = optimizePrintFString(CI, B)) {
3610 return V;
3611 }
3612
3613 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3614
3615 // printf(format, ...) -> iprintf(format, ...) if no floating point
3616 // arguments.
3617 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_iprintf) &&
3618 !callHasFloatingPointArgument(CI)) {
3619 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_iprintf, T: FT,
3620 AttributeList: Callee->getAttributes());
3621 CallInst *New = cast<CallInst>(Val: CI->clone());
3622 New->setCalledFunction(IPrintFFn);
3623 B.Insert(I: New);
3624 return New;
3625 }
3626
3627 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3628 // arguments.
3629 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_printf) &&
3630 !callHasFP128Argument(CI)) {
3631 auto SmallPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_printf, T: FT,
3632 AttributeList: Callee->getAttributes());
3633 CallInst *New = cast<CallInst>(Val: CI->clone());
3634 New->setCalledFunction(SmallPrintFFn);
3635 B.Insert(I: New);
3636 return New;
3637 }
3638
3639 return nullptr;
3640}
3641
3642Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3643 IRBuilderBase &B) {
3644 // Check for a fixed format string.
3645 StringRef FormatStr;
3646 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3647 return nullptr;
3648
3649 // If we just have a format string (nothing else crazy) transform it.
3650 Value *Dest = CI->getArgOperand(i: 0);
3651 if (CI->arg_size() == 2) {
3652 // Make sure there's no % in the constant array. We could try to handle
3653 // %% -> % in the future if we cared.
3654 if (FormatStr.contains(C: '%'))
3655 return nullptr; // we found a format specifier, bail out.
3656
3657 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3658 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1),
3659 // Copy the null byte.
3660 Size: TLI->getAsSizeT(V: FormatStr.size() + 1, M: *CI->getModule()));
3661 return ConstantInt::get(Ty: CI->getType(), V: FormatStr.size());
3662 }
3663
3664 // The remaining optimizations require the format string to be "%s" or "%c"
3665 // and have an extra operand.
3666 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3667 return nullptr;
3668
3669 // Decode the second character of the format string.
3670 if (FormatStr[1] == 'c') {
3671 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3672 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3673 return nullptr;
3674 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 2), DestTy: B.getInt8Ty(), Name: "char");
3675 Value *Ptr = Dest;
3676 B.CreateStore(Val: V, Ptr);
3677 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3678 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3679
3680 return ConstantInt::get(Ty: CI->getType(), V: 1);
3681 }
3682
3683 if (FormatStr[1] == 's') {
3684 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3685 // strlen(str)+1)
3686 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3687 return nullptr;
3688
3689 if (CI->use_empty())
3690 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3691 return copyFlags(Old: *CI, New: emitStrCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI));
3692
3693 uint64_t SrcLen = GetStringLength(V: CI->getArgOperand(i: 2));
3694 if (SrcLen) {
3695 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1),
3696 Size: TLI->getAsSizeT(V: SrcLen, M: *CI->getModule()));
3697 // Returns total number of characters written without null-character.
3698 return ConstantInt::get(Ty: CI->getType(), V: SrcLen - 1);
3699 } else if (Value *V = emitStpCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI)) {
3700 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3701 Value *PtrDiff = B.CreatePtrDiff(LHS: V, RHS: Dest);
3702 return B.CreateIntCast(V: PtrDiff, DestTy: CI->getType(), isSigned: false);
3703 }
3704
3705 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3706 QueryType: PGSOQueryType::IRPass))
3707 return nullptr;
3708
3709 Value *Len = emitStrLen(Ptr: CI->getArgOperand(i: 2), B, DL, TLI);
3710 if (!Len)
3711 return nullptr;
3712 Value *IncLen =
3713 B.CreateAdd(LHS: Len, RHS: ConstantInt::get(Ty: Len->getType(), V: 1), Name: "leninc");
3714 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1), Size: IncLen);
3715
3716 // The sprintf result is the unincremented number of bytes in the string.
3717 return B.CreateIntCast(V: Len, DestTy: CI->getType(), isSigned: false);
3718 }
3719 return nullptr;
3720}
3721
3722Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3723 Module *M = CI->getModule();
3724 Function *Callee = CI->getCalledFunction();
3725 FunctionType *FT = Callee->getFunctionType();
3726 if (Value *V = optimizeSPrintFString(CI, B)) {
3727 return V;
3728 }
3729
3730 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
3731
3732 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3733 // point arguments.
3734 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_siprintf) &&
3735 !callHasFloatingPointArgument(CI)) {
3736 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_siprintf,
3737 T: FT, AttributeList: Callee->getAttributes());
3738 CallInst *New = cast<CallInst>(Val: CI->clone());
3739 New->setCalledFunction(SIPrintFFn);
3740 B.Insert(I: New);
3741 return New;
3742 }
3743
3744 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3745 // floating point arguments.
3746 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_sprintf) &&
3747 !callHasFP128Argument(CI)) {
3748 auto SmallSPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_sprintf, T: FT,
3749 AttributeList: Callee->getAttributes());
3750 CallInst *New = cast<CallInst>(Val: CI->clone());
3751 New->setCalledFunction(SmallSPrintFFn);
3752 B.Insert(I: New);
3753 return New;
3754 }
3755
3756 return nullptr;
3757}
3758
3759// Transform an snprintf call CI with the bound N to format the string Str
3760// either to a call to memcpy, or to single character a store, or to nothing,
3761// and fold the result to a constant. A nonnull StrArg refers to the string
3762// argument being formatted. Otherwise the call is one with N < 2 and
3763// the "%c" directive to format a single character.
3764Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3765 StringRef Str, uint64_t N,
3766 IRBuilderBase &B) {
3767 assert(StrArg || (N < 2 && Str.size() == 1));
3768
3769 unsigned IntBits = TLI->getIntSize();
3770 uint64_t IntMax = maxIntN(N: IntBits);
3771 if (Str.size() > IntMax)
3772 // Bail if the string is longer than INT_MAX. POSIX requires
3773 // implementations to set errno to EOVERFLOW in this case, in
3774 // addition to when N is larger than that (checked by the caller).
3775 return nullptr;
3776
3777 Value *StrLen = ConstantInt::get(Ty: CI->getType(), V: Str.size());
3778 if (N == 0)
3779 return StrLen;
3780
3781 // Set to the number of bytes to copy fron StrArg which is also
3782 // the offset of the terinating nul.
3783 uint64_t NCopy;
3784 if (N > Str.size())
3785 // Copy the full string, including the terminating nul (which must
3786 // be present regardless of the bound).
3787 NCopy = Str.size() + 1;
3788 else
3789 NCopy = N - 1;
3790
3791 Value *DstArg = CI->getArgOperand(i: 0);
3792 if (NCopy && StrArg)
3793 // Transform the call to lvm.memcpy(dst, fmt, N).
3794 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst: DstArg, DstAlign: Align(1), Src: StrArg, SrcAlign: Align(1),
3795 Size: TLI->getAsSizeT(V: NCopy, M: *CI->getModule())));
3796
3797 if (N > Str.size())
3798 // Return early when the whole format string, including the final nul,
3799 // has been copied.
3800 return StrLen;
3801
3802 // Otherwise, when truncating the string append a terminating nul.
3803 Type *Int8Ty = B.getInt8Ty();
3804 Value *NulOff = B.getIntN(N: IntBits, C: NCopy);
3805 Value *DstEnd = B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: DstArg, IdxList: NulOff, Name: "endptr");
3806 B.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: 0), Ptr: DstEnd);
3807 return StrLen;
3808}
3809
3810Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3811 IRBuilderBase &B) {
3812 // Check for size
3813 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3814 if (!Size)
3815 return nullptr;
3816
3817 uint64_t N = Size->getZExtValue();
3818 uint64_t IntMax = maxIntN(N: TLI->getIntSize());
3819 if (N > IntMax)
3820 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3821 // to set errno to EOVERFLOW in this case.
3822 return nullptr;
3823
3824 Value *DstArg = CI->getArgOperand(i: 0);
3825 Value *FmtArg = CI->getArgOperand(i: 2);
3826
3827 // Check for a fixed format string.
3828 StringRef FormatStr;
3829 if (!getConstantStringInfo(V: FmtArg, Str&: FormatStr))
3830 return nullptr;
3831
3832 // If we just have a format string (nothing else crazy) transform it.
3833 if (CI->arg_size() == 3) {
3834 if (FormatStr.contains(C: '%'))
3835 // Bail if the format string contains a directive and there are
3836 // no arguments. We could handle "%%" in the future.
3837 return nullptr;
3838
3839 return emitSnPrintfMemCpy(CI, StrArg: FmtArg, Str: FormatStr, N, B);
3840 }
3841
3842 // The remaining optimizations require the format string to be "%s" or "%c"
3843 // and have an extra operand.
3844 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3845 return nullptr;
3846
3847 // Decode the second character of the format string.
3848 if (FormatStr[1] == 'c') {
3849 if (N <= 1) {
3850 // Use an arbitary string of length 1 to transform the call into
3851 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3852 // to one.
3853 StringRef CharStr("*");
3854 return emitSnPrintfMemCpy(CI, StrArg: nullptr, Str: CharStr, N, B);
3855 }
3856
3857 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3858 if (!CI->getArgOperand(i: 3)->getType()->isIntegerTy())
3859 return nullptr;
3860 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 3), DestTy: B.getInt8Ty(), Name: "char");
3861 Value *Ptr = DstArg;
3862 B.CreateStore(Val: V, Ptr);
3863 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3864 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3865 return ConstantInt::get(Ty: CI->getType(), V: 1);
3866 }
3867
3868 if (FormatStr[1] != 's')
3869 return nullptr;
3870
3871 Value *StrArg = CI->getArgOperand(i: 3);
3872 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3873 StringRef Str;
3874 if (!getConstantStringInfo(V: StrArg, Str))
3875 return nullptr;
3876
3877 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3878}
3879
3880Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3881 if (Value *V = optimizeSnPrintFString(CI, B)) {
3882 return V;
3883 }
3884
3885 if (isKnownNonZero(V: CI->getOperand(i_nocapture: 1), Q: DL))
3886 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3887 return nullptr;
3888}
3889
3890Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3891 IRBuilderBase &B) {
3892 optimizeErrorReporting(CI, B, StreamArg: 0);
3893
3894 // All the optimizations depend on the format string.
3895 StringRef FormatStr;
3896 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3897 return nullptr;
3898
3899 // Do not do any of the following transformations if the fprintf return
3900 // value is used, in general the fprintf return value is not compatible
3901 // with fwrite(), fputc() or fputs().
3902 if (!CI->use_empty())
3903 return nullptr;
3904
3905 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3906 if (CI->arg_size() == 2) {
3907 // Could handle %% -> % if we cared.
3908 if (FormatStr.contains(C: '%'))
3909 return nullptr; // We found a format specifier.
3910
3911 return copyFlags(
3912 Old: *CI, New: emitFWrite(Ptr: CI->getArgOperand(i: 1),
3913 Size: TLI->getAsSizeT(V: FormatStr.size(), M: *CI->getModule()),
3914 File: CI->getArgOperand(i: 0), B, DL, TLI));
3915 }
3916
3917 // The remaining optimizations require the format string to be "%s" or "%c"
3918 // and have an extra operand.
3919 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3920 return nullptr;
3921
3922 // Decode the second character of the format string.
3923 if (FormatStr[1] == 'c') {
3924 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3925 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3926 return nullptr;
3927 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3928 Value *V = B.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntTy, /*isSigned*/ true,
3929 Name: "chari");
3930 return copyFlags(Old: *CI, New: emitFPutC(Char: V, File: CI->getArgOperand(i: 0), B, TLI));
3931 }
3932
3933 if (FormatStr[1] == 's') {
3934 // fprintf(F, "%s", str) --> fputs(str, F)
3935 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3936 return nullptr;
3937 return copyFlags(
3938 Old: *CI, New: emitFPutS(Str: CI->getArgOperand(i: 2), File: CI->getArgOperand(i: 0), B, TLI));
3939 }
3940 return nullptr;
3941}
3942
3943Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3944 Module *M = CI->getModule();
3945 Function *Callee = CI->getCalledFunction();
3946 FunctionType *FT = Callee->getFunctionType();
3947 if (Value *V = optimizeFPrintFString(CI, B)) {
3948 return V;
3949 }
3950
3951 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3952 // floating point arguments.
3953 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_fiprintf) &&
3954 !callHasFloatingPointArgument(CI)) {
3955 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_fiprintf,
3956 T: FT, AttributeList: Callee->getAttributes());
3957 CallInst *New = cast<CallInst>(Val: CI->clone());
3958 New->setCalledFunction(FIPrintFFn);
3959 B.Insert(I: New);
3960 return New;
3961 }
3962
3963 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3964 // 128-bit floating point arguments.
3965 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_fprintf) &&
3966 !callHasFP128Argument(CI)) {
3967 auto SmallFPrintFFn =
3968 getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_fprintf, T: FT,
3969 AttributeList: Callee->getAttributes());
3970 CallInst *New = cast<CallInst>(Val: CI->clone());
3971 New->setCalledFunction(SmallFPrintFFn);
3972 B.Insert(I: New);
3973 return New;
3974 }
3975
3976 return nullptr;
3977}
3978
3979Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3980 optimizeErrorReporting(CI, B, StreamArg: 3);
3981
3982 // Get the element size and count.
3983 ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3984 ConstantInt *CountC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
3985 if (SizeC && CountC) {
3986 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3987
3988 // If this is writing zero records, remove the call (it's a noop).
3989 if (Bytes == 0)
3990 return ConstantInt::get(Ty: CI->getType(), V: 0);
3991
3992 // If this is writing one byte, turn it into fputc.
3993 // This optimisation is only valid, if the return value is unused.
3994 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
3995 Value *Char = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0), Name: "char");
3996 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3997 Value *Cast = B.CreateIntCast(V: Char, DestTy: IntTy, /*isSigned*/ true, Name: "chari");
3998 Value *NewCI = emitFPutC(Char: Cast, File: CI->getArgOperand(i: 3), B, TLI);
3999 return NewCI ? ConstantInt::get(Ty: CI->getType(), V: 1) : nullptr;
4000 }
4001 }
4002
4003 return nullptr;
4004}
4005
4006Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
4007 optimizeErrorReporting(CI, B, StreamArg: 1);
4008
4009 // Don't rewrite fputs to fwrite when optimising for size because fwrite
4010 // requires more arguments and thus extra MOVs are required.
4011 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
4012 QueryType: PGSOQueryType::IRPass))
4013 return nullptr;
4014
4015 // We can't optimize if return value is used.
4016 if (!CI->use_empty())
4017 return nullptr;
4018
4019 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
4020 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: 0));
4021 if (!Len)
4022 return nullptr;
4023
4024 // Known to have no uses (see above).
4025 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4026 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4027 return copyFlags(
4028 Old: *CI,
4029 New: emitFWrite(Ptr: CI->getArgOperand(i: 0),
4030 Size: ConstantInt::get(Ty: SizeTTy, V: Len - 1),
4031 File: CI->getArgOperand(i: 1), B, DL, TLI));
4032}
4033
4034Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
4035 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
4036 if (!CI->use_empty())
4037 return nullptr;
4038
4039 // Check for a constant string.
4040 // puts("") -> putchar('\n')
4041 StringRef Str;
4042 if (getConstantStringInfo(V: CI->getArgOperand(i: 0), Str) && Str.empty()) {
4043 // putchar takes an argument of the same type as puts returns, i.e.,
4044 // int, which need not be 32 bits wide.
4045 Type *IntTy = CI->getType();
4046 return copyFlags(Old: *CI, New: emitPutChar(Char: ConstantInt::get(Ty: IntTy, V: '\n'), B, TLI));
4047 }
4048
4049 return nullptr;
4050}
4051
4052Value *LibCallSimplifier::optimizeExit(CallInst *CI) {
4053
4054 // Mark 'exit' as cold if its not exit(0) (success).
4055 const APInt *C;
4056 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
4057 match(V: CI->getArgOperand(i: 0), P: m_APInt(Res&: C)) && !C->isZero()) {
4058 CI->addFnAttr(Kind: Attribute::Cold);
4059 }
4060 return nullptr;
4061}
4062
4063Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
4064 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
4065 return copyFlags(Old: *CI, New: B.CreateMemMove(Dst: CI->getArgOperand(i: 1), DstAlign: Align(1),
4066 Src: CI->getArgOperand(i: 0), SrcAlign: Align(1),
4067 Size: CI->getArgOperand(i: 2)));
4068}
4069
4070bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
4071 SmallString<20> FloatFuncName = FuncName;
4072 FloatFuncName += 'f';
4073 return isLibFuncEmittable(M, TLI, Name: FloatFuncName);
4074}
4075
4076Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
4077 IRBuilderBase &Builder) {
4078 Module *M = CI->getModule();
4079 Function *Callee = CI->getCalledFunction();
4080 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4081
4082 // Check for string/memory library functions.
4083 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4084 // Make sure we never change the calling convention.
4085 assert(
4086 (ignoreCallingConv(Func) ||
4087 TargetLibraryInfoImpl::isCallingConvCCompatible(CI)) &&
4088 "Optimizing string/memory libcall would change the calling convention");
4089 switch (Func) {
4090 case LibFunc_strcat:
4091 return optimizeStrCat(CI, B&: Builder);
4092 case LibFunc_strncat:
4093 return optimizeStrNCat(CI, B&: Builder);
4094 case LibFunc_strchr:
4095 return optimizeStrChr(CI, B&: Builder);
4096 case LibFunc_strrchr:
4097 return optimizeStrRChr(CI, B&: Builder);
4098 case LibFunc_strcmp:
4099 return optimizeStrCmp(CI, B&: Builder);
4100 case LibFunc_strncmp:
4101 return optimizeStrNCmp(CI, B&: Builder);
4102 case LibFunc_strcpy:
4103 return optimizeStrCpy(CI, B&: Builder);
4104 case LibFunc_stpcpy:
4105 return optimizeStpCpy(CI, B&: Builder);
4106 case LibFunc_strlcpy:
4107 return optimizeStrLCpy(CI, B&: Builder);
4108 case LibFunc_stpncpy:
4109 return optimizeStringNCpy(CI, /*RetEnd=*/true, B&: Builder);
4110 case LibFunc_strncpy:
4111 return optimizeStringNCpy(CI, /*RetEnd=*/false, B&: Builder);
4112 case LibFunc_strlen:
4113 return optimizeStrLen(CI, B&: Builder);
4114 case LibFunc_strnlen:
4115 return optimizeStrNLen(CI, B&: Builder);
4116 case LibFunc_strpbrk:
4117 return optimizeStrPBrk(CI, B&: Builder);
4118 case LibFunc_strndup:
4119 return optimizeStrNDup(CI, B&: Builder);
4120 case LibFunc_strtol:
4121 case LibFunc_strtod:
4122 case LibFunc_strtof:
4123 case LibFunc_strtoul:
4124 case LibFunc_strtoll:
4125 case LibFunc_strtold:
4126 case LibFunc_strtoull:
4127 return optimizeStrTo(CI, B&: Builder);
4128 case LibFunc_strspn:
4129 return optimizeStrSpn(CI, B&: Builder);
4130 case LibFunc_strcspn:
4131 return optimizeStrCSpn(CI, B&: Builder);
4132 case LibFunc_strstr:
4133 return optimizeStrStr(CI, B&: Builder);
4134 case LibFunc_memchr:
4135 return optimizeMemChr(CI, B&: Builder);
4136 case LibFunc_memrchr:
4137 return optimizeMemRChr(CI, B&: Builder);
4138 case LibFunc_bcmp:
4139 return optimizeBCmp(CI, B&: Builder);
4140 case LibFunc_memcmp:
4141 return optimizeMemCmp(CI, B&: Builder);
4142 case LibFunc_memcpy:
4143 return optimizeMemCpy(CI, B&: Builder);
4144 case LibFunc_memccpy:
4145 return optimizeMemCCpy(CI, B&: Builder);
4146 case LibFunc_mempcpy:
4147 return optimizeMemPCpy(CI, B&: Builder);
4148 case LibFunc_memmove:
4149 return optimizeMemMove(CI, B&: Builder);
4150 case LibFunc_memset:
4151 return optimizeMemSet(CI, B&: Builder);
4152 case LibFunc_realloc:
4153 return optimizeRealloc(CI, B&: Builder);
4154 case LibFunc_wcslen:
4155 return optimizeWcslen(CI, B&: Builder);
4156 case LibFunc_bcopy:
4157 return optimizeBCopy(CI, B&: Builder);
4158 case LibFunc_Znwm:
4159 case LibFunc_ZnwmRKSt9nothrow_t:
4160 case LibFunc_ZnwmSt11align_val_t:
4161 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
4162 case LibFunc_Znam:
4163 case LibFunc_ZnamRKSt9nothrow_t:
4164 case LibFunc_ZnamSt11align_val_t:
4165 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
4166 case LibFunc_Znwm12__hot_cold_t:
4167 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
4168 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
4169 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4170 case LibFunc_Znam12__hot_cold_t:
4171 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
4172 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
4173 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4174 case LibFunc_size_returning_new:
4175 case LibFunc_size_returning_new_hot_cold:
4176 case LibFunc_size_returning_new_aligned:
4177 case LibFunc_size_returning_new_aligned_hot_cold:
4178 return optimizeNew(CI, B&: Builder, Func);
4179 default:
4180 break;
4181 }
4182 }
4183 return nullptr;
4184}
4185
4186/// Constant folding nan/nanf/nanl.
4187static Value *optimizeNaN(CallInst *CI) {
4188 StringRef CharSeq;
4189 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: CharSeq))
4190 return nullptr;
4191
4192 APInt Fill;
4193 // Treat empty strings as if they were zero.
4194 if (CharSeq.empty())
4195 Fill = APInt(32, 0);
4196 else if (CharSeq.getAsInteger(Radix: 0, Result&: Fill))
4197 return nullptr;
4198
4199 return ConstantFP::getQNaN(Ty: CI->getType(), /*Negative=*/false, Payload: &Fill);
4200}
4201
4202Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
4203 LibFunc Func,
4204 IRBuilderBase &Builder) {
4205 const Module *M = CI->getModule();
4206
4207 // Don't optimize calls that require strict floating point semantics.
4208 if (CI->isStrictFP())
4209 return nullptr;
4210
4211 if (Value *V = optimizeSymmetric(CI, Func, B&: Builder))
4212 return V;
4213
4214 switch (Func) {
4215 case LibFunc_sinpif:
4216 case LibFunc_sinpi:
4217 return optimizeSinCosPi(CI, /*IsSin*/true, B&: Builder);
4218 case LibFunc_cospif:
4219 case LibFunc_cospi:
4220 return optimizeSinCosPi(CI, /*IsSin*/false, B&: Builder);
4221 case LibFunc_sinf:
4222 case LibFunc_sinl:
4223 if (CI->doesNotAccessMemory())
4224 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::sin);
4225 return nullptr;
4226 case LibFunc_cosf:
4227 case LibFunc_cosl:
4228 if (CI->doesNotAccessMemory())
4229 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::cos);
4230 return nullptr;
4231 case LibFunc_powf:
4232 case LibFunc_pow:
4233 case LibFunc_powl:
4234 return optimizePow(Pow: CI, B&: Builder);
4235 case LibFunc_exp2l:
4236 case LibFunc_exp2:
4237 case LibFunc_exp2f:
4238 return optimizeExp2(CI, B&: Builder);
4239 case LibFunc_scalbn:
4240 case LibFunc_scalbnf:
4241 case LibFunc_scalbnl:
4242 // LLVM floating-point types have radix 2, so scalbn is equivalent to
4243 // ldexp. Do not replace a libcall that may set errno.
4244 if (CI->doesNotAccessMemory()) {
4245 Value *NewCall =
4246 Builder.CreateLdexp(Src: CI->getArgOperand(i: 0), Exp: CI->getArgOperand(i: 1), FMFSource: CI);
4247 NewCall->takeName(V: CI);
4248 return copyFlags(Old: *CI, New: NewCall);
4249 }
4250 return nullptr;
4251 case LibFunc_fabsf:
4252 case LibFunc_fabs:
4253 case LibFunc_fabsl:
4254 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::fabs);
4255 case LibFunc_sqrtf:
4256 case LibFunc_sqrt:
4257 case LibFunc_sqrtl:
4258 if (Value *V = optimizeSqrt(CI, B&: Builder))
4259 return V;
4260 if (CI->doesNotAccessMemory())
4261 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::sqrt);
4262 return nullptr;
4263 case LibFunc_fmod:
4264 case LibFunc_fmodf:
4265 case LibFunc_fmodl:
4266 return optimizeFMod(CI, B&: Builder);
4267 case LibFunc_logf:
4268 case LibFunc_log:
4269 case LibFunc_logl:
4270 case LibFunc_log10f:
4271 case LibFunc_log10:
4272 case LibFunc_log10l:
4273 case LibFunc_log1pf:
4274 case LibFunc_log1p:
4275 case LibFunc_log1pl:
4276 case LibFunc_log2f:
4277 case LibFunc_log2:
4278 case LibFunc_log2l:
4279 case LibFunc_logbf:
4280 case LibFunc_logb:
4281 case LibFunc_logbl:
4282 return optimizeLog(Log: CI, B&: Builder);
4283 case LibFunc_tan:
4284 case LibFunc_tanf:
4285 case LibFunc_tanl:
4286 case LibFunc_sinh:
4287 case LibFunc_sinhf:
4288 case LibFunc_sinhl:
4289 case LibFunc_asinh:
4290 case LibFunc_asinhf:
4291 case LibFunc_asinhl:
4292 case LibFunc_cosh:
4293 case LibFunc_coshf:
4294 case LibFunc_coshl:
4295 case LibFunc_atanh:
4296 case LibFunc_atanhf:
4297 case LibFunc_atanhl:
4298 return optimizeTrigInversionPairs(CI, B&: Builder);
4299 case LibFunc_ceilf:
4300 case LibFunc_ceil:
4301 case LibFunc_ceill:
4302 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::ceil);
4303 case LibFunc_floorf:
4304 case LibFunc_floor:
4305 case LibFunc_floorl:
4306 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::floor);
4307 case LibFunc_roundf:
4308 case LibFunc_round:
4309 case LibFunc_roundl:
4310 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::round);
4311 case LibFunc_roundevenf:
4312 case LibFunc_roundeven:
4313 case LibFunc_roundevenl:
4314 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::roundeven);
4315 case LibFunc_nearbyintf:
4316 case LibFunc_nearbyint:
4317 case LibFunc_nearbyintl:
4318 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::nearbyint);
4319 case LibFunc_rintf:
4320 case LibFunc_rint:
4321 case LibFunc_rintl:
4322 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::rint);
4323 case LibFunc_truncf:
4324 case LibFunc_trunc:
4325 case LibFunc_truncl:
4326 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::trunc);
4327 case LibFunc_sin:
4328 case LibFunc_cos:
4329 if (UnsafeFPShrink &&
4330 hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4331 if (Value *V = optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true))
4332 return V;
4333 if (CI->doesNotAccessMemory())
4334 return replaceUnaryCall(
4335 CI, B&: Builder, IID: Func == LibFunc_sin ? Intrinsic::sin : Intrinsic::cos);
4336 return nullptr;
4337 case LibFunc_acos:
4338 case LibFunc_acosh:
4339 case LibFunc_asin:
4340 case LibFunc_atan:
4341 case LibFunc_cbrt:
4342 case LibFunc_exp:
4343 case LibFunc_exp10:
4344 case LibFunc_expm1:
4345 case LibFunc_tanh:
4346 if (UnsafeFPShrink && hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4347 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true);
4348 return nullptr;
4349 case LibFunc_copysign:
4350 if (hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4351 if (Value *V = optimizeBinaryDoubleFP(CI, B&: Builder, TLI))
4352 return V;
4353 [[fallthrough]];
4354 case LibFunc_copysignf:
4355 case LibFunc_copysignl:
4356 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::copysign);
4357 case LibFunc_fdim:
4358 case LibFunc_fdimf:
4359 case LibFunc_fdiml:
4360 return optimizeFdim(CI, B&: Builder);
4361 case LibFunc_fminf:
4362 case LibFunc_fmin:
4363 case LibFunc_fminl:
4364 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::minnum);
4365 case LibFunc_fmaxf:
4366 case LibFunc_fmax:
4367 case LibFunc_fmaxl:
4368 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::maxnum);
4369 case LibFunc_fminimum_numf:
4370 case LibFunc_fminimum_num:
4371 case LibFunc_fminimum_numl:
4372 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::minimumnum);
4373 case LibFunc_fmaximum_numf:
4374 case LibFunc_fmaximum_num:
4375 case LibFunc_fmaximum_numl:
4376 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::maximumnum);
4377 case LibFunc_cabs:
4378 case LibFunc_cabsf:
4379 case LibFunc_cabsl:
4380 return optimizeCAbs(CI, B&: Builder);
4381 case LibFunc_remquo:
4382 case LibFunc_remquof:
4383 case LibFunc_remquol:
4384 return optimizeRemquo(CI, B&: Builder);
4385 case LibFunc_nan:
4386 case LibFunc_nanf:
4387 case LibFunc_nanl:
4388 return optimizeNaN(CI);
4389 default:
4390 return nullptr;
4391 }
4392}
4393
4394Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) {
4395 Module *M = CI->getModule();
4396 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
4397
4398 // TODO: Split out the code below that operates on FP calls so that
4399 // we can all non-FP calls with the StrictFP attribute to be
4400 // optimized.
4401 if (CI->isNoBuiltin()) {
4402 // Optionally update operator new calls.
4403 return maybeOptimizeNoBuiltinOperatorNew(CI, B&: Builder);
4404 }
4405
4406 Function *Callee = CI->getCalledFunction();
4407 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4408 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4409
4410 SmallVector<OperandBundleDef, 2> OpBundles;
4411 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4412
4413 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4414 Builder.setDefaultOperandBundles(OpBundles);
4415
4416 // Command-line parameter overrides instruction attribute.
4417 // This can't be moved to optimizeFloatingPointLibCall() because it may be
4418 // used by the intrinsic optimizations.
4419 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
4420 UnsafeFPShrink = EnableUnsafeFPShrink;
4421 else if (isa<FPMathOperator>(Val: CI) && CI->isFast())
4422 UnsafeFPShrink = true;
4423
4424 // First, check for intrinsics.
4425 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI)) {
4426 if (!IsCallingConvC)
4427 return nullptr;
4428 // The FP intrinsics have corresponding constrained versions so we don't
4429 // need to check for the StrictFP attribute here.
4430 switch (II->getIntrinsicID()) {
4431 case Intrinsic::pow:
4432 return optimizePow(Pow: CI, B&: Builder);
4433 case Intrinsic::exp2:
4434 return optimizeExp2(CI, B&: Builder);
4435 case Intrinsic::log:
4436 case Intrinsic::log2:
4437 case Intrinsic::log10:
4438 return optimizeLog(Log: CI, B&: Builder);
4439 case Intrinsic::sqrt:
4440 return optimizeSqrt(CI, B&: Builder);
4441 case Intrinsic::memset:
4442 return optimizeMemSet(CI, B&: Builder);
4443 case Intrinsic::memcpy:
4444 return optimizeMemCpy(CI, B&: Builder);
4445 case Intrinsic::memmove:
4446 return optimizeMemMove(CI, B&: Builder);
4447 case Intrinsic::sin:
4448 case Intrinsic::cos:
4449 if (UnsafeFPShrink)
4450 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, /*isPrecise=*/true);
4451 return nullptr;
4452 case Intrinsic::sincos:
4453 if (UnsafeFPShrink)
4454 return optimizeSinCosDoubleFP(CI, B&: Builder);
4455 return nullptr;
4456 default:
4457 return nullptr;
4458 }
4459 }
4460
4461 // Also try to simplify calls to fortified library functions.
4462 if (Value *SimplifiedFortifiedCI =
4463 FortifiedSimplifier.optimizeCall(CI, B&: Builder))
4464 return SimplifiedFortifiedCI;
4465
4466 // Then check for known library functions.
4467 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4468 // We never change the calling convention.
4469 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4470 return nullptr;
4471 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
4472 return V;
4473 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
4474 return V;
4475 switch (Func) {
4476 case LibFunc_ffs:
4477 case LibFunc_ffsl:
4478 case LibFunc_ffsll:
4479 return optimizeFFS(CI, B&: Builder);
4480 case LibFunc_fls:
4481 case LibFunc_flsl:
4482 case LibFunc_flsll:
4483 return optimizeFls(CI, B&: Builder);
4484 case LibFunc_abs:
4485 case LibFunc_labs:
4486 case LibFunc_llabs:
4487 return optimizeAbs(CI, B&: Builder);
4488 case LibFunc_isdigit:
4489 return optimizeIsDigit(CI, B&: Builder);
4490 case LibFunc_isascii:
4491 return optimizeIsAscii(CI, B&: Builder);
4492 case LibFunc_toascii:
4493 return optimizeToAscii(CI, B&: Builder);
4494 case LibFunc_atoi:
4495 case LibFunc_atol:
4496 case LibFunc_atoll:
4497 return optimizeAtoi(CI, B&: Builder);
4498 case LibFunc_strtol:
4499 case LibFunc_strtoll:
4500 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/true);
4501 case LibFunc_strtoul:
4502 case LibFunc_strtoull:
4503 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/false);
4504 case LibFunc_printf:
4505 return optimizePrintF(CI, B&: Builder);
4506 case LibFunc_sprintf:
4507 return optimizeSPrintF(CI, B&: Builder);
4508 case LibFunc_snprintf:
4509 return optimizeSnPrintF(CI, B&: Builder);
4510 case LibFunc_fprintf:
4511 return optimizeFPrintF(CI, B&: Builder);
4512 case LibFunc_fwrite:
4513 return optimizeFWrite(CI, B&: Builder);
4514 case LibFunc_fputs:
4515 return optimizeFPuts(CI, B&: Builder);
4516 case LibFunc_puts:
4517 return optimizePuts(CI, B&: Builder);
4518 case LibFunc_perror:
4519 return optimizeErrorReporting(CI, B&: Builder);
4520 case LibFunc_vfprintf:
4521 case LibFunc_fiprintf:
4522 return optimizeErrorReporting(CI, B&: Builder, StreamArg: 0);
4523 case LibFunc_exit:
4524 case LibFunc_Exit:
4525 return optimizeExit(CI);
4526 default:
4527 return nullptr;
4528 }
4529 }
4530 return nullptr;
4531}
4532
4533LibCallSimplifier::LibCallSimplifier(
4534 const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT,
4535 DomConditionCache *DC, AssumptionCache *AC, OptimizationRemarkEmitter &ORE,
4536 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
4537 function_ref<void(Instruction *, Value *)> Replacer,
4538 function_ref<void(Instruction *)> Eraser)
4539 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), DT(DT), DC(DC), AC(AC),
4540 ORE(ORE), BFI(BFI), PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
4541
4542void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
4543 // Indirect through the replacer used in this instance.
4544 Replacer(I, With);
4545}
4546
4547void LibCallSimplifier::eraseFromParent(Instruction *I) {
4548 Eraser(I);
4549}
4550
4551// TODO:
4552// Additional cases that we need to add to this file:
4553//
4554// cbrt:
4555// * cbrt(expN(X)) -> expN(x/3)
4556// * cbrt(sqrt(x)) -> pow(x,1/6)
4557// * cbrt(cbrt(x)) -> pow(x,1/9)
4558//
4559// exp, expf, expl:
4560// * exp(log(x)) -> x
4561//
4562// log, logf, logl:
4563// * log(exp(x)) -> x
4564// * log(exp(y)) -> y*log(e)
4565// * log(exp10(y)) -> y*log(10)
4566// * log(sqrt(x)) -> 0.5*log(x)
4567//
4568// pow, powf, powl:
4569// * pow(sqrt(x),y) -> pow(x,y*0.5)
4570// * pow(pow(x,y),z)-> pow(x,y*z)
4571//
4572// signbit:
4573// * signbit(cnst) -> cnst'
4574// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
4575//
4576// sqrt, sqrtf, sqrtl:
4577// * sqrt(expN(x)) -> expN(x*0.5)
4578// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
4579// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
4580//
4581
4582//===----------------------------------------------------------------------===//
4583// Fortified Library Call Optimizations
4584//===----------------------------------------------------------------------===//
4585
4586bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
4587 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
4588 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
4589 // If this function takes a flag argument, the implementation may use it to
4590 // perform extra checks. Don't fold into the non-checking variant.
4591 if (FlagOp) {
4592 ConstantInt *Flag = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *FlagOp));
4593 if (!Flag || !Flag->isZero())
4594 return false;
4595 }
4596
4597 if (SizeOp && CI->getArgOperand(i: ObjSizeOp) == CI->getArgOperand(i: *SizeOp))
4598 return true;
4599
4600 if (ConstantInt *ObjSizeCI =
4601 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: ObjSizeOp))) {
4602 if (ObjSizeCI->isMinusOne())
4603 return true;
4604 // If the object size wasn't -1 (unknown), bail out if we were asked to.
4605 if (OnlyLowerUnknownSize)
4606 return false;
4607 if (StrOp) {
4608 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: *StrOp));
4609 // If the length is 0 we don't know how long it is and so we can't
4610 // remove the check.
4611 if (Len)
4612 annotateDereferenceableBytes(CI, ArgNos: *StrOp, DereferenceableBytes: Len);
4613 else
4614 return false;
4615 return ObjSizeCI->getZExtValue() >= Len;
4616 }
4617
4618 if (SizeOp) {
4619 if (ConstantInt *SizeCI =
4620 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *SizeOp)))
4621 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4622 }
4623 }
4624 return false;
4625}
4626
4627Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4628 IRBuilderBase &B) {
4629 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4630 CallInst *NewCI =
4631 B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4632 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4633 mergeAttributesAndFlags(NewCI, Old: *CI);
4634 return CI->getArgOperand(i: 0);
4635 }
4636 return nullptr;
4637}
4638
4639Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4640 IRBuilderBase &B) {
4641 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4642 CallInst *NewCI =
4643 B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4644 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4645 mergeAttributesAndFlags(NewCI, Old: *CI);
4646 return CI->getArgOperand(i: 0);
4647 }
4648 return nullptr;
4649}
4650
4651Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4652 IRBuilderBase &B) {
4653 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4654 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
4655 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val,
4656 Size: CI->getArgOperand(i: 2), Align: Align(1));
4657 mergeAttributesAndFlags(NewCI, Old: *CI);
4658 return CI->getArgOperand(i: 0);
4659 }
4660 return nullptr;
4661}
4662
4663Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4664 IRBuilderBase &B) {
4665 const DataLayout &DL = CI->getDataLayout();
4666 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2))
4667 if (Value *Call = emitMemPCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4668 Len: CI->getArgOperand(i: 2), B, DL, TLI)) {
4669 return mergeAttributesAndFlags(NewCI: cast<CallInst>(Val: Call), Old: *CI);
4670 }
4671 return nullptr;
4672}
4673
4674Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4675 IRBuilderBase &B,
4676 LibFunc Func) {
4677 const DataLayout &DL = CI->getDataLayout();
4678 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1),
4679 *ObjSize = CI->getArgOperand(i: 2);
4680
4681 // __stpcpy_chk(x,x,...) -> x+strlen(x)
4682 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4683 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
4684 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
4685 }
4686
4687 // If a) we don't have any length information, or b) we know this will
4688 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4689 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
4690 // TODO: It might be nice to get a maximum length out of the possible
4691 // string lengths for varying.
4692 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: 1)) {
4693 if (Func == LibFunc_strcpy_chk)
4694 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
4695 else
4696 return copyFlags(Old: *CI, New: emitStpCpy(Dst, Src, B, TLI));
4697 }
4698
4699 if (OnlyLowerUnknownSize)
4700 return nullptr;
4701
4702 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4703 uint64_t Len = GetStringLength(V: Src);
4704 if (Len)
4705 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
4706 else
4707 return nullptr;
4708
4709 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4710 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4711 Value *LenV = ConstantInt::get(Ty: SizeTTy, V: Len);
4712 Value *Ret = emitMemCpyChk(Dst, Src, Len: LenV, ObjSize, B, DL, TLI);
4713 // If the function was an __stpcpy_chk, and we were able to fold it into
4714 // a __memcpy_chk, we still need to return the correct end pointer.
4715 if (Ret && Func == LibFunc_stpcpy_chk)
4716 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst,
4717 IdxList: ConstantInt::get(Ty: SizeTTy, V: Len - 1));
4718 return copyFlags(Old: *CI, New: cast<CallInst>(Val: Ret));
4719}
4720
4721Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4722 IRBuilderBase &B) {
4723 if (isFortifiedCallFoldable(CI, ObjSizeOp: 1, SizeOp: std::nullopt, StrOp: 0))
4724 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B,
4725 DL: CI->getDataLayout(), TLI));
4726 return nullptr;
4727}
4728
4729Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4730 IRBuilderBase &B,
4731 LibFunc Func) {
4732 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4733 if (Func == LibFunc_strncpy_chk)
4734 return copyFlags(Old: *CI,
4735 New: emitStrNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4736 Len: CI->getArgOperand(i: 2), B, TLI));
4737 else
4738 return copyFlags(Old: *CI,
4739 New: emitStpNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4740 Len: CI->getArgOperand(i: 2), B, TLI));
4741 }
4742
4743 return nullptr;
4744}
4745
4746Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4747 IRBuilderBase &B) {
4748 if (isFortifiedCallFoldable(CI, ObjSizeOp: 4, SizeOp: 3))
4749 return copyFlags(
4750 Old: *CI, New: emitMemCCpy(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
4751 Val: CI->getArgOperand(i: 2), Len: CI->getArgOperand(i: 3), B, TLI));
4752
4753 return nullptr;
4754}
4755
4756Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4757 IRBuilderBase &B) {
4758 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2)) {
4759 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 5));
4760 return copyFlags(Old: *CI,
4761 New: emitSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4762 Fmt: CI->getArgOperand(i: 4), Args: VariadicArgs, B, TLI));
4763 }
4764
4765 return nullptr;
4766}
4767
4768Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4769 IRBuilderBase &B) {
4770 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1)) {
4771 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 4));
4772 return copyFlags(Old: *CI,
4773 New: emitSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4774 VariadicArgs, B, TLI));
4775 }
4776
4777 return nullptr;
4778}
4779
4780Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4781 IRBuilderBase &B) {
4782 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2))
4783 return copyFlags(
4784 Old: *CI, New: emitStrCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1), B, TLI));
4785
4786 return nullptr;
4787}
4788
4789Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4790 IRBuilderBase &B) {
4791 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4792 return copyFlags(Old: *CI,
4793 New: emitStrLCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4794 Size: CI->getArgOperand(i: 2), B, TLI));
4795
4796 return nullptr;
4797}
4798
4799Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4800 IRBuilderBase &B) {
4801 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4802 return copyFlags(Old: *CI,
4803 New: emitStrNCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4804 Size: CI->getArgOperand(i: 2), B, TLI));
4805
4806 return nullptr;
4807}
4808
4809Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4810 IRBuilderBase &B) {
4811 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4812 return copyFlags(Old: *CI,
4813 New: emitStrLCpy(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4814 Size: CI->getArgOperand(i: 2), B, TLI));
4815
4816 return nullptr;
4817}
4818
4819Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4820 IRBuilderBase &B) {
4821 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2))
4822 return copyFlags(
4823 Old: *CI, New: emitVSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4824 Fmt: CI->getArgOperand(i: 4), VAList: CI->getArgOperand(i: 5), B, TLI));
4825
4826 return nullptr;
4827}
4828
4829Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4830 IRBuilderBase &B) {
4831 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1))
4832 return copyFlags(Old: *CI,
4833 New: emitVSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4834 VAList: CI->getArgOperand(i: 4), B, TLI));
4835
4836 return nullptr;
4837}
4838
4839Value *FortifiedLibCallSimplifier::optimizeCall(CallInst *CI,
4840 IRBuilderBase &Builder) {
4841 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4842 // Some clang users checked for _chk libcall availability using:
4843 // __has_builtin(__builtin___memcpy_chk)
4844 // When compiling with -fno-builtin, this is always true.
4845 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4846 // end up with fortified libcalls, which isn't acceptable in a freestanding
4847 // environment which only provides their non-fortified counterparts.
4848 //
4849 // Until we change clang and/or teach external users to check for availability
4850 // differently, disregard the "nobuiltin" attribute and TLI::has.
4851 //
4852 // PR23093.
4853
4854 Function *Callee = CI->getCalledFunction();
4855 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4856
4857 SmallVector<OperandBundleDef, 2> OpBundles;
4858 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4859
4860 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4861 Builder.setDefaultOperandBundles(OpBundles);
4862
4863 // First, check that this is a known library functions and that the prototype
4864 // is correct.
4865 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4866 if (Func == NotLibFunc)
4867 return nullptr;
4868
4869 // We never change the calling convention.
4870 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4871 return nullptr;
4872
4873 switch (Func) {
4874 case LibFunc_memcpy_chk:
4875 return optimizeMemCpyChk(CI, B&: Builder);
4876 case LibFunc_mempcpy_chk:
4877 return optimizeMemPCpyChk(CI, B&: Builder);
4878 case LibFunc_memmove_chk:
4879 return optimizeMemMoveChk(CI, B&: Builder);
4880 case LibFunc_memset_chk:
4881 return optimizeMemSetChk(CI, B&: Builder);
4882 case LibFunc_stpcpy_chk:
4883 case LibFunc_strcpy_chk:
4884 return optimizeStrpCpyChk(CI, B&: Builder, Func);
4885 case LibFunc_strlen_chk:
4886 return optimizeStrLenChk(CI, B&: Builder);
4887 case LibFunc_stpncpy_chk:
4888 case LibFunc_strncpy_chk:
4889 return optimizeStrpNCpyChk(CI, B&: Builder, Func);
4890 case LibFunc_memccpy_chk:
4891 return optimizeMemCCpyChk(CI, B&: Builder);
4892 case LibFunc_snprintf_chk:
4893 return optimizeSNPrintfChk(CI, B&: Builder);
4894 case LibFunc_sprintf_chk:
4895 return optimizeSPrintfChk(CI, B&: Builder);
4896 case LibFunc_strcat_chk:
4897 return optimizeStrCatChk(CI, B&: Builder);
4898 case LibFunc_strlcat_chk:
4899 return optimizeStrLCat(CI, B&: Builder);
4900 case LibFunc_strncat_chk:
4901 return optimizeStrNCatChk(CI, B&: Builder);
4902 case LibFunc_strlcpy_chk:
4903 return optimizeStrLCpyChk(CI, B&: Builder);
4904 case LibFunc_vsnprintf_chk:
4905 return optimizeVSNPrintfChk(CI, B&: Builder);
4906 case LibFunc_vsprintf_chk:
4907 return optimizeVSPrintfChk(CI, B&: Builder);
4908 default:
4909 break;
4910 }
4911 return nullptr;
4912}
4913
4914FortifiedLibCallSimplifier::FortifiedLibCallSimplifier(
4915 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4916 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
4917