1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/SmallString.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/Analysis/ConstantFolding.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/Utils/Local.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/AttributeMask.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/MDBuilder.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/PatternMatch.h"
34#include "llvm/IR/ProfDataUtils.h"
35#include "llvm/Support/Casting.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Support/KnownBits.h"
38#include "llvm/Support/KnownFPClass.h"
39#include "llvm/Support/MathExtras.h"
40#include "llvm/TargetParser/Triple.h"
41#include "llvm/Transforms/Utils/BuildLibCalls.h"
42#include "llvm/Transforms/Utils/Local.h"
43#include "llvm/Transforms/Utils/SizeOpts.h"
44
45#include <cmath>
46
47using namespace llvm;
48using namespace PatternMatch;
49
50#define DEBUG_TYPE "simplify-lib-calls"
51
52static cl::opt<bool>
53 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
54 cl::init(Val: false),
55 cl::desc("Enable unsafe double to float "
56 "shrinking for math lib calls"));
57
58// Enable conversion of operator new calls with a MemProf hot or cold hint
59// to an operator new call that takes a hot/cold hint. Off by default since
60// not all allocators currently support this extension.
61static cl::opt<bool>
62 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(Val: false),
63 cl::desc("Enable hot/cold operator new library calls"));
64enum class OptimizeExistingHotColdNewKind {
65 None,
66 Cold,
67 Always,
68};
69static cl::opt<OptimizeExistingHotColdNewKind> OptimizeExistingHotColdNew(
70 "optimize-existing-hot-cold-new", cl::Hidden,
71 cl::desc(
72 "Enable optimization of existing hot/cold operator new library calls"),
73 cl::values(
74 clEnumValN(
75 OptimizeExistingHotColdNewKind::None, "none",
76 "Do not optimize existing hot/cold operator new library calls"),
77 clEnumValN(OptimizeExistingHotColdNewKind::Cold, "cold",
78 "Only optimize existing hot/cold operator new library calls "
79 "if determined to be cold"),
80 clEnumValN(
81 OptimizeExistingHotColdNewKind::Always, "always",
82 "Always optimize existing hot/cold operator new library calls"),
83 clEnumValN(
84 OptimizeExistingHotColdNewKind::Always, "",
85 "Always optimize existing hot/cold operator new library calls")),
86 cl::init(Val: OptimizeExistingHotColdNewKind::None), cl::ValueOptional);
87static cl::opt<bool> OptimizeNoBuiltinHotColdNew(
88 "optimize-nobuiltin-hot-cold-new-new", cl::Hidden, cl::init(Val: false),
89 cl::desc("Enable transformation of nobuiltin operator new library calls"));
90static cl::opt<bool> MinExistingHotColdNewHint(
91 "min-existing-hot-cold-new-hint", cl::Hidden, cl::init(Val: false),
92 cl::desc("Take the minimum of compiler hint and existing hint when "
93 "optimizing existing hot/cold operator new library calls"));
94
95namespace llvm {
96extern cl::opt<bool> ProfcheckDisableMetadataFixes;
97} // namespace llvm
98
99namespace {
100
101// Specialized parser to ensure the hint is an 8 bit value (we can't specify
102// uint8_t to opt<> as that is interpreted to mean that we are passing a char
103// option with a specific set of values.
104struct HotColdHintParser : public cl::parser<unsigned> {
105 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
106
107 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
108 if (Arg.getAsInteger(Radix: 0, Result&: Value))
109 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
110
111 if (Value > 255)
112 return O.error(Message: "'" + Arg + "' value must be in the range [0, 255]!");
113
114 return false;
115 }
116};
117
118} // end anonymous namespace
119
120// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
121// and 255 is the hottest. Default to 1 value away from the coldest and hottest
122// hints, so that the compiler hinted allocations are slightly less strong than
123// manually inserted hints at the two extremes.
124static cl::opt<unsigned, false, HotColdHintParser> ColdNewHintValue(
125 "cold-new-hint-value", cl::Hidden, cl::init(Val: 1),
126 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
127static cl::opt<unsigned, false, HotColdHintParser>
128 NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(Val: 128),
129 cl::desc("Value to pass to hot/cold operator new for "
130 "notcold (warm) allocation"));
131static cl::opt<unsigned, false, HotColdHintParser> HotNewHintValue(
132 "hot-new-hint-value", cl::Hidden, cl::init(Val: 254),
133 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
134static cl::opt<unsigned, false, HotColdHintParser> AmbiguousNewHintValue(
135 "ambiguous-new-hint-value", cl::Hidden, cl::init(Val: 222),
136 cl::desc(
137 "Value to pass to hot/cold operator new for ambiguous allocation"));
138
139//===----------------------------------------------------------------------===//
140// Helper Functions
141//===----------------------------------------------------------------------===//
142
143static bool ignoreCallingConv(LibFunc Func) {
144 return Func == LibFunc_abs || Func == LibFunc_labs ||
145 Func == LibFunc_llabs || Func == LibFunc_strlen;
146}
147
148/// Return true if it is only used in equality comparisons with With.
149static bool isOnlyUsedInEqualityComparison(Value *V, Value *With) {
150 for (User *U : V->users()) {
151 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
152 if (IC->isEquality() && IC->getOperand(i_nocapture: 1) == With)
153 continue;
154 // Unknown instruction.
155 return false;
156 }
157 return true;
158}
159
160static bool callHasFloatingPointArgument(const CallInst *CI) {
161 return any_of(Range: CI->operands(), P: [](const Use &OI) {
162 return OI->getType()->isFloatingPointTy();
163 });
164}
165
166static bool callHasFP128Argument(const CallInst *CI) {
167 return any_of(Range: CI->operands(), P: [](const Use &OI) {
168 return OI->getType()->isFP128Ty();
169 });
170}
171
172// Convert the entire string Str representing an integer in Base, up to
173// the terminating nul if present, to a constant according to the rules
174// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
175// return the result, otherwise null.
176// The function assumes the string is encoded in ASCII and carefully
177// avoids converting sequences (including "") that the corresponding
178// library call might fail and set errno for.
179static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
180 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
181 if (Base < 2 || Base > 36)
182 if (Base != 0)
183 // Fail for an invalid base (required by POSIX).
184 return nullptr;
185
186 // Current offset into the original string to reflect in EndPtr.
187 size_t Offset = 0;
188 // Strip leading whitespace.
189 for ( ; Offset != Str.size(); ++Offset)
190 if (!isSpace(C: (unsigned char)Str[Offset])) {
191 Str = Str.substr(Start: Offset);
192 break;
193 }
194
195 if (Str.empty())
196 // Fail for empty subject sequences (POSIX allows but doesn't require
197 // strtol[l]/strtoul[l] to fail with EINVAL).
198 return nullptr;
199
200 // Strip but remember the sign.
201 bool Negate = Str[0] == '-';
202 if (Str[0] == '-' || Str[0] == '+') {
203 Str = Str.drop_front();
204 if (Str.empty())
205 // Fail for a sign with nothing after it.
206 return nullptr;
207 ++Offset;
208 }
209
210 // Set Max to the absolute value of the minimum (for signed), or
211 // to the maximum (for unsigned) value representable in the type.
212 Type *RetTy = CI->getType();
213 unsigned NBits = RetTy->getPrimitiveSizeInBits();
214 uint64_t Max = AsSigned && Negate ? 1 : 0;
215 Max += AsSigned ? maxIntN(N: NBits) : maxUIntN(N: NBits);
216
217 // Autodetect Base if it's zero and consume the "0x" prefix.
218 if (Str.size() > 1) {
219 if (Str[0] == '0') {
220 if (toUpper(x: (unsigned char)Str[1]) == 'X') {
221 if (Str.size() == 2 || (Base && Base != 16))
222 // Fail if Base doesn't allow the "0x" prefix or for the prefix
223 // alone that implementations like BSD set errno to EINVAL for.
224 return nullptr;
225
226 Str = Str.drop_front(N: 2);
227 Offset += 2;
228 Base = 16;
229 }
230 else if (Base == 0)
231 Base = 8;
232 } else if (Base == 0)
233 Base = 10;
234 }
235 else if (Base == 0)
236 Base = 10;
237
238 // Convert the rest of the subject sequence, not including the sign,
239 // to its uint64_t representation (this assumes the source character
240 // set is ASCII).
241 uint64_t Result = 0;
242 for (unsigned i = 0; i != Str.size(); ++i) {
243 unsigned char DigVal = Str[i];
244 if (isDigit(C: DigVal))
245 DigVal = DigVal - '0';
246 else {
247 DigVal = toUpper(x: DigVal);
248 if (isAlpha(C: DigVal))
249 DigVal = DigVal - 'A' + 10;
250 else
251 return nullptr;
252 }
253
254 if (DigVal >= Base)
255 // Fail if the digit is not valid in the Base.
256 return nullptr;
257
258 // Add the digit and fail if the result is not representable in
259 // the (unsigned form of the) destination type.
260 bool VFlow;
261 Result = SaturatingMultiplyAdd(X: Result, Y: Base, A: (uint64_t)DigVal, ResultOverflowed: &VFlow);
262 if (VFlow || Result > Max)
263 return nullptr;
264 }
265
266 if (EndPtr) {
267 // Store the pointer to the end.
268 Value *Off = B.getInt64(C: Offset + Str.size());
269 Value *StrBeg = CI->getArgOperand(i: 0);
270 Value *StrEnd = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: StrBeg, IdxList: Off, Name: "endptr");
271 B.CreateStore(Val: StrEnd, Ptr: EndPtr);
272 }
273
274 if (Negate) {
275 // Unsigned negation doesn't overflow.
276 Result = -Result;
277 // For unsigned numbers, discard sign bits.
278 if (!AsSigned)
279 Result &= maxUIntN(N: NBits);
280 }
281
282 return ConstantInt::get(Ty: RetTy, V: Result, IsSigned: AsSigned);
283}
284
285static bool isOnlyUsedInComparisonWithZero(Value *V) {
286 for (User *U : V->users()) {
287 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
288 if (Constant *C = dyn_cast<Constant>(Val: IC->getOperand(i_nocapture: 1)))
289 if (C->isNullValue())
290 continue;
291 // Unknown instruction.
292 return false;
293 }
294 return true;
295}
296
297static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
298 const SimplifyQuery &SQ) {
299 if (!isOnlyUsedInComparisonWithZero(V: CI))
300 return false;
301
302 if (!isDereferenceablePointer(V: Str, Size: APInt(64, Len), Q: SQ))
303 return false;
304
305 if (CI->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemory))
306 return false;
307
308 return true;
309}
310
311static void annotateDereferenceableBytes(CallInst *CI,
312 ArrayRef<unsigned> ArgNos,
313 uint64_t DereferenceableBytes) {
314 const Function *F = CI->getCaller();
315 if (!F)
316 return;
317 for (unsigned ArgNo : ArgNos) {
318 uint64_t DerefBytes = DereferenceableBytes;
319 unsigned AS = CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
320 if (!llvm::NullPointerIsDefined(F, AS) ||
321 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
322 DerefBytes = std::max(a: CI->getParamDereferenceableOrNullBytes(i: ArgNo),
323 b: DereferenceableBytes);
324
325 if (CI->getParamDereferenceableBytes(i: ArgNo) < DerefBytes) {
326 CI->removeParamAttr(ArgNo, Kind: Attribute::Dereferenceable);
327 if (!llvm::NullPointerIsDefined(F, AS) ||
328 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
329 CI->removeParamAttr(ArgNo, Kind: Attribute::DereferenceableOrNull);
330 CI->addParamAttr(ArgNo, Attr: Attribute::getWithDereferenceableBytes(
331 Context&: CI->getContext(), Bytes: DerefBytes));
332 }
333 }
334}
335
336static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI,
337 ArrayRef<unsigned> ArgNos) {
338 Function *F = CI->getCaller();
339 if (!F)
340 return;
341
342 for (unsigned ArgNo : ArgNos) {
343 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NoUndef))
344 CI->addParamAttr(ArgNo, Kind: Attribute::NoUndef);
345
346 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull)) {
347 unsigned AS =
348 CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
349 if (llvm::NullPointerIsDefined(F, AS))
350 continue;
351 CI->addParamAttr(ArgNo, Kind: Attribute::NonNull);
352 }
353
354 annotateDereferenceableBytes(CI, ArgNos: ArgNo, DereferenceableBytes: 1);
355 }
356}
357
358static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> ArgNos,
359 Value *Size, const DataLayout &DL) {
360 if (ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size)) {
361 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
362 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: LenC->getZExtValue());
363 } else if (isKnownNonZero(V: Size, Q: DL)) {
364 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
365 uint64_t X, Y;
366 uint64_t DerefMin = 1;
367 if (match(V: Size, P: m_Select(C: m_Value(), L: m_ConstantInt(V&: X), R: m_ConstantInt(V&: Y)))) {
368 DerefMin = std::min(a: X, b: Y);
369 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: DerefMin);
370 }
371 }
372}
373
374// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
375// easier chaining. Calls to emit* and B.createCall should probably be wrapped
376// in this function when New is created to replace Old. Callers should take
377// care to check Old.isMustTailCall() if they aren't replacing Old directly
378// with New.
379static Value *copyFlags(const CallInst &Old, Value *New) {
380 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
381 assert(!Old.isNoTailCall() && "do not copy notail call flags");
382 if (auto *NewCI = dyn_cast_or_null<CallInst>(Val: New))
383 NewCI->setTailCallKind(Old.getTailCallKind());
384 return New;
385}
386
387static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
388 NewCI->setAttributes(AttributeList::get(
389 C&: NewCI->getContext(), Attrs: {NewCI->getAttributes(), Old.getAttributes()}));
390 NewCI->removeRetAttrs(AttrsToRemove: AttributeFuncs::typeIncompatible(
391 Ty: NewCI->getType(), AS: NewCI->getRetAttributes()));
392 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
393 NewCI->removeParamAttrs(
394 ArgNo: I, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewCI->getArgOperand(i: I)->getType(),
395 AS: NewCI->getParamAttributes(ArgNo: I)));
396
397 return copyFlags(Old, New: NewCI);
398}
399
400// Helper to avoid truncating the length if size_t is 32-bits.
401static StringRef substr(StringRef Str, uint64_t Len) {
402 return Len >= Str.size() ? Str : Str.substr(Start: 0, N: Len);
403}
404
405//===----------------------------------------------------------------------===//
406// String and Memory Library Call Optimizations
407//===----------------------------------------------------------------------===//
408
409Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
410 // Extract some information from the instruction
411 Value *Dst = CI->getArgOperand(i: 0);
412 Value *Src = CI->getArgOperand(i: 1);
413 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
414
415 // See if we can get the length of the input string.
416 uint64_t Len = GetStringLength(V: Src);
417 if (Len)
418 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
419 else
420 return nullptr;
421 --Len; // Unbias length.
422
423 // Handle the simple, do-nothing case: strcat(x, "") -> x
424 if (Len == 0)
425 return Dst;
426
427 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len, B));
428}
429
430Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
431 IRBuilderBase &B) {
432 // We need to find the end of the destination string. That's where the
433 // memory is to be moved to. We just generate a call to strlen.
434 Value *DstLen = emitStrLen(Ptr: Dst, B, DL, TLI);
435 if (!DstLen)
436 return nullptr;
437
438 // Now that we have the destination's length, we must index into the
439 // destination's pointer to get the actual memcpy destination (end of
440 // the string .. we're concatenating).
441 Value *CpyDst = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: DstLen, Name: "endptr");
442
443 // We have enough information to now generate the memcpy call to do the
444 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
445 B.CreateMemCpy(Dst: CpyDst, DstAlign: Align(1), Src, SrcAlign: Align(1),
446 Size: TLI->getAsSizeT(V: Len + 1, M: *B.getModule()));
447 return Dst;
448}
449
450Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
451 // Extract some information from the instruction.
452 Value *Dst = CI->getArgOperand(i: 0);
453 Value *Src = CI->getArgOperand(i: 1);
454 Value *Size = CI->getArgOperand(i: 2);
455 uint64_t Len;
456 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
457 if (isKnownNonZero(V: Size, Q: DL))
458 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
459
460 // We don't do anything if length is not constant.
461 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size);
462 if (LengthArg) {
463 Len = LengthArg->getZExtValue();
464 // strncat(x, c, 0) -> x
465 if (!Len)
466 return Dst;
467 } else {
468 return nullptr;
469 }
470
471 // See if we can get the length of the input string.
472 uint64_t SrcLen = GetStringLength(V: Src);
473 if (SrcLen) {
474 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
475 --SrcLen; // Unbias length.
476 } else {
477 return nullptr;
478 }
479
480 // strncat(x, "", c) -> x
481 if (SrcLen == 0)
482 return Dst;
483
484 // We don't optimize this case.
485 if (Len < SrcLen)
486 return nullptr;
487
488 // strncat(x, s, c) -> strcat(x, s)
489 // s is constant so the strcat can be optimized further.
490 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len: SrcLen, B));
491}
492
493// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
494// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
495// is that either S is dereferenceable or the value of N is nonzero.
496static Value* memChrToCharCompare(CallInst *CI, Value *NBytes,
497 IRBuilderBase &B, const DataLayout &DL)
498{
499 Value *Src = CI->getArgOperand(i: 0);
500 Value *CharVal = CI->getArgOperand(i: 1);
501
502 // Fold memchr(A, C, N) == A to N && *A == C.
503 Type *CharTy = B.getInt8Ty();
504 Value *Char0 = B.CreateLoad(Ty: CharTy, Ptr: Src);
505 CharVal = B.CreateTrunc(V: CharVal, DestTy: CharTy);
506 Value *Cmp = B.CreateICmpEQ(LHS: Char0, RHS: CharVal, Name: "char0cmp");
507
508 if (NBytes) {
509 Value *Zero = ConstantInt::get(Ty: NBytes->getType(), V: 0);
510 Value *And = B.CreateICmpNE(LHS: NBytes, RHS: Zero);
511 Cmp = B.CreateLogicalAnd(Cond1: And, Cond2: Cmp);
512 // The and above is based on the byte count and the query, neither of which
513 // we know without value profiling, so mark the profile as unknown.
514 if (auto *SI = dyn_cast<SelectInst>(Val: Cmp))
515 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE);
516 }
517
518 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
519 return B.CreateSelect(C: Cmp, True: Src, False: NullPtr);
520}
521
522Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
523 Value *SrcStr = CI->getArgOperand(i: 0);
524 Value *CharVal = CI->getArgOperand(i: 1);
525 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
526
527 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
528 return memChrToCharCompare(CI, NBytes: nullptr, B, DL);
529
530 // If the second operand is non-constant, see if we can compute the length
531 // of the input string and turn this into memchr.
532 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
533 if (!CharC) {
534 uint64_t Len = GetStringLength(V: SrcStr);
535 if (Len)
536 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len);
537 else
538 return nullptr;
539
540 Function *Callee = CI->getCalledFunction();
541 FunctionType *FT = Callee->getFunctionType();
542 unsigned IntBits = TLI->getIntSize();
543 if (!FT->getParamType(i: 1)->isIntegerTy(BitWidth: IntBits)) // memchr needs 'int'.
544 return nullptr;
545
546 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
547 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
548 return copyFlags(Old: *CI,
549 New: emitMemChr(Ptr: SrcStr, Val: CharVal, // include nul.
550 Len: ConstantInt::get(Ty: SizeTTy, V: Len), B,
551 DL, TLI));
552 }
553
554 if (CharC->isZero()) {
555 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
556 if (isOnlyUsedInEqualityComparison(V: CI, With: NullPtr))
557 // Pre-empt the transformation to strlen below and fold
558 // strchr(A, '\0') == null to false.
559 return B.CreateIntToPtr(V: B.getTrue(), DestTy: CI->getType());
560 }
561
562 // Otherwise, the character is a constant, see if the first argument is
563 // a string literal. If so, we can constant fold.
564 StringRef Str;
565 if (!getConstantStringInfo(V: SrcStr, Str)) {
566 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
567 if (Value *StrLen = emitStrLen(Ptr: SrcStr, B, DL, TLI))
568 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: StrLen, Name: "strchr");
569 return nullptr;
570 }
571
572 // Compute the offset, make sure to handle the case when we're searching for
573 // zero (a weird way to spell strlen).
574 size_t I = (0xFF & CharC->getSExtValue()) == 0
575 ? Str.size()
576 : Str.find(C: CharC->getSExtValue());
577 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
578 return Constant::getNullValue(Ty: CI->getType());
579
580 // strchr(s+n,c) -> gep(s+n+i,c)
581 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: I), Name: "strchr");
582}
583
584Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
585 Value *SrcStr = CI->getArgOperand(i: 0);
586 Value *CharVal = CI->getArgOperand(i: 1);
587 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
588 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
589
590 StringRef Str;
591 if (!getConstantStringInfo(V: SrcStr, Str)) {
592 // strrchr(s, 0) -> strchr(s, 0)
593 if (CharC && CharC->isZero())
594 return copyFlags(Old: *CI, New: emitStrChr(Ptr: SrcStr, C: '\0', B, TLI));
595 return nullptr;
596 }
597
598 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
599 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
600
601 // Try to expand strrchr to the memrchr nonstandard extension if it's
602 // available, or simply fail otherwise.
603 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
604 Value *Size = ConstantInt::get(Ty: SizeTTy, V: NBytes);
605 return copyFlags(Old: *CI, New: emitMemRChr(Ptr: SrcStr, Val: CharVal, Len: Size, B, DL, TLI));
606}
607
608Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
609 Value *Str1P = CI->getArgOperand(i: 0), *Str2P = CI->getArgOperand(i: 1);
610 if (Str1P == Str2P) // strcmp(x,x) -> 0
611 return ConstantInt::get(Ty: CI->getType(), V: 0);
612
613 StringRef Str1, Str2;
614 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
615 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
616
617 // strcmp(x, y) -> cnst (if both x and y are constant strings)
618 if (HasStr1 && HasStr2)
619 return ConstantInt::getSigned(Ty: CI->getType(),
620 V: std::clamp(val: Str1.compare(RHS: Str2), lo: -1, hi: 1));
621
622 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
623 return B.CreateNeg(V: B.CreateZExt(
624 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
625
626 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
627 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
628 DestTy: CI->getType());
629
630 // strcmp(P, "x") -> memcmp(P, "x", 2)
631 uint64_t Len1 = GetStringLength(V: Str1P);
632 if (Len1)
633 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
634 uint64_t Len2 = GetStringLength(V: Str2P);
635 if (Len2)
636 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
637
638 if (Len1 && Len2) {
639 return copyFlags(
640 Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
641 Len: TLI->getAsSizeT(V: std::min(a: Len1, b: Len2), M: *CI->getModule()),
642 B, DL, TLI));
643 }
644
645 // strcmp to memcmp
646 SimplifyQuery SQ(DL, TLI, DT, AC, CI);
647 if (!HasStr1 && HasStr2) {
648 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ))
649 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
650 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
651 B, DL, TLI));
652 } else if (HasStr1 && !HasStr2) {
653 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ))
654 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
655 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
656 B, DL, TLI));
657 }
658
659 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
660 return nullptr;
661}
662
663// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
664// arrays LHS and RHS and nonconstant Size.
665static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
666 Value *Size, bool StrNCmp,
667 IRBuilderBase &B, const DataLayout &DL);
668
669Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
670 Value *Str1P = CI->getArgOperand(i: 0);
671 Value *Str2P = CI->getArgOperand(i: 1);
672 Value *Size = CI->getArgOperand(i: 2);
673 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
674 return ConstantInt::get(Ty: CI->getType(), V: 0);
675
676 if (isKnownNonZero(V: Size, Q: DL))
677 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
678 // Get the length argument if it is constant.
679 uint64_t Length;
680 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size))
681 Length = LengthArg->getZExtValue();
682 else
683 return optimizeMemCmpVarSize(CI, LHS: Str1P, RHS: Str2P, Size, StrNCmp: true, B, DL);
684
685 if (Length == 0) // strncmp(x,y,0) -> 0
686 return ConstantInt::get(Ty: CI->getType(), V: 0);
687
688 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
689 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P, Len: Size, B, DL, TLI));
690
691 StringRef Str1, Str2;
692 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
693 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
694
695 // strncmp(x, y) -> cnst (if both x and y are constant strings)
696 if (HasStr1 && HasStr2) {
697 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
698 StringRef SubStr1 = substr(Str: Str1, Len: Length);
699 StringRef SubStr2 = substr(Str: Str2, Len: Length);
700 return ConstantInt::getSigned(Ty: CI->getType(),
701 V: std::clamp(val: SubStr1.compare(RHS: SubStr2), lo: -1, hi: 1));
702 }
703
704 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
705 return B.CreateNeg(V: B.CreateZExt(
706 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
707
708 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
709 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
710 DestTy: CI->getType());
711
712 uint64_t Len1 = GetStringLength(V: Str1P);
713 if (Len1)
714 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
715 uint64_t Len2 = GetStringLength(V: Str2P);
716 if (Len2)
717 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
718
719 // strncmp to memcmp
720 if (!HasStr1 && HasStr2) {
721 Len2 = std::min(a: Len2, b: Length);
722 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ: DL))
723 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
724 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
725 B, DL, TLI));
726 } else if (HasStr1 && !HasStr2) {
727 Len1 = std::min(a: Len1, b: Length);
728 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ: DL))
729 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
730 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
731 B, DL, TLI));
732 }
733
734 return nullptr;
735}
736
737Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
738 Value *Src = CI->getArgOperand(i: 0);
739 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
740 uint64_t SrcLen = GetStringLength(V: Src);
741 if (SrcLen && Size) {
742 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: SrcLen);
743 if (SrcLen <= Size->getZExtValue() + 1)
744 return copyFlags(Old: *CI, New: emitStrDup(Ptr: Src, B, TLI));
745 }
746
747 return nullptr;
748}
749
750Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
751 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
752 if (Dst == Src) // strcpy(x,x) -> x
753 return Src;
754
755 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
756 // See if we can get the length of the input string.
757 uint64_t Len = GetStringLength(V: Src);
758 if (Len)
759 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
760 else
761 return nullptr;
762
763 // We have enough information to now generate the memcpy call to do the
764 // copy for us. Make a memcpy to copy the nul byte with align = 1.
765 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
766 Size: TLI->getAsSizeT(V: Len, M: *CI->getModule()));
767 mergeAttributesAndFlags(NewCI, Old: *CI);
768 return Dst;
769}
770
771Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
772 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
773
774 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
775 if (CI->use_empty())
776 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
777
778 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
779 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
780 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
781 }
782
783 // See if we can get the length of the input string.
784 uint64_t Len = GetStringLength(V: Src);
785 if (Len)
786 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
787 else
788 return nullptr;
789
790 Value *LenV = TLI->getAsSizeT(V: Len, M: *CI->getModule());
791 Value *DstEnd = B.CreateInBoundsGEP(
792 Ty: B.getInt8Ty(), Ptr: Dst, IdxList: TLI->getAsSizeT(V: Len - 1, M: *CI->getModule()));
793
794 // We have enough information to now generate the memcpy call to do the
795 // copy for us. Make a memcpy to copy the nul byte with align = 1.
796 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: LenV);
797 mergeAttributesAndFlags(NewCI, Old: *CI);
798 return DstEnd;
799}
800
801// Optimize a call to size_t strlcpy(char*, const char*, size_t).
802
803Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
804 Value *Size = CI->getArgOperand(i: 2);
805 if (isKnownNonZero(V: Size, Q: DL))
806 // Like snprintf, the function stores into the destination only when
807 // the size argument is nonzero.
808 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
809 // The function reads the source argument regardless of Size (it returns
810 // its length).
811 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
812
813 uint64_t NBytes;
814 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
815 NBytes = SizeC->getZExtValue();
816 else
817 return nullptr;
818
819 Value *Dst = CI->getArgOperand(i: 0);
820 Value *Src = CI->getArgOperand(i: 1);
821 if (NBytes <= 1) {
822 if (NBytes == 1)
823 // For a call to strlcpy(D, S, 1) first store a nul in *D.
824 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
825
826 // Transform strlcpy(D, S, 0) to a call to strlen(S).
827 return copyFlags(Old: *CI, New: emitStrLen(Ptr: Src, B, DL, TLI));
828 }
829
830 // Try to determine the length of the source, substituting its size
831 // when it's not nul-terminated (as it's required to be) to avoid
832 // reading past its end.
833 StringRef Str;
834 if (!getConstantStringInfo(V: Src, Str, /*TrimAtNul=*/false))
835 return nullptr;
836
837 uint64_t SrcLen = Str.find(C: '\0');
838 // Set if the terminating nul should be copied by the call to memcpy
839 // below.
840 bool NulTerm = SrcLen < NBytes;
841
842 if (NulTerm)
843 // Overwrite NBytes with the number of bytes to copy, including
844 // the terminating nul.
845 NBytes = SrcLen + 1;
846 else {
847 // Set the length of the source for the function to return to its
848 // size, and cap NBytes at the same.
849 SrcLen = std::min(a: SrcLen, b: uint64_t(Str.size()));
850 NBytes = std::min(a: NBytes - 1, b: SrcLen);
851 }
852
853 if (SrcLen == 0) {
854 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
855 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
856 return ConstantInt::get(Ty: CI->getType(), V: 0);
857 }
858
859 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
860 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
861 // D[N' - 1] if necessary.
862 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
863 Size: TLI->getAsSizeT(V: NBytes, M: *CI->getModule()));
864 mergeAttributesAndFlags(NewCI, Old: *CI);
865
866 if (!NulTerm) {
867 Value *EndOff = ConstantInt::get(Ty: CI->getType(), V: NBytes);
868 Value *EndPtr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: EndOff);
869 B.CreateStore(Val: B.getInt8(C: 0), Ptr: EndPtr);
870 }
871
872 // Like snprintf, strlcpy returns the number of nonzero bytes that would
873 // have been copied if the bound had been sufficiently big (which in this
874 // case is strlen(Src)).
875 return ConstantInt::get(Ty: CI->getType(), V: SrcLen);
876}
877
878// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
879// otherwise.
880Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
881 IRBuilderBase &B) {
882 Value *Dst = CI->getArgOperand(i: 0);
883 Value *Src = CI->getArgOperand(i: 1);
884 Value *Size = CI->getArgOperand(i: 2);
885
886 if (isKnownNonZero(V: Size, Q: DL)) {
887 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
888 // only when N is nonzero.
889 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
890 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
891 }
892
893 // If the "bound" argument is known set N to it. Otherwise set it to
894 // UINT64_MAX and handle it later.
895 uint64_t N = UINT64_MAX;
896 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
897 N = SizeC->getZExtValue();
898
899 if (N == 0)
900 // Fold st{p,r}ncpy(D, S, 0) to D.
901 return Dst;
902
903 if (N == 1) {
904 Type *CharTy = B.getInt8Ty();
905 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "stxncpy.char0");
906 B.CreateStore(Val: CharVal, Ptr: Dst);
907 if (!RetEnd)
908 // Transform strncpy(D, S, 1) to return (*D = *S), D.
909 return Dst;
910
911 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
912 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
913 Value *Cmp = B.CreateICmpEQ(LHS: CharVal, RHS: ZeroChar, Name: "stpncpy.char0cmp");
914
915 Value *Off1 = B.getInt32(C: 1);
916 Value *EndPtr = B.CreateInBoundsGEP(Ty: CharTy, Ptr: Dst, IdxList: Off1, Name: "stpncpy.end");
917 return B.CreateSelect(C: Cmp, True: Dst, False: EndPtr, Name: "stpncpy.sel");
918 }
919
920 // If the length of the input string is known set SrcLen to it.
921 uint64_t SrcLen = GetStringLength(V: Src);
922 if (SrcLen)
923 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
924 else
925 return nullptr;
926
927 --SrcLen; // Unbias length.
928
929 if (SrcLen == 0) {
930 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
931 Align MemSetAlign =
932 CI->getAttributes().getParamAttrs(ArgNo: 0).getAlignment().valueOrOne();
933 CallInst *NewCI = B.CreateMemSet(Ptr: Dst, Val: B.getInt8(C: '\0'), Size, Align: MemSetAlign);
934 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(ArgNo: 0));
935 NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
936 C&: CI->getContext(), ArgNo: 0, B: ArgAttrs));
937 copyFlags(Old: *CI, New: NewCI);
938 return Dst;
939 }
940
941 if (N > SrcLen + 1) {
942 if (N > 128)
943 // Bail if N is large or unknown.
944 return nullptr;
945
946 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
947 StringRef Str;
948 if (!getConstantStringInfo(V: Src, Str))
949 return nullptr;
950 std::string SrcStr = Str.str();
951 // Create a bigger, nul-padded array with the same length, SrcLen,
952 // as the original string.
953 SrcStr.resize(n: N, c: '\0');
954 Src = B.CreateGlobalString(Str: SrcStr, Name: "str", /*AddressSpace=*/0,
955 /*M=*/nullptr, /*AddNull=*/false);
956 }
957
958 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
959 // S and N are constant.
960 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
961 Size: TLI->getAsSizeT(V: N, M: *CI->getModule()));
962 mergeAttributesAndFlags(NewCI, Old: *CI);
963 if (!RetEnd)
964 return Dst;
965
966 // stpncpy(D, S, N) returns the address of the first null in D if it writes
967 // one, otherwise D + N.
968 Value *Off = B.getInt64(C: std::min(a: SrcLen, b: N));
969 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: Off, Name: "endptr");
970}
971
972Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
973 unsigned CharSize,
974 Value *Bound) {
975 Value *Src = CI->getArgOperand(i: 0);
976 Type *CharTy = B.getIntNTy(N: CharSize);
977
978 if (isOnlyUsedInZeroEqualityComparison(CtxI: CI) &&
979 (!Bound || isKnownNonZero(V: Bound, Q: DL))) {
980 // Fold strlen:
981 // strlen(x) != 0 --> *x != 0
982 // strlen(x) == 0 --> *x == 0
983 // and likewise strnlen with constant N > 0:
984 // strnlen(x, N) != 0 --> *x != 0
985 // strnlen(x, N) == 0 --> *x == 0
986 return B.CreateZExt(V: B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "char0"),
987 DestTy: CI->getType());
988 }
989
990 if (Bound) {
991 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Val: Bound)) {
992 if (BoundCst->isZero())
993 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
994 return ConstantInt::get(Ty: CI->getType(), V: 0);
995
996 if (BoundCst->isOne()) {
997 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
998 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "strnlen.char0");
999 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
1000 Value *Cmp = B.CreateICmpNE(LHS: CharVal, RHS: ZeroChar, Name: "strnlen.char0cmp");
1001 return B.CreateZExt(V: Cmp, DestTy: CI->getType());
1002 }
1003 }
1004 }
1005
1006 if (uint64_t Len = GetStringLength(V: Src, CharSize)) {
1007 Value *LenC = ConstantInt::get(Ty: CI->getType(), V: Len - 1);
1008 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
1009 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
1010 if (Bound)
1011 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: LenC, RHS: Bound);
1012 return LenC;
1013 }
1014
1015 if (Bound)
1016 // Punt for strnlen for now.
1017 return nullptr;
1018
1019 // If s is a constant pointer pointing to a string literal, we can fold
1020 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
1021 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
1022 // We only try to simplify strlen when the pointer s points to an array
1023 // of CharSize elements. Otherwise, we would need to scale the offset x before
1024 // doing the subtraction. This will make the optimization more complex, and
1025 // it's not very useful because calling strlen for a pointer of other types is
1026 // very uncommon.
1027 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: Src)) {
1028 unsigned BW = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1029 SmallMapVector<Value *, APInt, 4> VarOffsets;
1030 APInt ConstOffset(BW, 0);
1031 assert(CharSize % 8 == 0 && "Expected a multiple of 8 sized CharSize");
1032 // Check the gep is a single variable offset.
1033 if (!GEP->collectOffset(DL, BitWidth: BW, VariableOffsets&: VarOffsets, ConstantOffset&: ConstOffset) ||
1034 VarOffsets.size() != 1 || ConstOffset != 0 ||
1035 VarOffsets.begin()->second != CharSize / 8)
1036 return nullptr;
1037
1038 ConstantDataArraySlice Slice;
1039 if (getConstantDataArrayInfo(V: GEP->getOperand(i_nocapture: 0), Slice, ElementSize: CharSize)) {
1040 uint64_t NullTermIdx;
1041 if (Slice.Array == nullptr) {
1042 NullTermIdx = 0;
1043 } else {
1044 NullTermIdx = ~((uint64_t)0);
1045 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
1046 if (Slice.Array->getElementAsInteger(i: I + Slice.Offset) == 0) {
1047 NullTermIdx = I;
1048 break;
1049 }
1050 }
1051 // If the string does not have '\0', leave it to strlen to compute
1052 // its length.
1053 if (NullTermIdx == ~((uint64_t)0))
1054 return nullptr;
1055 }
1056
1057 Value *Offset = VarOffsets.begin()->first;
1058 KnownBits Known = computeKnownBits(V: Offset, DL, AC: nullptr, CtxI: CI, DT: nullptr);
1059
1060 // If Offset is not provably in the range [0, NullTermIdx], we can still
1061 // optimize if we can prove that the program has undefined behavior when
1062 // Offset is outside that range. That is the case when GEP->getOperand(0)
1063 // is a pointer to an object whose memory extent is NullTermIdx+1.
1064 if ((Known.isNonNegative() && Known.getMaxValue().ule(RHS: NullTermIdx)) ||
1065 (isa<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)) &&
1066 NullTermIdx == Slice.Length - 1)) {
1067 Offset = B.CreateSExtOrTrunc(V: Offset, DestTy: CI->getType());
1068 return B.CreateSub(LHS: ConstantInt::get(Ty: CI->getType(), V: NullTermIdx),
1069 RHS: Offset);
1070 }
1071 }
1072 }
1073
1074 // strlen(x?"foo":"bars") --> x ? 3 : 4
1075 if (SelectInst *SI = dyn_cast<SelectInst>(Val: Src)) {
1076 uint64_t LenTrue = GetStringLength(V: SI->getTrueValue(), CharSize);
1077 uint64_t LenFalse = GetStringLength(V: SI->getFalseValue(), CharSize);
1078 if (LenTrue && LenFalse) {
1079 ORE.emit(RemarkBuilder: [&]() {
1080 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1081 << "folded strlen(select) to select of constants";
1082 });
1083 return B.CreateSelect(C: SI->getCondition(),
1084 True: ConstantInt::get(Ty: CI->getType(), V: LenTrue - 1),
1085 False: ConstantInt::get(Ty: CI->getType(), V: LenFalse - 1), Name: "",
1086 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : SI);
1087 }
1088 }
1089
1090 return nullptr;
1091}
1092
1093Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1094 if (Value *V = optimizeStringLength(CI, B, CharSize: 8))
1095 return V;
1096 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1097 return nullptr;
1098}
1099
1100Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1101 Value *Bound = CI->getArgOperand(i: 1);
1102 if (Value *V = optimizeStringLength(CI, B, CharSize: 8, Bound))
1103 return V;
1104
1105 if (isKnownNonZero(V: Bound, Q: DL))
1106 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1107 return nullptr;
1108}
1109
1110Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1111 Module &M = *CI->getModule();
1112 unsigned WCharSize = TLI->getWCharSize(M) * 8;
1113 // We cannot perform this optimization without wchar_size metadata.
1114 if (WCharSize == 0)
1115 return nullptr;
1116
1117 return optimizeStringLength(CI, B, CharSize: WCharSize);
1118}
1119
1120Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1121 StringRef S1, S2;
1122 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1123 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1124
1125 // strpbrk(s, "") -> nullptr
1126 // strpbrk("", s) -> nullptr
1127 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1128 return Constant::getNullValue(Ty: CI->getType());
1129
1130 // Constant folding.
1131 if (HasS1 && HasS2) {
1132 size_t I = S1.find_first_of(Chars: S2);
1133 if (I == StringRef::npos) // No match.
1134 return Constant::getNullValue(Ty: CI->getType());
1135
1136 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1137 IdxList: B.getInt64(C: I), Name: "strpbrk");
1138 }
1139
1140 // strpbrk(s, "a") -> strchr(s, 'a')
1141 if (HasS2 && S2.size() == 1)
1142 return copyFlags(Old: *CI, New: emitStrChr(Ptr: CI->getArgOperand(i: 0), C: S2[0], B, TLI));
1143
1144 return nullptr;
1145}
1146
1147Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1148 Value *EndPtr = CI->getArgOperand(i: 1);
1149 if (isa<ConstantPointerNull>(Val: EndPtr)) {
1150 // With a null EndPtr, this function won't capture the main argument.
1151 // It would be readonly too, except that it still may write to errno.
1152 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
1153 CI: CaptureInfo::none()));
1154 }
1155
1156 return nullptr;
1157}
1158
1159Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1160 StringRef S1, S2;
1161 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1162 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1163
1164 // strspn(s, "") -> 0
1165 // strspn("", s) -> 0
1166 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1167 return Constant::getNullValue(Ty: CI->getType());
1168
1169 // Constant folding.
1170 if (HasS1 && HasS2) {
1171 size_t Pos = S1.find_first_not_of(Chars: S2);
1172 if (Pos == StringRef::npos)
1173 Pos = S1.size();
1174 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1175 }
1176
1177 return nullptr;
1178}
1179
1180Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1181 StringRef S1, S2;
1182 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1183 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1184
1185 // strcspn("", s) -> 0
1186 if (HasS1 && S1.empty())
1187 return Constant::getNullValue(Ty: CI->getType());
1188
1189 // Constant folding.
1190 if (HasS1 && HasS2) {
1191 size_t Pos = S1.find_first_of(Chars: S2);
1192 if (Pos == StringRef::npos)
1193 Pos = S1.size();
1194 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1195 }
1196
1197 // strcspn(s, "") -> strlen(s)
1198 if (HasS2 && S2.empty())
1199 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B, DL, TLI));
1200
1201 return nullptr;
1202}
1203
1204Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1205 // fold strstr(x, x) -> x.
1206 if (CI->getArgOperand(i: 0) == CI->getArgOperand(i: 1))
1207 return CI->getArgOperand(i: 0);
1208
1209 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1210 if (isOnlyUsedInEqualityComparison(V: CI, With: CI->getArgOperand(i: 0))) {
1211 Value *StrLen = emitStrLen(Ptr: CI->getArgOperand(i: 1), B, DL, TLI);
1212 if (!StrLen)
1213 return nullptr;
1214 Value *StrNCmp = emitStrNCmp(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
1215 Len: StrLen, B, DL, TLI);
1216 if (!StrNCmp)
1217 return nullptr;
1218 for (User *U : llvm::make_early_inc_range(Range: CI->users())) {
1219 ICmpInst *Old = cast<ICmpInst>(Val: U);
1220 Value *Cmp =
1221 B.CreateICmp(P: Old->getPredicate(), LHS: StrNCmp,
1222 RHS: ConstantInt::getNullValue(Ty: StrNCmp->getType()), Name: "cmp");
1223 replaceAllUsesWith(I: Old, With: Cmp);
1224 }
1225 return CI;
1226 }
1227
1228 // See if either input string is a constant string.
1229 StringRef SearchStr, ToFindStr;
1230 bool HasStr1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: SearchStr);
1231 bool HasStr2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: ToFindStr);
1232
1233 // fold strstr(x, "") -> x.
1234 if (HasStr2 && ToFindStr.empty())
1235 return CI->getArgOperand(i: 0);
1236
1237 // If both strings are known, constant fold it.
1238 if (HasStr1 && HasStr2) {
1239 size_t Offset = SearchStr.find(Str: ToFindStr);
1240
1241 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1242 return Constant::getNullValue(Ty: CI->getType());
1243
1244 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1245 return B.CreateConstInBoundsGEP1_64(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1246 Idx0: Offset, Name: "strstr");
1247 }
1248
1249 // fold strstr(x, "y") -> strchr(x, 'y').
1250 if (HasStr2 && ToFindStr.size() == 1) {
1251 return emitStrChr(Ptr: CI->getArgOperand(i: 0), C: ToFindStr[0], B, TLI);
1252 }
1253
1254 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
1255 return nullptr;
1256}
1257
1258Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1259 Value *SrcStr = CI->getArgOperand(i: 0);
1260 Value *Size = CI->getArgOperand(i: 2);
1261 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1262 Value *CharVal = CI->getArgOperand(i: 1);
1263 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1264 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1265
1266 if (LenC) {
1267 if (LenC->isZero())
1268 // Fold memrchr(x, y, 0) --> null.
1269 return NullPtr;
1270
1271 if (LenC->isOne()) {
1272 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1273 // constant or otherwise.
1274 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memrchr.char0");
1275 // Slice off the character's high end bits.
1276 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1277 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memrchr.char0cmp");
1278 return B.CreateSelectWithUnknownProfile(C: Cmp, True: SrcStr, False: NullPtr, DEBUG_TYPE,
1279 Name: "memrchr.sel");
1280 }
1281 }
1282
1283 StringRef Str;
1284 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1285 return nullptr;
1286
1287 if (Str.size() == 0)
1288 // If the array is empty fold memrchr(A, C, N) to null for any value
1289 // of C and N on the basis that the only valid value of N is zero
1290 // (otherwise the call is undefined).
1291 return NullPtr;
1292
1293 uint64_t EndOff = UINT64_MAX;
1294 if (LenC) {
1295 EndOff = LenC->getZExtValue();
1296 if (Str.size() < EndOff)
1297 // Punt out-of-bounds accesses to sanitizers and/or libc.
1298 return nullptr;
1299 }
1300
1301 if (ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal)) {
1302 // Fold memrchr(S, C, N) for a constant C.
1303 size_t Pos = Str.rfind(C: CharC->getZExtValue(), From: EndOff);
1304 if (Pos == StringRef::npos)
1305 // When the character is not in the source array fold the result
1306 // to null regardless of Size.
1307 return NullPtr;
1308
1309 if (LenC)
1310 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1311 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos));
1312
1313 if (Str.find(C: Str[Pos]) == Pos) {
1314 // When there is just a single occurrence of C in S, i.e., the one
1315 // in Str[Pos], fold
1316 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1317 // for nonconstant N.
1318 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1319 Name: "memrchr.cmp");
1320 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr,
1321 IdxList: B.getInt64(C: Pos), Name: "memrchr.ptr_plus");
1322 return B.CreateSelectWithUnknownProfile(C: Cmp, True: NullPtr, False: SrcPlus, DEBUG_TYPE,
1323 Name: "memrchr.sel");
1324 }
1325 }
1326
1327 // Truncate the string to search at most EndOff characters.
1328 Str = Str.substr(Start: 0, N: EndOff);
1329 if (Str.find_first_not_of(C: Str[0]) != StringRef::npos)
1330 return nullptr;
1331
1332 // If the source array consists of all equal characters, then for any
1333 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1334 // N != 0 && *S == C ? S + N - 1 : null
1335 Type *SizeTy = Size->getType();
1336 Type *Int8Ty = B.getInt8Ty();
1337 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1338 // Slice off the sought character's high end bits.
1339 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1340 Value *CEqS0 = B.CreateICmpEQ(LHS: ConstantInt::get(Ty: Int8Ty, V: Str[0]), RHS: CharVal);
1341 Value *And = B.CreateLogicalAnd(Cond1: NNeZ, Cond2: CEqS0);
1342 if (auto *AndSI = dyn_cast<SelectInst>(Val: And))
1343 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *AndSI, DEBUG_TYPE);
1344 Value *SizeM1 = B.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
1345 Value *SrcPlus =
1346 B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: SrcStr, IdxList: SizeM1, Name: "memrchr.ptr_plus");
1347 return B.CreateSelectWithUnknownProfile(C: And, True: SrcPlus, False: NullPtr, DEBUG_TYPE,
1348 Name: "memrchr.sel");
1349}
1350
1351Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1352 Value *SrcStr = CI->getArgOperand(i: 0);
1353 Value *Size = CI->getArgOperand(i: 2);
1354
1355 if (isKnownNonZero(V: Size, Q: DL)) {
1356 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1357 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1358 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1359 }
1360
1361 Value *CharVal = CI->getArgOperand(i: 1);
1362 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
1363 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1364 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1365
1366 // memchr(x, y, 0) -> null
1367 if (LenC) {
1368 if (LenC->isZero())
1369 return NullPtr;
1370
1371 if (LenC->isOne()) {
1372 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1373 // constant or otherwise.
1374 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memchr.char0");
1375 // Slice off the character's high end bits.
1376 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1377 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memchr.char0cmp");
1378 // The condition depends on the value of the string being equal to the
1379 // query, neither of which we know without value profiling, so mark the
1380 // profile unknown.
1381 return B.CreateSelectWithUnknownProfile(C: Cmp, True: SrcStr, False: NullPtr, DEBUG_TYPE,
1382 Name: "memchr.sel");
1383 }
1384 }
1385
1386 StringRef Str;
1387 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1388 return nullptr;
1389
1390 if (CharC) {
1391 size_t Pos = Str.find(C: CharC->getZExtValue());
1392 if (Pos == StringRef::npos)
1393 // When the character is not in the source array fold the result
1394 // to null regardless of Size.
1395 return NullPtr;
1396
1397 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1398 // When the constant Size is less than or equal to the character
1399 // position also fold the result to null.
1400 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1401 Name: "memchr.cmp");
1402 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos),
1403 Name: "memchr.ptr");
1404 // The condition is dependent upon the value of n, which we cannot infer
1405 // without value profiling, so mark the profile unknown.
1406 return B.CreateSelectWithUnknownProfile(C: Cmp, True: NullPtr, False: SrcPlus, DEBUG_TYPE);
1407 }
1408
1409 if (Str.size() == 0)
1410 // If the array is empty fold memchr(A, C, N) to null for any value
1411 // of C and N on the basis that the only valid value of N is zero
1412 // (otherwise the call is undefined).
1413 return NullPtr;
1414
1415 if (LenC)
1416 Str = substr(Str, Len: LenC->getZExtValue());
1417
1418 size_t Pos = Str.find_first_not_of(C: Str[0]);
1419 if (Pos == StringRef::npos
1420 || Str.find_first_not_of(C: Str[Pos], From: Pos) == StringRef::npos) {
1421 // If the source array consists of at most two consecutive sequences
1422 // of the same characters, then for any C and N (whether in bounds or
1423 // not), fold memchr(S, C, N) to
1424 // N != 0 && *S == C ? S : null
1425 // or for the two sequences to:
1426 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1427 // ^Sel2 ^Sel1 are denoted above.
1428 // The latter makes it also possible to fold strchr() calls with strings
1429 // of the same characters.
1430 Type *SizeTy = Size->getType();
1431 Type *Int8Ty = B.getInt8Ty();
1432
1433 // Slice off the sought character's high end bits.
1434 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1435
1436 Value *Sel1 = NullPtr;
1437 if (Pos != StringRef::npos) {
1438 // Handle two consecutive sequences of the same characters.
1439 Value *PosVal = ConstantInt::get(Ty: SizeTy, V: Pos);
1440 Value *StrPos = ConstantInt::get(Ty: Int8Ty, V: Str[Pos]);
1441 Value *CEqSPos = B.CreateICmpEQ(LHS: CharVal, RHS: StrPos);
1442 Value *NGtPos = B.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: Size, RHS: PosVal);
1443 Value *And = B.CreateAnd(LHS: CEqSPos, RHS: NGtPos);
1444 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: PosVal);
1445 // The condition depends on the value of the query and size, neither of
1446 // which we know without value profiling, so mark the profile unknown.
1447 Sel1 = B.CreateSelectWithUnknownProfile(C: And, True: SrcPlus, False: NullPtr, DEBUG_TYPE,
1448 Name: "memchr.sel1");
1449 }
1450
1451 Value *Str0 = ConstantInt::get(Ty: Int8Ty, V: Str[0]);
1452 Value *CEqS0 = B.CreateICmpEQ(LHS: Str0, RHS: CharVal);
1453 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1454 Value *And = B.CreateAnd(LHS: NNeZ, RHS: CEqS0);
1455 // The condition depends on the value of the query and size, neither of
1456 // which we know without value profiling, so mark the profile unknown.
1457 return B.CreateSelectWithUnknownProfile(C: And, True: SrcStr, False: Sel1, DEBUG_TYPE,
1458 Name: "memchr.sel2");
1459 }
1460
1461 if (!LenC) {
1462 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1463 // S is dereferenceable so it's safe to load from it and fold
1464 // memchr(S, C, N) == S to N && *S == C for any C and N.
1465 // TODO: This is safe even for nonconstant S.
1466 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1467
1468 // From now on we need a constant length and constant array.
1469 return nullptr;
1470 }
1471
1472 bool OptForSize = llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
1473 QueryType: PGSOQueryType::IRPass);
1474
1475 // If the char is variable but the input str and length are not we can turn
1476 // this memchr call into a simple bit field test. Of course this only works
1477 // when the return value is only checked against null.
1478 //
1479 // It would be really nice to reuse switch lowering here but we can't change
1480 // the CFG at this point.
1481 //
1482 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1483 // != 0
1484 // after bounds check.
1485 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CtxI: CI))
1486 return nullptr;
1487
1488 unsigned char Max =
1489 *std::max_element(first: reinterpret_cast<const unsigned char *>(Str.begin()),
1490 last: reinterpret_cast<const unsigned char *>(Str.end()));
1491
1492 // Make sure the bit field we're about to create fits in a register on the
1493 // target.
1494 // FIXME: On a 64 bit architecture this prevents us from using the
1495 // interesting range of alpha ascii chars. We could do better by emitting
1496 // two bitfields or shifting the range by 64 if no lower chars are used.
1497 if (!DL.fitsInLegalInteger(Width: Max + 1)) {
1498 // Build chain of ORs
1499 // Transform:
1500 // memchr("abcd", C, 4) != nullptr
1501 // to:
1502 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1503 std::string SortedStr = Str.str();
1504 llvm::sort(C&: SortedStr);
1505 // Compute the number of of non-contiguous ranges.
1506 unsigned NonContRanges = 1;
1507 for (size_t i = 1; i < SortedStr.size(); ++i) {
1508 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1509 NonContRanges++;
1510 }
1511 }
1512
1513 // Restrict this optimization to profitable cases with one or two range
1514 // checks.
1515 if (NonContRanges > 2)
1516 return nullptr;
1517
1518 // Slice off the character's high end bits.
1519 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1520
1521 SmallVector<Value *> CharCompares;
1522 for (unsigned char C : SortedStr)
1523 CharCompares.push_back(Elt: B.CreateICmpEQ(LHS: CharVal, RHS: B.getInt8(C)));
1524
1525 return B.CreateIntToPtr(V: B.CreateOr(Ops: CharCompares), DestTy: CI->getType());
1526 }
1527
1528 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1529 // creating unnecessary illegal types.
1530 unsigned char Width = NextPowerOf2(A: std::max(a: (unsigned char)7, b: Max));
1531
1532 // Now build the bit field.
1533 APInt Bitfield(Width, 0);
1534 for (char C : Str)
1535 Bitfield.setBit((unsigned char)C);
1536 Value *BitfieldC = B.getInt(AI: Bitfield);
1537
1538 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1539 Value *C = B.CreateZExtOrTrunc(V: CharVal, DestTy: BitfieldC->getType());
1540 C = B.CreateAnd(LHS: C, RHS: B.getIntN(N: Width, C: 0xFF));
1541
1542 // First check that the bit field access is within bounds.
1543 Value *Bounds = B.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: C, RHS: B.getIntN(N: Width, C: Width),
1544 Name: "memchr.bounds");
1545
1546 // Create code that checks if the given bit is set in the field.
1547 Value *Shl = B.CreateShl(LHS: B.getIntN(N: Width, C: 1ULL), RHS: C);
1548 Value *Bits = B.CreateIsNotNull(Arg: B.CreateAnd(LHS: Shl, RHS: BitfieldC), Name: "memchr.bits");
1549
1550 // Finally merge both checks and cast to pointer type. The inttoptr
1551 // implicitly zexts the i1 to intptr type.
1552 Value *Memchr = B.CreateLogicalAnd(Cond1: Bounds, Cond2: Bits, Name: "memchr");
1553 // We construct an and between the value of the memory and the bytes to search
1554 // for. We cannot infer how often this would be true without value profiling
1555 // for the query, so mark the profile unknown.
1556 if (auto *SI = dyn_cast<SelectInst>(Val: Memchr))
1557 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE);
1558 return B.CreateIntToPtr(V: Memchr, DestTy: CI->getType());
1559}
1560
1561// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1562// arrays LHS and RHS and nonconstant Size.
1563static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
1564 Value *Size, bool StrNCmp,
1565 IRBuilderBase &B, const DataLayout &DL) {
1566 if (LHS == RHS) // memcmp(s,s,x) -> 0
1567 return Constant::getNullValue(Ty: CI->getType());
1568
1569 StringRef LStr, RStr;
1570 if (!getConstantStringInfo(V: LHS, Str&: LStr, /*TrimAtNul=*/false) ||
1571 !getConstantStringInfo(V: RHS, Str&: RStr, /*TrimAtNul=*/false))
1572 return nullptr;
1573
1574 // If the contents of both constant arrays are known, fold a call to
1575 // memcmp(A, B, N) to
1576 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1577 // where Pos is the first mismatch between A and B, determined below.
1578
1579 uint64_t Pos = 0;
1580 Value *Zero = ConstantInt::get(Ty: CI->getType(), V: 0);
1581 for (uint64_t MinSize = std::min(a: LStr.size(), b: RStr.size()); ; ++Pos) {
1582 if (Pos == MinSize ||
1583 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1584 // One array is a leading part of the other of equal or greater
1585 // size, or for strncmp, the arrays are equal strings.
1586 // Fold the result to zero. Size is assumed to be in bounds, since
1587 // otherwise the call would be undefined.
1588 return Zero;
1589 }
1590
1591 if (LStr[Pos] != RStr[Pos])
1592 break;
1593 }
1594
1595 // Normalize the result.
1596 typedef unsigned char UChar;
1597 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1598 Value *MaxSize = ConstantInt::get(Ty: Size->getType(), V: Pos);
1599 Value *Cmp = B.CreateICmp(P: ICmpInst::ICMP_ULE, LHS: Size, RHS: MaxSize);
1600 Value *Res = ConstantInt::getSigned(Ty: CI->getType(), V: IRes);
1601 return B.CreateSelect(C: Cmp, True: Zero, False: Res);
1602}
1603
1604// Optimize a memcmp call CI with constant size Len.
1605static Value *optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS,
1606 uint64_t Len, IRBuilderBase &B,
1607 const DataLayout &DL) {
1608 if (Len == 0) // memcmp(s1,s2,0) -> 0
1609 return Constant::getNullValue(Ty: CI->getType());
1610
1611 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1612 if (Len == 1) {
1613 Value *LHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: LHS, Name: "lhsc"),
1614 DestTy: CI->getType(), Name: "lhsv");
1615 Value *RHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: RHS, Name: "rhsc"),
1616 DestTy: CI->getType(), Name: "rhsv");
1617 return B.CreateSub(LHS: LHSV, RHS: RHSV, Name: "chardiff");
1618 }
1619
1620 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1621 // TODO: The case where both inputs are constants does not need to be limited
1622 // to legal integers or equality comparison. See block below this.
1623 if (DL.isLegalInteger(Width: Len * 8) && isOnlyUsedInZeroEqualityComparison(CtxI: CI)) {
1624 IntegerType *IntType = IntegerType::get(C&: CI->getContext(), NumBits: Len * 8);
1625 Align PrefAlignment = DL.getPrefTypeAlign(Ty: IntType);
1626
1627 // First, see if we can fold either argument to a constant.
1628 Value *LHSV = nullptr;
1629 if (auto *LHSC = dyn_cast<Constant>(Val: LHS))
1630 LHSV = ConstantFoldLoadFromConstPtr(C: LHSC, Ty: IntType, DL);
1631
1632 Value *RHSV = nullptr;
1633 if (auto *RHSC = dyn_cast<Constant>(Val: RHS))
1634 RHSV = ConstantFoldLoadFromConstPtr(C: RHSC, Ty: IntType, DL);
1635
1636 // Don't generate unaligned loads. If either source is constant data,
1637 // alignment doesn't matter for that source because there is no load.
1638 if ((LHSV || getKnownAlignment(V: LHS, DL, CtxI: CI) >= PrefAlignment) &&
1639 (RHSV || getKnownAlignment(V: RHS, DL, CtxI: CI) >= PrefAlignment)) {
1640 if (!LHSV)
1641 LHSV = B.CreateLoad(Ty: IntType, Ptr: LHS, Name: "lhsv");
1642 if (!RHSV)
1643 RHSV = B.CreateLoad(Ty: IntType, Ptr: RHS, Name: "rhsv");
1644 return B.CreateZExt(V: B.CreateICmpNE(LHS: LHSV, RHS: RHSV), DestTy: CI->getType(), Name: "memcmp");
1645 }
1646 }
1647
1648 return nullptr;
1649}
1650
1651// Most simplifications for memcmp also apply to bcmp.
1652Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1653 IRBuilderBase &B) {
1654 Value *LHS = CI->getArgOperand(i: 0), *RHS = CI->getArgOperand(i: 1);
1655 Value *Size = CI->getArgOperand(i: 2);
1656
1657 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1658
1659 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, StrNCmp: false, B, DL))
1660 return Res;
1661
1662 // Handle constant Size.
1663 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1664 if (!LenC)
1665 return nullptr;
1666
1667 return optimizeMemCmpConstantSize(CI, LHS, RHS, Len: LenC->getZExtValue(), B, DL);
1668}
1669
1670Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1671 Module *M = CI->getModule();
1672 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1673 return V;
1674
1675 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1676 // bcmp can be more efficient than memcmp because it only has to know that
1677 // there is a difference, not how different one is to the other.
1678 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_bcmp) &&
1679 isOnlyUsedInZeroEqualityComparison(CtxI: CI)) {
1680 Value *LHS = CI->getArgOperand(i: 0);
1681 Value *RHS = CI->getArgOperand(i: 1);
1682 Value *Size = CI->getArgOperand(i: 2);
1683 return copyFlags(Old: *CI, New: emitBCmp(Ptr1: LHS, Ptr2: RHS, Len: Size, B, DL, TLI));
1684 }
1685
1686 return nullptr;
1687}
1688
1689Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1690 return optimizeMemCmpBCmpCommon(CI, B);
1691}
1692
1693Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1694 Value *Size = CI->getArgOperand(i: 2);
1695 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1696 if (isa<IntrinsicInst>(Val: CI))
1697 return nullptr;
1698
1699 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1700 CallInst *NewCI = B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1701 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1702 mergeAttributesAndFlags(NewCI, Old: *CI);
1703 return CI->getArgOperand(i: 0);
1704}
1705
1706Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1707 Value *Dst = CI->getArgOperand(i: 0);
1708 Value *Src = CI->getArgOperand(i: 1);
1709 ConstantInt *StopChar = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
1710 ConstantInt *N = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
1711 StringRef SrcStr;
1712 if (CI->use_empty() && Dst == Src)
1713 return Dst;
1714 // memccpy(d, s, c, 0) -> nullptr
1715 if (N) {
1716 if (N->isNullValue())
1717 return Constant::getNullValue(Ty: CI->getType());
1718 if (!getConstantStringInfo(V: Src, Str&: SrcStr, /*TrimAtNul=*/false) ||
1719 // TODO: Handle zeroinitializer.
1720 !StopChar)
1721 return nullptr;
1722 } else {
1723 return nullptr;
1724 }
1725
1726 // Wrap arg 'c' of type int to char
1727 size_t Pos = SrcStr.find(C: StopChar->getSExtValue() & 0xFF);
1728 if (Pos == StringRef::npos) {
1729 if (N->getZExtValue() <= SrcStr.size()) {
1730 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
1731 Size: CI->getArgOperand(i: 3)));
1732 return Constant::getNullValue(Ty: CI->getType());
1733 }
1734 return nullptr;
1735 }
1736
1737 Value *NewN =
1738 ConstantInt::get(Ty: N->getType(), V: std::min(a: uint64_t(Pos + 1), b: N->getZExtValue()));
1739 // memccpy -> llvm.memcpy
1740 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: NewN));
1741 return Pos + 1 <= N->getZExtValue()
1742 ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: NewN)
1743 : Constant::getNullValue(Ty: CI->getType());
1744}
1745
1746Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1747 Value *Dst = CI->getArgOperand(i: 0);
1748 Value *N = CI->getArgOperand(i: 2);
1749 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1750 CallInst *NewCI =
1751 B.CreateMemCpy(Dst, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size: N);
1752 // Propagate attributes, but memcpy has no return value, so make sure that
1753 // any return attributes are compliant.
1754 // TODO: Attach return value attributes to the 1st operand to preserve them?
1755 mergeAttributesAndFlags(NewCI, Old: *CI);
1756 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: N);
1757}
1758
1759Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1760 Value *Size = CI->getArgOperand(i: 2);
1761 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1762 if (isa<IntrinsicInst>(Val: CI))
1763 return nullptr;
1764
1765 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1766 CallInst *NewCI = B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1767 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1768 mergeAttributesAndFlags(NewCI, Old: *CI);
1769 return CI->getArgOperand(i: 0);
1770}
1771
1772Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1773 Value *Size = CI->getArgOperand(i: 2);
1774 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1775 if (isa<IntrinsicInst>(Val: CI))
1776 return nullptr;
1777
1778 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1779 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
1780 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val, Size, Align: Align(1));
1781 mergeAttributesAndFlags(NewCI, Old: *CI);
1782 return CI->getArgOperand(i: 0);
1783}
1784
1785Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1786 if (isa<ConstantPointerNull>(Val: CI->getArgOperand(i: 0))) {
1787 Value *Malloc = emitMalloc(Num: CI->getArgOperand(i: 1), B, DL, TLI);
1788 if (auto *MallocCI = dyn_cast_or_null<CallInst>(Val: Malloc))
1789 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_alloc_token))
1790 MallocCI->setMetadata(KindID: LLVMContext::MD_alloc_token, Node: MD);
1791 return copyFlags(Old: *CI, New: Malloc);
1792 }
1793
1794 return nullptr;
1795}
1796
1797// Optionally allow optimization of nobuiltin calls to operator new and its
1798// variants.
1799Value *LibCallSimplifier::maybeOptimizeNoBuiltinOperatorNew(CallInst *CI,
1800 IRBuilderBase &B) {
1801 if (!OptimizeHotColdNew)
1802 return nullptr;
1803 Function *Callee = CI->getCalledFunction();
1804 if (!Callee)
1805 return nullptr;
1806 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
1807 if (Func == NotLibFunc)
1808 return nullptr;
1809 switch (Func) {
1810 case LibFunc_Znwm:
1811 case LibFunc_ZnwmRKSt9nothrow_t:
1812 case LibFunc_ZnwmSt11align_val_t:
1813 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1814 case LibFunc_Znam:
1815 case LibFunc_ZnamRKSt9nothrow_t:
1816 case LibFunc_ZnamSt11align_val_t:
1817 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1818 case LibFunc_size_returning_new:
1819 case LibFunc_size_returning_new_aligned:
1820 // By default normal operator new calls (not already passing a hot_cold_t
1821 // parameter) are not mutated if the call is not marked builtin. Optionally
1822 // enable that in cases where it is known to be safe.
1823 if (!OptimizeNoBuiltinHotColdNew)
1824 return nullptr;
1825 break;
1826 case LibFunc_Znwm12__hot_cold_t:
1827 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1828 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1829 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1830 case LibFunc_Znam12__hot_cold_t:
1831 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1832 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1833 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1834 case LibFunc_size_returning_new_hot_cold:
1835 case LibFunc_size_returning_new_aligned_hot_cold:
1836 // If the nobuiltin call already passes a hot_cold_t parameter, allow update
1837 // of that parameter when enabled.
1838 if (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::None)
1839 return nullptr;
1840 break;
1841 default:
1842 return nullptr;
1843 }
1844 return optimizeNew(CI, B, Func);
1845}
1846
1847// When enabled, replace operator new() calls marked with a hot or cold memprof
1848// attribute with an operator new() call that takes a __hot_cold_t parameter.
1849// Currently this is supported by the open source version of tcmalloc, see:
1850// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1851Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1852 LibFunc &Func) {
1853 if (!OptimizeHotColdNew)
1854 return nullptr;
1855
1856 uint8_t HotCold;
1857 bool IsCold = false;
1858 if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold") {
1859 HotCold = ColdNewHintValue;
1860 IsCold = true;
1861 } else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1862 "notcold")
1863 HotCold = NotColdNewHintValue;
1864 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "hot")
1865 HotCold = HotNewHintValue;
1866 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1867 "ambiguous")
1868 HotCold = AmbiguousNewHintValue;
1869 else
1870 return nullptr;
1871
1872 bool ShouldOptimizeExistingHotColdNew =
1873 OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Always ||
1874 (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Cold &&
1875 IsCold);
1876
1877 Value *HotColdVal = B.getInt8(C: HotCold);
1878 auto getHotColdHintForExisting = [&](uint8_t HotCold) -> Value * {
1879 // If not taking the minimum, simply use the compiler hint value.
1880 if (!MinExistingHotColdNewHint)
1881 return HotColdVal;
1882 Value *ExistingHint = CI->getArgOperand(i: CI->arg_size() - 1);
1883 if (ExistingHint->getType() != B.getInt8Ty())
1884 ExistingHint = B.CreateTruncOrBitCast(V: ExistingHint, DestTy: B.getInt8Ty());
1885 // Emit a umin intrinsic to take the minimum of the existing hint and the
1886 // compiler hint. When the existing hint is a compile-time constant, the
1887 // IRBuilder folder will automatically constant-fold this into a constant.
1888 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: ExistingHint, RHS: HotColdVal);
1889 };
1890
1891 // For calls that already pass a hot/cold hint, only update the hint if
1892 // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
1893 // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
1894 // Note that in cases where we decide it is "notcold", it might be slightly
1895 // better to replace the hinted call with a non hinted call, to avoid the
1896 // extra parameter and the if condition check of the hint value in the
1897 // allocator. This can be considered in the future.
1898 Value *NewCall = nullptr;
1899 switch (Func) {
1900 case LibFunc_Znwm12__hot_cold_t:
1901 if (ShouldOptimizeExistingHotColdNew)
1902 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1903 NewFunc: LibFunc_Znwm12__hot_cold_t,
1904 HotCold: getHotColdHintForExisting(HotCold));
1905 break;
1906 case LibFunc_Znwm:
1907 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1908 NewFunc: LibFunc_Znwm12__hot_cold_t, HotCold: HotColdVal);
1909 break;
1910 case LibFunc_Znam12__hot_cold_t:
1911 if (ShouldOptimizeExistingHotColdNew)
1912 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1913 NewFunc: LibFunc_Znam12__hot_cold_t,
1914 HotCold: getHotColdHintForExisting(HotCold));
1915 break;
1916 case LibFunc_Znam:
1917 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1918 NewFunc: LibFunc_Znam12__hot_cold_t, HotCold: HotColdVal);
1919 break;
1920 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1921 if (ShouldOptimizeExistingHotColdNew)
1922 NewCall =
1923 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1924 TLI, NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
1925 HotCold: getHotColdHintForExisting(HotCold));
1926 break;
1927 case LibFunc_ZnwmRKSt9nothrow_t:
1928 NewCall = emitHotColdNewNoThrow(
1929 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1930 NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1931 break;
1932 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1933 if (ShouldOptimizeExistingHotColdNew)
1934 NewCall =
1935 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1936 TLI, NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
1937 HotCold: getHotColdHintForExisting(HotCold));
1938 break;
1939 case LibFunc_ZnamRKSt9nothrow_t:
1940 NewCall = emitHotColdNewNoThrow(
1941 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1942 NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1943 break;
1944 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1945 if (ShouldOptimizeExistingHotColdNew)
1946 NewCall =
1947 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1948 TLI, NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
1949 HotCold: getHotColdHintForExisting(HotCold));
1950 break;
1951 case LibFunc_ZnwmSt11align_val_t:
1952 NewCall = emitHotColdNewAligned(
1953 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1954 NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1955 break;
1956 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1957 if (ShouldOptimizeExistingHotColdNew)
1958 NewCall =
1959 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1960 TLI, NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t,
1961 HotCold: getHotColdHintForExisting(HotCold));
1962 break;
1963 case LibFunc_ZnamSt11align_val_t:
1964 NewCall = emitHotColdNewAligned(
1965 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1966 NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1967 break;
1968 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1969 if (ShouldOptimizeExistingHotColdNew)
1970 NewCall = emitHotColdNewAlignedNoThrow(
1971 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1972 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1973 HotCold: getHotColdHintForExisting(HotCold));
1974 break;
1975 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1976 NewCall = emitHotColdNewAlignedNoThrow(
1977 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1978 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1979 HotCold: HotColdVal);
1980 break;
1981 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1982 if (ShouldOptimizeExistingHotColdNew)
1983 NewCall = emitHotColdNewAlignedNoThrow(
1984 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1985 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1986 HotCold: getHotColdHintForExisting(HotCold));
1987 break;
1988 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1989 NewCall = emitHotColdNewAlignedNoThrow(
1990 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1991 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1992 HotCold: HotColdVal);
1993 break;
1994 case LibFunc_size_returning_new:
1995 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
1996 NewFunc: LibFunc_size_returning_new_hot_cold,
1997 HotCold: HotColdVal);
1998 break;
1999 case LibFunc_size_returning_new_hot_cold:
2000 if (ShouldOptimizeExistingHotColdNew)
2001 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
2002 NewFunc: LibFunc_size_returning_new_hot_cold,
2003 HotCold: getHotColdHintForExisting(HotCold));
2004 break;
2005 case LibFunc_size_returning_new_aligned:
2006 NewCall = emitHotColdSizeReturningNewAligned(
2007 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
2008 NewFunc: LibFunc_size_returning_new_aligned_hot_cold, HotCold: HotColdVal);
2009 break;
2010 case LibFunc_size_returning_new_aligned_hot_cold:
2011 if (ShouldOptimizeExistingHotColdNew)
2012 NewCall = emitHotColdSizeReturningNewAligned(
2013 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
2014 NewFunc: LibFunc_size_returning_new_aligned_hot_cold,
2015 HotCold: getHotColdHintForExisting(HotCold));
2016 break;
2017 default:
2018 return nullptr;
2019 }
2020
2021 if (auto *NewCI = dyn_cast_or_null<Instruction>(Val: NewCall))
2022 NewCI->copyMetadata(SrcInst: *CI);
2023
2024 return NewCall;
2025}
2026
2027//===----------------------------------------------------------------------===//
2028// Math Library Optimizations
2029//===----------------------------------------------------------------------===//
2030
2031/// Preserve the accuracy requirement of \p Old on the replacement \p New.
2032static void copyFPMath(const CallInst &Old, Value *New) {
2033 if (auto *NewI = dyn_cast<Instruction>(Val: New))
2034 if (MDNode *MD = Old.getMetadata(KindID: LLVMContext::MD_fpmath))
2035 NewI->setMetadata(KindID: LLVMContext::MD_fpmath, Node: MD);
2036}
2037
2038// Replace a libcall \p CI with a call to intrinsic \p IID
2039static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
2040 Intrinsic::ID IID) {
2041 Value *NewCall = B.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0), FMFSource: CI);
2042 NewCall->takeName(V: CI);
2043 copyFPMath(Old: *CI, New: NewCall);
2044 return copyFlags(Old: *CI, New: NewCall);
2045}
2046
2047static Value *replaceBinaryCall(CallInst *CI, IRBuilderBase &B,
2048 Intrinsic::ID IID) {
2049 Value *NewCall = B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2050 RHS: CI->getArgOperand(i: 1), FMFSource: CI);
2051 NewCall->takeName(V: CI);
2052 copyFPMath(Old: *CI, New: NewCall);
2053 return copyFlags(Old: *CI, New: NewCall);
2054}
2055
2056/// Return a variant of Val with float type.
2057/// Currently this works in two cases: If Val is an FPExtension of a float
2058/// value to something bigger, simply return the operand.
2059/// If Val is a ConstantFP but can be converted to a float ConstantFP without
2060/// loss of precision do so.
2061static Value *valueHasFloatPrecision(Value *Val) {
2062 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
2063 Value *Op = Cast->getOperand(i_nocapture: 0);
2064 if (Op->getType()->isFloatTy())
2065 return Op;
2066 }
2067 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
2068 APFloat F = Const->getValueAPF();
2069 bool losesInfo;
2070 (void)F.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
2071 losesInfo: &losesInfo);
2072 if (!losesInfo)
2073 return ConstantFP::get(Context&: Const->getContext(), V: F);
2074 }
2075 return nullptr;
2076}
2077
2078/// Shrink double -> float functions.
2079static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
2080 bool isBinary, const TargetLibraryInfo *TLI,
2081 bool isPrecise = false) {
2082 Function *CalleeFn = CI->getCalledFunction();
2083 if (!CI->getType()->isDoubleTy() || !CalleeFn)
2084 return nullptr;
2085
2086 // If not all the uses of the function are converted to float, then bail out.
2087 // This matters if the precision of the result is more important than the
2088 // precision of the arguments.
2089 if (isPrecise)
2090 for (User *U : CI->users()) {
2091 FPTruncInst *Cast = dyn_cast<FPTruncInst>(Val: U);
2092 if (!Cast || !Cast->getType()->isFloatTy())
2093 return nullptr;
2094 }
2095
2096 // If this is something like 'g((double) float)', convert to 'gf(float)'.
2097 Value *V[2];
2098 V[0] = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2099 V[1] = isBinary ? valueHasFloatPrecision(Val: CI->getArgOperand(i: 1)) : nullptr;
2100 if (!V[0] || (isBinary && !V[1]))
2101 return nullptr;
2102
2103 // If call isn't an intrinsic, check that it isn't within a function with the
2104 // same name as the float version of this call, otherwise the result is an
2105 // infinite loop. For example, from MinGW-w64:
2106 //
2107 // float expf(float val) { return (float) exp((double) val); }
2108 StringRef CalleeName = CalleeFn->getName();
2109 bool IsIntrinsic = CalleeFn->isIntrinsic();
2110 if (!IsIntrinsic) {
2111 StringRef CallerName = CI->getFunction()->getName();
2112 if (CallerName.ends_with(Suffix: 'f') &&
2113 CallerName.size() == (CalleeName.size() + 1) &&
2114 CallerName.starts_with(Prefix: CalleeName))
2115 return nullptr;
2116 }
2117
2118 // Propagate the math semantics from the current function to the new function.
2119 IRBuilderBase::FastMathFlagGuard Guard(B);
2120 B.setFastMathFlags(CI->getFastMathFlags());
2121
2122 // g((double) float) -> (double) gf(float)
2123 Value *R;
2124 if (IsIntrinsic) {
2125 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
2126 R = isBinary ? B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V)
2127 : B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V[0]);
2128 } else {
2129 AttributeList CallsiteAttrs = CI->getAttributes();
2130 R = isBinary
2131 ? emitBinaryFloatFnCall(Op1: V[0], Op2: V[1], TLI, Name: CalleeName, B,
2132 Attrs: CallsiteAttrs)
2133 : emitUnaryFloatFnCall(Op: V[0], TLI, Name: CalleeName, B, Attrs: CallsiteAttrs);
2134 }
2135 return B.CreateFPExt(V: R, DestTy: B.getDoubleTy());
2136}
2137
2138/// Shrink double -> float for unary functions.
2139static Value *optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2140 const TargetLibraryInfo *TLI,
2141 bool isPrecise = false) {
2142 return optimizeDoubleFP(CI, B, isBinary: false, TLI, isPrecise);
2143}
2144
2145/// Shrink double -> float for binary functions.
2146static Value *optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2147 const TargetLibraryInfo *TLI,
2148 bool isPrecise = false) {
2149 return optimizeDoubleFP(CI, B, isBinary: true, TLI, isPrecise);
2150}
2151
2152/// Shrink double -> float for llvm.sincos.
2153static Value *optimizeSinCosDoubleFP(CallInst *CI, IRBuilderBase &B) {
2154 auto *RetTy = dyn_cast<StructType>(Val: CI->getType());
2155 if (!RetTy || RetTy->getNumElements() != 2 ||
2156 !RetTy->getElementType(N: 0)->getScalarType()->isDoubleTy())
2157 return nullptr;
2158
2159 Value *X = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2160 if (!X)
2161 if (auto *Ext = dyn_cast<FPExtInst>(Val: CI->getArgOperand(i: 0)))
2162 if (Ext->getOperand(i_nocapture: 0)->getType()->getScalarType()->isFloatTy())
2163 X = Ext->getOperand(i_nocapture: 0);
2164 if (!X)
2165 return nullptr;
2166
2167 for (User *U : CI->users()) {
2168 auto *EV = dyn_cast<ExtractValueInst>(Val: U);
2169 if (!EV)
2170 return nullptr;
2171 for (User *EVU : EV->users()) {
2172 auto *Cast = dyn_cast<FPTruncInst>(Val: EVU);
2173 if (!Cast || !Cast->getType()->getScalarType()->isFloatTy())
2174 return nullptr;
2175 }
2176 }
2177
2178 IRBuilderBase::FastMathFlagGuard Guard(B);
2179 B.setFastMathFlags(CI->getFastMathFlags());
2180
2181 Value *NewCall = B.CreateIntrinsic(ID: Intrinsic::sincos, OverloadTypes: X->getType(), Args: X);
2182 cast<Instruction>(Val: NewCall)->setMetadata(
2183 KindID: LLVMContext::MD_fpmath, Node: CI->getMetadata(KindID: LLVMContext::MD_fpmath));
2184 Value *Res = PoisonValue::get(T: RetTy);
2185 for (unsigned I = 0; I != 2; ++I) {
2186 Value *Ext = B.CreateFPExt(V: B.CreateExtractValue(Agg: NewCall, Idxs: I),
2187 DestTy: RetTy->getElementType(N: I));
2188 Res = B.CreateInsertValue(Agg: Res, Val: Ext, Idxs: I);
2189 }
2190 return Res;
2191}
2192
2193// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
2194Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
2195 Value *Real, *Imag;
2196
2197 if (CI->arg_size() == 1) {
2198
2199 if (!CI->isFast())
2200 return nullptr;
2201
2202 Value *Op = CI->getArgOperand(i: 0);
2203 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
2204
2205 Real = B.CreateExtractValue(Agg: Op, Idxs: 0, Name: "real");
2206 Imag = B.CreateExtractValue(Agg: Op, Idxs: 1, Name: "imag");
2207
2208 } else {
2209 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
2210
2211 Real = CI->getArgOperand(i: 0);
2212 Imag = CI->getArgOperand(i: 1);
2213
2214 // if real or imaginary part is zero, simplify to abs(cimag(z))
2215 // or abs(creal(z))
2216 Value *AbsOp = nullptr;
2217 if (ConstantFP *ConstReal = dyn_cast<ConstantFP>(Val: Real)) {
2218 if (ConstReal->isZero())
2219 AbsOp = Imag;
2220
2221 } else if (ConstantFP *ConstImag = dyn_cast<ConstantFP>(Val: Imag)) {
2222 if (ConstImag->isZero())
2223 AbsOp = Real;
2224 }
2225
2226 if (AbsOp)
2227 return copyFlags(Old: *CI, New: B.CreateFAbs(V: AbsOp, FMFSource: CI, Name: "cabs"));
2228
2229 if (!CI->isFast())
2230 return nullptr;
2231 }
2232
2233 // Propagate fast-math flags from the existing call to new instructions.
2234 Value *RealReal = B.CreateFMulFMF(L: Real, R: Real, FMFSource: CI);
2235 Value *ImagImag = B.CreateFMulFMF(L: Imag, R: Imag, FMFSource: CI);
2236 return copyFlags(
2237 Old: *CI, New: B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt,
2238 Op: B.CreateFAddFMF(L: RealReal, R: ImagImag, FMFSource: CI), FMFSource: CI,
2239 Name: "cabs"));
2240}
2241
2242// Return a properly extended integer (DstWidth bits wide) if the operation is
2243// an itofp.
2244static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
2245 if (isa<SIToFPInst>(Val: I2F) || isa<UIToFPInst>(Val: I2F)) {
2246 Value *Op = cast<Instruction>(Val: I2F)->getOperand(i: 0);
2247 // Make sure that the exponent fits inside an "int" of size DstWidth,
2248 // thus avoiding any range issues that FP has not.
2249 unsigned BitWidth = Op->getType()->getScalarSizeInBits();
2250 if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(Val: I2F))) {
2251 Type *IntTy = Op->getType()->getWithNewBitWidth(NewBitWidth: DstWidth);
2252 return isa<SIToFPInst>(Val: I2F) ? B.CreateSExt(V: Op, DestTy: IntTy)
2253 : B.CreateZExt(V: Op, DestTy: IntTy);
2254 }
2255 }
2256
2257 return nullptr;
2258}
2259
2260/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
2261/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
2262/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
2263Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
2264 Module *M = Pow->getModule();
2265 Value *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2266 Type *Ty = Pow->getType();
2267 bool Ignored;
2268
2269 // Evaluate special cases related to a nested function as the base.
2270
2271 // pow(exp(x), y) -> exp(x * y)
2272 // pow(exp2(x), y) -> exp2(x * y)
2273 // If exp{,2}() is used only once, it is better to fold two transcendental
2274 // math functions into one. If used again, exp{,2}() would still have to be
2275 // called with the original argument, then keep both original transcendental
2276 // functions. However, this transformation is only safe with fully relaxed
2277 // math semantics, since, besides rounding differences, it changes overflow
2278 // and underflow behavior quite dramatically. For example:
2279 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
2280 // Whereas:
2281 // exp(1000 * 0.001) = exp(1)
2282 // TODO: Loosen the requirement for fully relaxed math semantics.
2283 // TODO: Handle exp10() when more targets have it available.
2284 CallInst *BaseFn = dyn_cast<CallInst>(Val: Base);
2285 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
2286 Function *CalleeFn = BaseFn->getCalledFunction();
2287 LibFunc LibFn =
2288 CalleeFn ? TLI->getLibFunc(funcName: CalleeFn->getName()) : NotLibFunc;
2289 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFn)) {
2290 StringRef ExpName;
2291 Intrinsic::ID ID;
2292 Value *ExpFn;
2293 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2294
2295 switch (LibFn) {
2296 default:
2297 return nullptr;
2298 case LibFunc_expf:
2299 case LibFunc_exp:
2300 case LibFunc_expl:
2301 ExpName = TLI->getName(F: LibFunc_exp);
2302 ID = Intrinsic::exp;
2303 LibFnFloat = LibFunc_expf;
2304 LibFnDouble = LibFunc_exp;
2305 LibFnLongDouble = LibFunc_expl;
2306 break;
2307 case LibFunc_exp2f:
2308 case LibFunc_exp2:
2309 case LibFunc_exp2l:
2310 ExpName = TLI->getName(F: LibFunc_exp2);
2311 ID = Intrinsic::exp2;
2312 LibFnFloat = LibFunc_exp2f;
2313 LibFnDouble = LibFunc_exp2;
2314 LibFnLongDouble = LibFunc_exp2l;
2315 break;
2316 }
2317
2318 // Create new exp{,2}() with the product as its argument.
2319 Value *FMul = B.CreateFMul(L: BaseFn->getArgOperand(i: 0), R: Expo, Name: "mul");
2320 ExpFn = BaseFn->doesNotAccessMemory()
2321 ? B.CreateUnaryIntrinsic(ID, Op: FMul, FMFSource: nullptr, Name: ExpName)
2322 : emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFnDouble, FloatFn: LibFnFloat,
2323 LongDoubleFn: LibFnLongDouble, B,
2324 Attrs: BaseFn->getAttributes());
2325
2326 // Since the new exp{,2}() is different from the original one, dead code
2327 // elimination cannot be trusted to remove it, since it may have side
2328 // effects (e.g., errno). When the only consumer for the original
2329 // exp{,2}() is pow(), then it has to be explicitly erased.
2330 substituteInParent(I: BaseFn, With: ExpFn);
2331 return ExpFn;
2332 }
2333 }
2334
2335 // Evaluate special cases related to a constant base.
2336
2337 const APFloat *BaseF;
2338 if (!match(V: Base, P: m_APFloat(Res&: BaseF)))
2339 return nullptr;
2340
2341 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2342
2343 const bool UseIntrinsic = Pow->doesNotAccessMemory();
2344
2345 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2346 if ((UseIntrinsic || !Ty->isVectorTy()) && BaseF->isExactlyValue(V: 2.0) &&
2347 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo)) &&
2348 (UseIntrinsic ||
2349 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2350
2351 // TODO: Shouldn't really need to depend on getIntToFPVal for intrinsic. Can
2352 // just directly use the original integer type.
2353 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize())) {
2354 Constant *One = ConstantFP::get(Ty, V: 1.0);
2355
2356 if (UseIntrinsic) {
2357 return copyFlags(Old: *Pow, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2358 OverloadTypes: {Ty, ExpoI->getType()},
2359 Args: {One, ExpoI}, FMFSource: Pow, Name: "exp2"));
2360 }
2361
2362 return copyFlags(Old: *Pow, New: emitBinaryFloatFnCall(
2363 Op1: One, Op2: ExpoI, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2364 LongDoubleFn: LibFunc_ldexpl, B, Attrs: NoAttrs));
2365 }
2366 }
2367
2368 // pow(2.0 ** n, x) -> exp2(n * x)
2369 if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f, LongDoubleFn: LibFunc_exp2l)) {
2370 APFloat BaseR = APFloat(1.0);
2371 BaseR.convert(ToSemantics: BaseF->getSemantics(), RM: APFloat::rmTowardZero, losesInfo: &Ignored);
2372 BaseR = BaseR / *BaseF;
2373 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2374 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2375 APSInt NI(64, false);
2376 if ((IsInteger || IsReciprocal) &&
2377 NF->convertToInteger(Result&: NI, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2378 APFloat::opOK &&
2379 NI > 1 && NI.isPowerOf2()) {
2380 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2381 Value *FMul = B.CreateFMul(L: Expo, R: ConstantFP::get(Ty, V: N), Name: "mul");
2382 if (Pow->doesNotAccessMemory())
2383 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2384 FMFSource: nullptr, Name: "exp2"));
2385 else
2386 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2387 FloatFn: LibFunc_exp2f,
2388 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2389 }
2390 }
2391
2392 // pow(10.0, x) -> exp10(x)
2393 if (BaseF->isExactlyValue(V: 10.0) &&
2394 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp10, FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l)) {
2395
2396 if (Pow->doesNotAccessMemory()) {
2397 return B.CreateIntrinsic(ID: Intrinsic::exp10, OverloadTypes: {Ty}, Args: {Expo}, FMFSource: Pow, Name: "exp10", OpBundles: {},
2398 SetFn: [Pow](CallInst *CI) { CI->copyIRFlags(V: Pow); });
2399 }
2400
2401 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: Expo, TLI, DoubleFn: LibFunc_exp10,
2402 FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l,
2403 B, Attrs: NoAttrs));
2404 }
2405
2406 // pow(x, y) -> exp2(log2(x) * y)
2407 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2408 !BaseF->isNegative()) {
2409 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2410 // Luckily optimizePow has already handled the x == 1 case.
2411 assert(!match(Base, m_FPOne()) &&
2412 "pow(1.0, y) should have been simplified earlier!");
2413
2414 Value *Log = nullptr;
2415 if (Ty->isFloatTy())
2416 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToFloat()));
2417 else if (Ty->isDoubleTy())
2418 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToDouble()));
2419
2420 if (Log) {
2421 Value *FMul = B.CreateFMul(L: Log, R: Expo, Name: "mul");
2422 if (Pow->doesNotAccessMemory())
2423 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2424 FMFSource: nullptr, Name: "exp2"));
2425 else if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f,
2426 LongDoubleFn: LibFunc_exp2l))
2427 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2428 FloatFn: LibFunc_exp2f,
2429 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2430 }
2431 }
2432
2433 return nullptr;
2434}
2435
2436static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2437 Module *M, IRBuilderBase &B,
2438 const TargetLibraryInfo *TLI) {
2439 // If errno is never set, then use the intrinsic for sqrt().
2440 if (NoErrno)
2441 return B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: V, FMFSource: nullptr, Name: "sqrt");
2442
2443 // Otherwise, use the libcall for sqrt().
2444 if (hasFloatFn(M, TLI, Ty: V->getType(), DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2445 LongDoubleFn: LibFunc_sqrtl))
2446 // TODO: We also should check that the target can in fact lower the sqrt()
2447 // libcall. We currently have no way to ask this question, so we ask if
2448 // the target has a sqrt() libcall, which is not exactly the same.
2449 return emitUnaryFloatFnCall(Op: V, TLI, DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2450 LongDoubleFn: LibFunc_sqrtl, B, Attrs);
2451
2452 return nullptr;
2453}
2454
2455/// Use square root in place of pow(x, +/-0.5).
2456Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2457 Value *Sqrt, *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2458 Module *Mod = Pow->getModule();
2459 Type *Ty = Pow->getType();
2460
2461 const APFloat *ExpoF;
2462 if (!match(V: Expo, P: m_APFloat(Res&: ExpoF)) ||
2463 (!ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)))
2464 return nullptr;
2465
2466 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2467 // so that requires fast-math-flags (afn or reassoc).
2468 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2469 return nullptr;
2470
2471 // If we have a pow() library call (accesses memory) and we can't guarantee
2472 // that the base is not an infinity, give up:
2473 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2474 // errno), but sqrt(-Inf) is required by various standards to set errno.
2475 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2476 !isKnownNeverInfinity(
2477 V: Base, SQ: SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
2478 return nullptr;
2479
2480 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M: Mod, B,
2481 TLI);
2482 if (!Sqrt)
2483 return nullptr;
2484
2485 // Handle signed zero base by expanding to fabs(sqrt(x)).
2486 if (!Pow->hasNoSignedZeros())
2487 Sqrt = B.CreateFAbs(V: Sqrt, FMFSource: nullptr, Name: "abs");
2488
2489 Sqrt = copyFlags(Old: *Pow, New: Sqrt);
2490
2491 // Handle non finite base by expanding to
2492 // (x == -infinity ? +infinity : sqrt(x)).
2493 if (!Pow->hasNoInfs()) {
2494 Value *PosInf = ConstantFP::getInfinity(Ty),
2495 *NegInf = ConstantFP::getInfinity(Ty, Negative: true);
2496 Value *FCmp = B.CreateFCmpOEQ(LHS: Base, RHS: NegInf, Name: "isinf");
2497 Sqrt = B.CreateSelect(C: FCmp, True: PosInf, False: Sqrt);
2498 // We assume that the case where x == -infinity is unlikely, so we assign
2499 // unlikely branch weights to that arm of the select.
2500 if (!ProfcheckDisableMetadataFixes) {
2501 if (auto *SqrtSI = dyn_cast<SelectInst>(Val: Sqrt))
2502 setBranchWeights(
2503 I&: *SqrtSI,
2504 Weights: {MDBuilder::kUnlikelyBranchWeight, MDBuilder::kLikelyBranchWeight},
2505 /*IsExpected=*/false);
2506 }
2507 }
2508
2509 // If the exponent is negative, then get the reciprocal.
2510 if (ExpoF->isNegative())
2511 Sqrt = B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Sqrt, Name: "reciprocal");
2512
2513 return Sqrt;
2514}
2515
2516static Value *createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M,
2517 IRBuilderBase &B) {
2518 Value *Args[] = {Base, Expo};
2519 Type *Types[] = {Base->getType(), Expo->getType()};
2520 return B.CreateIntrinsic(ID: Intrinsic::powi, OverloadTypes: Types, Args);
2521}
2522
2523Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2524 Value *Base = Pow->getArgOperand(i: 0);
2525 Value *Expo = Pow->getArgOperand(i: 1);
2526 Function *Callee = Pow->getCalledFunction();
2527 StringRef Name = Callee->getName();
2528 Type *Ty = Pow->getType();
2529 Module *M = Pow->getModule();
2530 bool AllowApprox = Pow->hasApproxFunc();
2531 bool Ignored;
2532
2533 // Propagate the math semantics from the call to any created instructions.
2534 IRBuilderBase::FastMathFlagGuard Guard(B);
2535 B.setFastMathFlags(Pow->getFastMathFlags());
2536 // Evaluate special cases related to the base.
2537
2538 // pow(1.0, x) -> 1.0
2539 if (match(V: Base, P: m_FPOne()))
2540 return Base;
2541
2542 if (Value *Exp = replacePowWithExp(Pow, B))
2543 return Exp;
2544
2545 // Evaluate special cases related to the exponent.
2546
2547 // pow(x, -1.0) -> 1.0 / x
2548 if (match(V: Expo, P: m_SpecificFP(V: -1.0)))
2549 return B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Base, Name: "reciprocal");
2550
2551 // pow(x, +/-0.0) -> 1.0
2552 if (match(V: Expo, P: m_AnyZeroFP()))
2553 return ConstantFP::get(Ty, V: 1.0);
2554
2555 // pow(x, 1.0) -> x
2556 if (match(V: Expo, P: m_FPOne()))
2557 return Base;
2558
2559 // pow(x, 2.0) -> x * x
2560 if (match(V: Expo, P: m_SpecificFP(V: 2.0)) && Pow->doesNotAccessMemory())
2561 return B.CreateFMul(L: Base, R: Base, Name: "square");
2562
2563 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2564 return Sqrt;
2565
2566 // If we can approximate pow:
2567 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2568 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2569 const APFloat *ExpoF;
2570 if (AllowApprox && match(V: Expo, P: m_APFloat(Res&: ExpoF)) &&
2571 !ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)) {
2572 APFloat ExpoA(abs(X: *ExpoF));
2573 APFloat ExpoI(*ExpoF);
2574 Value *Sqrt = nullptr;
2575 if (!ExpoA.isInteger()) {
2576 APFloat Expo2 = ExpoA;
2577 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2578 // is no floating point exception and the result is an integer, then
2579 // ExpoA == integer + 0.5
2580 if (Expo2.add(RHS: ExpoA, RM: APFloat::rmNearestTiesToEven) != APFloat::opOK)
2581 return nullptr;
2582
2583 if (!Expo2.isInteger())
2584 return nullptr;
2585
2586 if (ExpoI.roundToIntegral(RM: APFloat::rmTowardNegative) !=
2587 APFloat::opInexact)
2588 return nullptr;
2589 if (!ExpoI.isInteger())
2590 return nullptr;
2591 ExpoF = &ExpoI;
2592
2593 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M,
2594 B, TLI);
2595 if (!Sqrt)
2596 return nullptr;
2597 }
2598
2599 // 0.5 fraction is now optionally handled.
2600 // Do pow -> powi for remaining integer exponent
2601 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2602 if (ExpoF->isInteger() &&
2603 ExpoF->convertToInteger(Result&: IntExpo, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2604 APFloat::opOK) {
2605 Value *PowI = copyFlags(
2606 Old: *Pow,
2607 New: createPowWithIntegerExponent(
2608 Base, Expo: ConstantInt::get(Ty: B.getIntNTy(N: TLI->getIntSize()), V: IntExpo),
2609 M, B));
2610
2611 if (PowI && Sqrt)
2612 return B.CreateFMul(L: PowI, R: Sqrt);
2613
2614 return PowI;
2615 }
2616 }
2617
2618 // powf(x, itofp(y)) -> powi(x, y)
2619 // The powi exponent must be a scalar integer, so a vector y is not usable.
2620 if (AllowApprox && !Expo->getType()->isVectorTy() &&
2621 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo))) {
2622 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize()))
2623 return copyFlags(Old: *Pow, New: createPowWithIntegerExponent(Base, Expo: ExpoI, M, B));
2624 }
2625
2626 // Shrink pow() to powf() if the arguments are single precision,
2627 // unless the result is expected to be double precision.
2628 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_pow) &&
2629 hasFloatVersion(M, FuncName: Name)) {
2630 if (Value *Shrunk = optimizeBinaryDoubleFP(CI: Pow, B, TLI, isPrecise: true))
2631 return Shrunk;
2632 }
2633
2634 return nullptr;
2635}
2636
2637Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2638 Module *M = CI->getModule();
2639 Function *Callee = CI->getCalledFunction();
2640 StringRef Name = Callee->getName();
2641 Value *Ret = nullptr;
2642 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_exp2) &&
2643 hasFloatVersion(M, FuncName: Name))
2644 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2645
2646 // If we have an llvm.exp2 intrinsic, emit the llvm.ldexp intrinsic. If we
2647 // have the libcall, emit the libcall.
2648 //
2649 // TODO: In principle we should be able to just always use the intrinsic for
2650 // any doesNotAccessMemory callsite.
2651
2652 const bool UseIntrinsic = Callee->isIntrinsic();
2653 // Bail out for vectors because the code below only expects scalars.
2654 Type *Ty = CI->getType();
2655 if (!UseIntrinsic && Ty->isVectorTy())
2656 return Ret;
2657
2658 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2659 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2660 Value *Op = CI->getArgOperand(i: 0);
2661 if ((isa<SIToFPInst>(Val: Op) || isa<UIToFPInst>(Val: Op)) &&
2662 (UseIntrinsic ||
2663 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2664 if (Value *Exp = getIntToFPVal(I2F: Op, B, DstWidth: TLI->getIntSize())) {
2665 Constant *One = ConstantFP::get(Ty, V: 1.0);
2666
2667 if (UseIntrinsic) {
2668 return copyFlags(Old: *CI, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2669 OverloadTypes: {Ty, Exp->getType()},
2670 Args: {One, Exp}, FMFSource: CI));
2671 }
2672
2673 IRBuilderBase::FastMathFlagGuard Guard(B);
2674 B.setFastMathFlags(CI->getFastMathFlags());
2675 return copyFlags(Old: *CI, New: emitBinaryFloatFnCall(
2676 Op1: One, Op2: Exp, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2677 LongDoubleFn: LibFunc_ldexpl, B, Attrs: AttributeList()));
2678 }
2679 }
2680
2681 return Ret;
2682}
2683
2684Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B,
2685 Intrinsic::ID IID) {
2686 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2687 // the intrinsics for improved optimization (for example, vectorization).
2688 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2689 // From the C standard draft WG14/N1256:
2690 // "Ideally, fmax would be sensitive to the sign of zero, for example
2691 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2692 // might be impractical."
2693 FastMathFlags FMF = CI->getFastMathFlags();
2694 FMF.setNoSignedZeros();
2695 return copyFlags(Old: *CI, New: B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2696 RHS: CI->getArgOperand(i: 1), FMFSource: FMF));
2697}
2698
2699Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2700 Function *LogFn = Log->getCalledFunction();
2701 StringRef LogNm = LogFn->getName();
2702 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2703 Module *Mod = Log->getModule();
2704 Type *Ty = Log->getType();
2705
2706 if (UnsafeFPShrink && hasFloatVersion(M: Mod, FuncName: LogNm))
2707 if (Value *Ret = optimizeUnaryDoubleFP(CI: Log, B, TLI, isPrecise: true))
2708 return Ret;
2709
2710 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2711
2712 // This is only applicable to log(), log2(), log10().
2713 LogLb = TLI->getLibFunc(funcName: LogNm);
2714 if (LogLb != NotLibFunc) {
2715 switch (LogLb) {
2716 case LibFunc_logf:
2717 LogID = Intrinsic::log;
2718 ExpLb = LibFunc_expf;
2719 Exp2Lb = LibFunc_exp2f;
2720 Exp10Lb = LibFunc_exp10f;
2721 PowLb = LibFunc_powf;
2722 break;
2723 case LibFunc_log:
2724 LogID = Intrinsic::log;
2725 ExpLb = LibFunc_exp;
2726 Exp2Lb = LibFunc_exp2;
2727 Exp10Lb = LibFunc_exp10;
2728 PowLb = LibFunc_pow;
2729 break;
2730 case LibFunc_logl:
2731 LogID = Intrinsic::log;
2732 ExpLb = LibFunc_expl;
2733 Exp2Lb = LibFunc_exp2l;
2734 Exp10Lb = LibFunc_exp10l;
2735 PowLb = LibFunc_powl;
2736 break;
2737 case LibFunc_log2f:
2738 LogID = Intrinsic::log2;
2739 ExpLb = LibFunc_expf;
2740 Exp2Lb = LibFunc_exp2f;
2741 Exp10Lb = LibFunc_exp10f;
2742 PowLb = LibFunc_powf;
2743 break;
2744 case LibFunc_log2:
2745 LogID = Intrinsic::log2;
2746 ExpLb = LibFunc_exp;
2747 Exp2Lb = LibFunc_exp2;
2748 Exp10Lb = LibFunc_exp10;
2749 PowLb = LibFunc_pow;
2750 break;
2751 case LibFunc_log2l:
2752 LogID = Intrinsic::log2;
2753 ExpLb = LibFunc_expl;
2754 Exp2Lb = LibFunc_exp2l;
2755 Exp10Lb = LibFunc_exp10l;
2756 PowLb = LibFunc_powl;
2757 break;
2758 case LibFunc_log10f:
2759 LogID = Intrinsic::log10;
2760 ExpLb = LibFunc_expf;
2761 Exp2Lb = LibFunc_exp2f;
2762 Exp10Lb = LibFunc_exp10f;
2763 PowLb = LibFunc_powf;
2764 break;
2765 case LibFunc_log10:
2766 LogID = Intrinsic::log10;
2767 ExpLb = LibFunc_exp;
2768 Exp2Lb = LibFunc_exp2;
2769 Exp10Lb = LibFunc_exp10;
2770 PowLb = LibFunc_pow;
2771 break;
2772 case LibFunc_log10l:
2773 LogID = Intrinsic::log10;
2774 ExpLb = LibFunc_expl;
2775 Exp2Lb = LibFunc_exp2l;
2776 Exp10Lb = LibFunc_exp10l;
2777 PowLb = LibFunc_powl;
2778 break;
2779 default:
2780 return nullptr;
2781 }
2782
2783 // Convert libcall to intrinsic if the value is known > 0.
2784 bool IsKnownNoErrno = Log->hasNoNaNs() && Log->hasNoInfs();
2785 if (!IsKnownNoErrno) {
2786 SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
2787 KnownFPClass Known = computeKnownFPClass(
2788 V: Log->getOperand(i_nocapture: 0),
2789 InterestedClasses: KnownFPClass::OrderedLessThanZeroMask | fcSubnormal, SQ);
2790 Function *F = Log->getParent()->getParent();
2791 const fltSemantics &FltSem = Ty->getScalarType()->getFltSemantics();
2792 IsKnownNoErrno =
2793 Known.cannotBeOrderedLessThanZero() &&
2794 Known.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
2795 }
2796 if (IsKnownNoErrno) {
2797 Value *NewLog = B.CreateUnaryIntrinsic(ID: LogID, Op: Log->getArgOperand(i: 0), FMFSource: Log);
2798 if (auto *I = dyn_cast<Instruction>(Val: NewLog)) {
2799 I->copyMetadata(SrcInst: *Log);
2800 return copyFlags(Old: *Log, New: I);
2801 }
2802 return NewLog;
2803 }
2804 } else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2805 LogID == Intrinsic::log10) {
2806 if (Ty->getScalarType()->isFloatTy()) {
2807 ExpLb = LibFunc_expf;
2808 Exp2Lb = LibFunc_exp2f;
2809 Exp10Lb = LibFunc_exp10f;
2810 PowLb = LibFunc_powf;
2811 } else if (Ty->getScalarType()->isDoubleTy()) {
2812 ExpLb = LibFunc_exp;
2813 Exp2Lb = LibFunc_exp2;
2814 Exp10Lb = LibFunc_exp10;
2815 PowLb = LibFunc_pow;
2816 } else
2817 return nullptr;
2818 } else
2819 return nullptr;
2820
2821 // The earlier call must also be 'fast' in order to do these transforms.
2822 CallInst *Arg = dyn_cast<CallInst>(Val: Log->getArgOperand(i: 0));
2823 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2824 return nullptr;
2825
2826 IRBuilderBase::FastMathFlagGuard Guard(B);
2827 B.setFastMathFlags(FastMathFlags::getFast());
2828
2829 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2830 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2831
2832 // log(pow(x,y)) -> y*log(x)
2833 AttributeList NoAttrs;
2834 if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2835 Value *LogX =
2836 Log->doesNotAccessMemory()
2837 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Arg->getOperand(i_nocapture: 0), FMFSource: nullptr, Name: "log")
2838 : emitUnaryFloatFnCall(Op: Arg->getOperand(i_nocapture: 0), TLI, Name: LogNm, B, Attrs: NoAttrs);
2839 Value *Y = Arg->getArgOperand(i: 1);
2840 // Cast exponent to FP if integer.
2841 if (ArgID == Intrinsic::powi)
2842 Y = B.CreateSIToFP(V: Y, DestTy: Ty, Name: "cast");
2843 Value *MulY = B.CreateFMul(L: Y, R: LogX, Name: "mul");
2844 // Since pow() may have side effects, e.g. errno,
2845 // dead code elimination may not be trusted to remove it.
2846 substituteInParent(I: Arg, With: MulY);
2847 return MulY;
2848 }
2849
2850 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2851 // TODO: There is no exp10() intrinsic yet.
2852 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2853 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
2854 Constant *Eul;
2855 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2856 // FIXME: Add more precise value of e for long double.
2857 Eul = ConstantFP::get(Ty: Log->getType(), V: numbers::e);
2858 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2859 Eul = ConstantFP::get(Ty: Log->getType(), V: 2.0);
2860 else
2861 Eul = ConstantFP::get(Ty: Log->getType(), V: 10.0);
2862 Value *LogE = Log->doesNotAccessMemory()
2863 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Eul, FMFSource: nullptr, Name: "log")
2864 : emitUnaryFloatFnCall(Op: Eul, TLI, Name: LogNm, B, Attrs: NoAttrs);
2865 Value *MulY = B.CreateFMul(L: Arg->getArgOperand(i: 0), R: LogE, Name: "mul");
2866 // Since exp() may have side effects, e.g. errno,
2867 // dead code elimination may not be trusted to remove it.
2868 substituteInParent(I: Arg, With: MulY);
2869 return MulY;
2870 }
2871
2872 return nullptr;
2873}
2874
2875// sqrt(exp(X)) -> exp(X * 0.5)
2876Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2877 if (!CI->hasAllowReassoc())
2878 return nullptr;
2879
2880 Function *SqrtFn = CI->getCalledFunction();
2881 CallInst *Arg = dyn_cast<CallInst>(Val: CI->getArgOperand(i: 0));
2882 if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2883 return nullptr;
2884 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2885 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2886
2887 LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2888
2889 SqrtLb = TLI->getLibFunc(funcName: SqrtFn->getName());
2890 if (SqrtLb != NotLibFunc)
2891 switch (SqrtLb) {
2892 case LibFunc_sqrtf:
2893 ExpLb = LibFunc_expf;
2894 Exp2Lb = LibFunc_exp2f;
2895 Exp10Lb = LibFunc_exp10f;
2896 break;
2897 case LibFunc_sqrt:
2898 ExpLb = LibFunc_exp;
2899 Exp2Lb = LibFunc_exp2;
2900 Exp10Lb = LibFunc_exp10;
2901 break;
2902 case LibFunc_sqrtl:
2903 ExpLb = LibFunc_expl;
2904 Exp2Lb = LibFunc_exp2l;
2905 Exp10Lb = LibFunc_exp10l;
2906 break;
2907 default:
2908 return nullptr;
2909 }
2910 else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2911 if (CI->getType()->getScalarType()->isFloatTy()) {
2912 ExpLb = LibFunc_expf;
2913 Exp2Lb = LibFunc_exp2f;
2914 Exp10Lb = LibFunc_exp10f;
2915 } else if (CI->getType()->getScalarType()->isDoubleTy()) {
2916 ExpLb = LibFunc_exp;
2917 Exp2Lb = LibFunc_exp2;
2918 Exp10Lb = LibFunc_exp10;
2919 } else
2920 return nullptr;
2921 } else
2922 return nullptr;
2923
2924 if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2925 ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2926 return nullptr;
2927
2928 IRBuilderBase::InsertPointGuard Guard(B);
2929 B.SetInsertPoint(Arg);
2930 auto *ExpOperand = Arg->getOperand(i_nocapture: 0);
2931 auto *FMul =
2932 B.CreateFMulFMF(L: ExpOperand, R: ConstantFP::get(Ty: ExpOperand->getType(), V: 0.5),
2933 FMFSource: CI, Name: "merged.sqrt");
2934
2935 Arg->setOperand(i_nocapture: 0, Val_nocapture: FMul);
2936 return Arg;
2937}
2938
2939Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2940 Module *M = CI->getModule();
2941 Function *Callee = CI->getCalledFunction();
2942 Value *Ret = nullptr;
2943 // TODO: Once we have a way (other than checking for the existince of the
2944 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2945 // condition below.
2946 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_sqrtf) &&
2947 (Callee->getName() == "sqrt" ||
2948 Callee->getIntrinsicID() == Intrinsic::sqrt))
2949 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2950
2951 if (Value *Opt = mergeSqrtToExp(CI, B))
2952 return Opt;
2953
2954 if (!CI->isFast())
2955 return Ret;
2956
2957 Instruction *I = dyn_cast<Instruction>(Val: CI->getArgOperand(i: 0));
2958 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2959 return Ret;
2960
2961 // We're looking for a repeated factor in a multiplication tree,
2962 // so we can do this fold: sqrt(x * x) -> fabs(x);
2963 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2964 Value *Op0 = I->getOperand(i: 0);
2965 Value *Op1 = I->getOperand(i: 1);
2966 Value *RepeatOp = nullptr;
2967 Value *OtherOp = nullptr;
2968 if (Op0 == Op1) {
2969 // Simple match: the operands of the multiply are identical.
2970 RepeatOp = Op0;
2971 } else {
2972 // Look for a more complicated pattern: one of the operands is itself
2973 // a multiply, so search for a common factor in that multiply.
2974 // Note: We don't bother looking any deeper than this first level or for
2975 // variations of this pattern because instcombine's visitFMUL and/or the
2976 // reassociation pass should give us this form.
2977 Value *MulOp;
2978 if (match(V: Op0, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2979 cast<Instruction>(Val: Op0)->isFast()) {
2980 // Pattern: sqrt((x * x) * z)
2981 RepeatOp = MulOp;
2982 OtherOp = Op1;
2983 } else if (match(V: Op1, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2984 cast<Instruction>(Val: Op1)->isFast()) {
2985 // Pattern: sqrt(z * (x * x))
2986 RepeatOp = MulOp;
2987 OtherOp = Op0;
2988 }
2989 }
2990 if (!RepeatOp)
2991 return Ret;
2992
2993 // Fast math flags for any created instructions should match the sqrt
2994 // and multiply.
2995
2996 // If we found a repeated factor, hoist it out of the square root and
2997 // replace it with the fabs of that factor.
2998 Value *FabsCall = B.CreateFAbs(V: RepeatOp, FMFSource: I, Name: "fabs");
2999 if (OtherOp) {
3000 // If we found a non-repeated factor, we still need to get its square
3001 // root. We then multiply that by the value that was simplified out
3002 // of the square root calculation.
3003 Value *SqrtCall =
3004 B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: OtherOp, FMFSource: I, Name: "sqrt");
3005 return copyFlags(Old: *CI, New: B.CreateFMulFMF(L: FabsCall, R: SqrtCall, FMFSource: I));
3006 }
3007 return copyFlags(Old: *CI, New: FabsCall);
3008}
3009
3010Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
3011
3012 // fmod(x,y) sets errno if y == 0 or x == +/-inf. frem does not set errno,
3013 // so the fold is valid only when we can prove fmod wouldn't either.
3014 bool IsNoErrno = CI->hasNoNaNs();
3015 if (!IsNoErrno) {
3016 SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
3017 KnownFPClass Known0 = computeKnownFPClass(V: CI->getOperand(i_nocapture: 0), InterestedClasses: fcInf, SQ);
3018 if (Known0.isKnownNeverInfinity()) {
3019 KnownFPClass Known1 =
3020 computeKnownFPClass(V: CI->getOperand(i_nocapture: 1), InterestedClasses: fcZero | fcSubnormal, SQ);
3021 Function *F = CI->getParent()->getParent();
3022 const fltSemantics &FltSem =
3023 CI->getType()->getScalarType()->getFltSemantics();
3024 IsNoErrno = Known1.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
3025 }
3026 }
3027
3028 if (IsNoErrno)
3029 return B.CreateFRemFMF(L: CI->getOperand(i_nocapture: 0), R: CI->getOperand(i_nocapture: 1), FMFSource: CI);
3030 return nullptr;
3031}
3032
3033Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
3034 IRBuilderBase &B) {
3035 Module *M = CI->getModule();
3036 Function *Callee = CI->getCalledFunction();
3037 Value *Ret = nullptr;
3038 StringRef Name = Callee->getName();
3039 if (UnsafeFPShrink &&
3040 (Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
3041 Name == "asinh") &&
3042 hasFloatVersion(M, FuncName: Name))
3043 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
3044
3045 Value *Op1 = CI->getArgOperand(i: 0);
3046 auto *OpC = dyn_cast<CallInst>(Val: Op1);
3047 if (!OpC)
3048 return Ret;
3049
3050 // Both calls must be 'fast' in order to remove them.
3051 if (!CI->isFast() || !OpC->isFast())
3052 return Ret;
3053
3054 // tan(atan(x)) -> x
3055 // atanh(tanh(x)) -> x
3056 // sinh(asinh(x)) -> x
3057 // asinh(sinh(x)) -> x
3058 // cosh(acosh(x)) -> x
3059 Function *F = OpC->getCalledFunction();
3060 LibFunc Func = F ? TLI->getLibFunc(funcName: F->getName()) : NotLibFunc;
3061 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3062 LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
3063 .Case(S: "tan", Value: LibFunc_atan)
3064 .Case(S: "atanh", Value: LibFunc_tanh)
3065 .Case(S: "sinh", Value: LibFunc_asinh)
3066 .Case(S: "cosh", Value: LibFunc_acosh)
3067 .Case(S: "tanf", Value: LibFunc_atanf)
3068 .Case(S: "atanhf", Value: LibFunc_tanhf)
3069 .Case(S: "sinhf", Value: LibFunc_asinhf)
3070 .Case(S: "coshf", Value: LibFunc_acoshf)
3071 .Case(S: "tanl", Value: LibFunc_atanl)
3072 .Case(S: "atanhl", Value: LibFunc_tanhl)
3073 .Case(S: "sinhl", Value: LibFunc_asinhl)
3074 .Case(S: "coshl", Value: LibFunc_acoshl)
3075 .Case(S: "asinh", Value: LibFunc_sinh)
3076 .Case(S: "asinhf", Value: LibFunc_sinhf)
3077 .Case(S: "asinhl", Value: LibFunc_sinhl)
3078 .Default(Value: NotLibFunc); // Used as error value
3079 if (Func == inverseFunc)
3080 Ret = OpC->getArgOperand(i: 0);
3081 }
3082 return Ret;
3083}
3084
3085static bool isTrigLibCall(CallInst *CI) {
3086 // We can only hope to do anything useful if we can ignore things like errno
3087 // and floating-point exceptions.
3088 // We already checked the prototype.
3089 return CI->doesNotThrow() && CI->doesNotAccessMemory();
3090}
3091
3092static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
3093 bool UseFloat, Value *&Sin, Value *&Cos,
3094 Value *&SinCos, const TargetLibraryInfo *TLI) {
3095 Module *M = OrigCallee->getParent();
3096 Type *ArgTy = Arg->getType();
3097 Type *ResTy;
3098 StringRef Name;
3099
3100 Triple T(OrigCallee->getParent()->getTargetTriple());
3101 if (UseFloat) {
3102 Name = "__sincospif_stret";
3103
3104 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
3105 // x86_64 can't use {float, float} since that would be returned in both
3106 // xmm0 and xmm1, which isn't what a real struct would do.
3107 ResTy = T.getArch() == Triple::x86_64
3108 ? static_cast<Type *>(FixedVectorType::get(ElementType: ArgTy, NumElts: 2))
3109 : static_cast<Type *>(StructType::get(elt1: ArgTy, elts: ArgTy));
3110 } else {
3111 Name = "__sincospi_stret";
3112 ResTy = StructType::get(elt1: ArgTy, elts: ArgTy);
3113 }
3114
3115 if (!isLibFuncEmittable(M, TLI, Name))
3116 return false;
3117 LibFunc TheLibFunc = TLI->getLibFunc(funcName: Name);
3118 FunctionCallee Callee = getOrInsertLibFunc(
3119 M, TLI: *TLI, TheLibFunc, AttributeList: OrigCallee->getAttributes(), RetTy: ResTy, Args: ArgTy);
3120
3121 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
3122 // If the argument is an instruction, it must dominate all uses so put our
3123 // sincos call there.
3124 B.SetInsertPoint(++ArgInst->getIterator());
3125 } else {
3126 // Otherwise (e.g. for a constant) the beginning of the function is as
3127 // good a place as any.
3128 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
3129 B.SetInsertPoint(EntryBB.begin());
3130 }
3131
3132 SinCos = B.CreateCall(Callee, Args: Arg, Name: "sincospi");
3133
3134 if (SinCos->getType()->isStructTy()) {
3135 Sin = B.CreateExtractValue(Agg: SinCos, Idxs: 0, Name: "sinpi");
3136 Cos = B.CreateExtractValue(Agg: SinCos, Idxs: 1, Name: "cospi");
3137 } else {
3138 Sin = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{0}, Name: "sinpi");
3139 Cos = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{1}, Name: "cospi");
3140 }
3141
3142 return true;
3143}
3144
3145/// Flushing a denormal to +0.0 breaks f(-x) = -f(x) for odd f.
3146static bool mayFlushDenormalsToPositiveZero(const CallInst *CI) {
3147 DenormalMode Mode = CI->getFunction()->getDenormalMode(
3148 FPType: CI->getType()->getScalarType()->getFltSemantics());
3149 return Mode.inputsMayBePositiveZero() || Mode.outputsMayBePositiveZero();
3150}
3151
3152static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
3153 IRBuilderBase &B) {
3154 Value *X;
3155 Value *Src = CI->getArgOperand(i: 0);
3156
3157 if (match(V: Src, P: m_OneUse(SubPattern: m_FNeg(X: m_Value(V&: X)))) &&
3158 (IsEven || !mayFlushDenormalsToPositiveZero(CI))) {
3159 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3160 auto *CallInst = copyFlags(Old: *CI, New: Call);
3161 if (IsEven) {
3162 // Even function: f(-x) = f(x)
3163 return CallInst;
3164 }
3165 // Odd function: f(-x) = -f(x)
3166 return B.CreateFNegFMF(V: CallInst, FMFSource: CI);
3167 }
3168
3169 // Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
3170 if (IsEven && (match(V: Src, P: m_FAbs(Op0: m_Value(V&: X))) ||
3171 match(V: Src, P: m_CopySign(Op0: m_Value(V&: X), Op1: m_Value())))) {
3172 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3173 return copyFlags(Old: *CI, New: Call);
3174 }
3175
3176 return nullptr;
3177}
3178
3179Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
3180 IRBuilderBase &B) {
3181 switch (Func) {
3182 case LibFunc_cos:
3183 case LibFunc_cosf:
3184 case LibFunc_cosl:
3185
3186 case LibFunc_cosh:
3187 case LibFunc_coshf:
3188 case LibFunc_coshl:
3189 return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
3190
3191 case LibFunc_cbrt:
3192 case LibFunc_cbrtf:
3193 case LibFunc_cbrtl:
3194
3195 case LibFunc_sin:
3196 case LibFunc_sinf:
3197 case LibFunc_sinl:
3198
3199 case LibFunc_sinh:
3200 case LibFunc_sinhf:
3201 case LibFunc_sinhl:
3202
3203 case LibFunc_asin:
3204 case LibFunc_asinf:
3205 case LibFunc_asinl:
3206
3207 case LibFunc_asinh:
3208 case LibFunc_asinhf:
3209 case LibFunc_asinhl:
3210
3211 case LibFunc_tan:
3212 case LibFunc_tanf:
3213 case LibFunc_tanl:
3214
3215 case LibFunc_tanh:
3216 case LibFunc_tanhf:
3217 case LibFunc_tanhl:
3218
3219 case LibFunc_atan:
3220 case LibFunc_atanf:
3221 case LibFunc_atanl:
3222
3223 case LibFunc_erf:
3224 case LibFunc_erff:
3225 case LibFunc_erfl:
3226 return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
3227
3228 default:
3229 return nullptr;
3230 }
3231}
3232
3233Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
3234 // Make sure the prototype is as expected, otherwise the rest of the
3235 // function is probably invalid and likely to abort.
3236 if (!isTrigLibCall(CI))
3237 return nullptr;
3238
3239 Value *Arg = CI->getArgOperand(i: 0);
3240 if (isa<ConstantData>(Val: Arg))
3241 return nullptr;
3242
3243 SmallVector<CallInst *, 1> SinCalls;
3244 SmallVector<CallInst *, 1> CosCalls;
3245 SmallVector<CallInst *, 1> SinCosCalls;
3246
3247 bool IsFloat = Arg->getType()->isFloatTy();
3248
3249 // Look for all compatible sinpi, cospi and sincospi calls with the same
3250 // argument. If there are enough (in some sense) we can make the
3251 // substitution.
3252 Function *F = CI->getFunction();
3253 for (User *U : Arg->users())
3254 classifyArgUse(Val: U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
3255
3256 // It's only worthwhile if both sinpi and cospi are actually used.
3257 if (SinCalls.empty() || CosCalls.empty())
3258 return nullptr;
3259
3260 Value *Sin, *Cos, *SinCos;
3261 if (!insertSinCosCall(B, OrigCallee: CI->getCalledFunction(), Arg, UseFloat: IsFloat, Sin, Cos,
3262 SinCos, TLI))
3263 return nullptr;
3264
3265 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
3266 Value *Res) {
3267 for (CallInst *C : Calls)
3268 replaceAllUsesWith(I: C, With: Res);
3269 };
3270
3271 replaceTrigInsts(SinCalls, Sin);
3272 replaceTrigInsts(CosCalls, Cos);
3273 replaceTrigInsts(SinCosCalls, SinCos);
3274
3275 return IsSin ? Sin : Cos;
3276}
3277
3278void LibCallSimplifier::classifyArgUse(
3279 Value *Val, Function *F, bool IsFloat,
3280 SmallVectorImpl<CallInst *> &SinCalls,
3281 SmallVectorImpl<CallInst *> &CosCalls,
3282 SmallVectorImpl<CallInst *> &SinCosCalls) {
3283 auto *CI = dyn_cast<CallInst>(Val);
3284 if (!CI || CI->use_empty())
3285 return;
3286
3287 // Don't consider calls in other functions.
3288 if (CI->getFunction() != F)
3289 return;
3290
3291 Module *M = CI->getModule();
3292 Function *Callee = CI->getCalledFunction();
3293 LibFunc Func = Callee ? TLI->getLibFunc(FDecl: *Callee) : NotLibFunc;
3294 if (!isLibFuncEmittable(M, TLI, TheLibFunc: Func) || !isTrigLibCall(CI))
3295 return;
3296
3297 if (IsFloat) {
3298 if (Func == LibFunc_sinpif)
3299 SinCalls.push_back(Elt: CI);
3300 else if (Func == LibFunc_cospif)
3301 CosCalls.push_back(Elt: CI);
3302 else if (Func == LibFunc_sincospif_stret)
3303 SinCosCalls.push_back(Elt: CI);
3304 } else {
3305 if (Func == LibFunc_sinpi)
3306 SinCalls.push_back(Elt: CI);
3307 else if (Func == LibFunc_cospi)
3308 CosCalls.push_back(Elt: CI);
3309 else if (Func == LibFunc_sincospi_stret)
3310 SinCosCalls.push_back(Elt: CI);
3311 }
3312}
3313
3314/// Constant folds remquo
3315Value *LibCallSimplifier::optimizeRemquo(CallInst *CI, IRBuilderBase &B) {
3316 const APFloat *X, *Y;
3317 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3318 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3319 return nullptr;
3320
3321 APFloat::opStatus Status;
3322 APFloat Quot = *X;
3323 Status = Quot.divide(RHS: *Y, RM: APFloat::rmNearestTiesToEven);
3324 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3325 return nullptr;
3326 APFloat Rem = *X;
3327 if (Rem.remainder(RHS: *Y) != APFloat::opOK)
3328 return nullptr;
3329
3330 // TODO: We can only keep at least the three of the last bits of x/y
3331 unsigned IntBW = TLI->getIntSize();
3332 APSInt QuotInt(IntBW, /*isUnsigned=*/false);
3333 bool IsExact;
3334 Status =
3335 Quot.convertToInteger(Result&: QuotInt, RM: APFloat::rmNearestTiesToEven, IsExact: &IsExact);
3336 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3337 return nullptr;
3338
3339 B.CreateAlignedStore(
3340 Val: ConstantInt::getSigned(Ty: B.getIntNTy(N: IntBW), V: QuotInt.getExtValue()),
3341 Ptr: CI->getArgOperand(i: 2), Align: CI->getParamAlign(ArgNo: 2));
3342 return ConstantFP::get(Ty: CI->getType(), V: Rem);
3343}
3344
3345/// Constant folds fdim
3346Value *LibCallSimplifier::optimizeFdim(CallInst *CI, IRBuilderBase &B) {
3347 // Cannot perform the fold unless the call has attribute memory(none)
3348 if (!CI->doesNotAccessMemory())
3349 return nullptr;
3350
3351 // TODO : Handle undef values
3352 // Propagate poison if any
3353 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 0)))
3354 return CI->getArgOperand(i: 0);
3355 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 1)))
3356 return CI->getArgOperand(i: 1);
3357
3358 const APFloat *X, *Y;
3359 // Check if both values are constants
3360 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3361 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3362 return nullptr;
3363
3364 // C99 fdim(x, y) = (x > y) ? x - y : +0.
3365 if (X->compare(RHS: *Y) != APFloat::cmpGreaterThan && !X->isNaN() && !Y->isNaN())
3366 return ConstantFP::getZero(Ty: CI->getType());
3367 APFloat Difference = *X;
3368 Difference.subtract(RHS: *Y, RM: RoundingMode::NearestTiesToEven);
3369 return ConstantFP::get(Ty: CI->getType(), V: Difference);
3370}
3371
3372//===----------------------------------------------------------------------===//
3373// Integer Library Call Optimizations
3374//===----------------------------------------------------------------------===//
3375
3376Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
3377 // All variants of ffs return int which need not be 32 bits wide.
3378 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
3379 Type *RetType = CI->getType();
3380 Value *Op = CI->getArgOperand(i: 0);
3381 Type *ArgType = Op->getType();
3382 Value *V = B.CreateIntrinsic(ID: Intrinsic::cttz, OverloadTypes: {ArgType}, Args: {Op, B.getTrue()},
3383 FMFSource: nullptr, Name: "cttz");
3384 V = B.CreateAdd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 1));
3385 V = B.CreateIntCast(V, DestTy: RetType, isSigned: false);
3386
3387 Value *Cond = B.CreateICmpNE(LHS: Op, RHS: Constant::getNullValue(Ty: ArgType));
3388 Value *S = B.CreateSelect(C: Cond, True: V, False: ConstantInt::get(Ty: RetType, V: 0));
3389 if (ProfcheckDisableMetadataFixes)
3390 return S;
3391 if (auto *SI = dyn_cast<SelectInst>(Val: S))
3392 setBranchWeights(
3393 I&: *SI, Weights: {MDBuilder::kLikelyBranchWeight, MDBuilder::kUnlikelyBranchWeight},
3394 /*IsExpected=*/false);
3395 return S;
3396}
3397
3398Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
3399 // All variants of fls return int which need not be 32 bits wide.
3400 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
3401 Value *Op = CI->getArgOperand(i: 0);
3402 Type *ArgType = Op->getType();
3403 Value *V = B.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {ArgType}, Args: {Op, B.getFalse()},
3404 FMFSource: nullptr, Name: "ctlz");
3405 V = B.CreateSub(LHS: ConstantInt::get(Ty: V->getType(), V: ArgType->getIntegerBitWidth()),
3406 RHS: V);
3407 return B.CreateIntCast(V, DestTy: CI->getType(), isSigned: false);
3408}
3409
3410Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
3411 // abs(x) -> x <s 0 ? -x : x
3412 // The negation has 'nsw' because abs of INT_MIN is undefined.
3413 Value *X = CI->getArgOperand(i: 0);
3414 Value *IsNeg = B.CreateIsNeg(Arg: X);
3415 Value *NegX = B.CreateNSWNeg(V: X, Name: "neg");
3416 return B.CreateSelect(C: IsNeg, True: NegX, False: X);
3417}
3418
3419Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
3420 // isdigit(c) -> (c-'0') <u 10
3421 Value *Op = CI->getArgOperand(i: 0);
3422 Type *ArgType = Op->getType();
3423 Op = B.CreateSub(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: '0'), Name: "isdigittmp");
3424 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 10), Name: "isdigit");
3425 return B.CreateZExt(V: Op, DestTy: CI->getType());
3426}
3427
3428Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
3429 // isascii(c) -> c <u 128
3430 Value *Op = CI->getArgOperand(i: 0);
3431 Type *ArgType = Op->getType();
3432 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 128), Name: "isascii");
3433 return B.CreateZExt(V: Op, DestTy: CI->getType());
3434}
3435
3436Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
3437 // toascii(c) -> c & 0x7f
3438 return B.CreateAnd(LHS: CI->getArgOperand(i: 0),
3439 RHS: ConstantInt::get(Ty: CI->getType(), V: 0x7F));
3440}
3441
3442// Fold calls to atoi, atol, and atoll.
3443Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
3444 StringRef Str;
3445 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3446 return nullptr;
3447
3448 return convertStrToInt(CI, Str, EndPtr: nullptr, Base: 10, /*AsSigned=*/true, B);
3449}
3450
3451// Fold calls to strtol, strtoll, strtoul, and strtoull.
3452Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
3453 bool AsSigned) {
3454 Value *EndPtr = CI->getArgOperand(i: 1);
3455 if (isa<ConstantPointerNull>(Val: EndPtr)) {
3456 // With a null EndPtr, this function won't capture the main argument.
3457 // It would be readonly too, except that it still may write to errno.
3458 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
3459 CI: CaptureInfo::none()));
3460 EndPtr = nullptr;
3461 } else if (!isKnownNonZero(V: EndPtr, Q: DL))
3462 return nullptr;
3463
3464 StringRef Str;
3465 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3466 return nullptr;
3467
3468 if (ConstantInt *CInt = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2))) {
3469 return convertStrToInt(CI, Str, EndPtr, Base: CInt->getSExtValue(), AsSigned, B);
3470 }
3471
3472 return nullptr;
3473}
3474
3475//===----------------------------------------------------------------------===//
3476// Formatting and IO Library Call Optimizations
3477//===----------------------------------------------------------------------===//
3478
3479static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
3480
3481Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3482 int StreamArg) {
3483 Function *Callee = CI->getCalledFunction();
3484 // Error reporting calls should be cold, mark them as such.
3485 // This applies even to non-builtin calls: it is only a hint and applies to
3486 // functions that the frontend might not understand as builtins.
3487
3488 // This heuristic was suggested in:
3489 // Improving Static Branch Prediction in a Compiler
3490 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3491 // Proceedings of PACT'98, Oct. 1998, IEEE
3492 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
3493 isReportingError(Callee, CI, StreamArg)) {
3494 CI->addFnAttr(Kind: Attribute::Cold);
3495 }
3496
3497 return nullptr;
3498}
3499
3500static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3501 if (!Callee || !Callee->isDeclaration())
3502 return false;
3503
3504 if (StreamArg < 0)
3505 return true;
3506
3507 // These functions might be considered cold, but only if their stream
3508 // argument is stderr.
3509
3510 if (StreamArg >= (int)CI->arg_size())
3511 return false;
3512 LoadInst *LI = dyn_cast<LoadInst>(Val: CI->getArgOperand(i: StreamArg));
3513 if (!LI)
3514 return false;
3515 GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: LI->getPointerOperand());
3516 if (!GV || !GV->isDeclaration())
3517 return false;
3518 return GV->getName() == "stderr";
3519}
3520
3521Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3522 // Check for a fixed format string.
3523 StringRef FormatStr;
3524 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: FormatStr))
3525 return nullptr;
3526
3527 // Empty format string -> noop.
3528 if (FormatStr.empty()) // Tolerate printf's declared void.
3529 return CI->use_empty() ? (Value *)CI : ConstantInt::get(Ty: CI->getType(), V: 0);
3530
3531 // Do not do any of the following transformations if the printf return value
3532 // is used, in general the printf return value is not compatible with either
3533 // putchar() or puts().
3534 if (!CI->use_empty())
3535 return nullptr;
3536
3537 Type *IntTy = CI->getType();
3538 // printf("x") -> putchar('x'), even for "%" and "%%".
3539 if (FormatStr.size() == 1 || FormatStr == "%%") {
3540 // Convert the character to unsigned char before passing it to putchar
3541 // to avoid host-specific sign extension in the IR. Putchar converts
3542 // it to unsigned char regardless.
3543 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)FormatStr[0]);
3544 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3545 }
3546
3547 // Try to remove call or emit putchar/puts.
3548 if (FormatStr == "%s" && CI->arg_size() > 1) {
3549 StringRef OperandStr;
3550 if (!getConstantStringInfo(V: CI->getOperand(i_nocapture: 1), Str&: OperandStr))
3551 return nullptr;
3552 // printf("%s", "") --> NOP
3553 if (OperandStr.empty())
3554 return (Value *)CI;
3555 // printf("%s", "a") --> putchar('a')
3556 if (OperandStr.size() == 1) {
3557 // Convert the character to unsigned char before passing it to putchar
3558 // to avoid host-specific sign extension in the IR. Putchar converts
3559 // it to unsigned char regardless.
3560 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)OperandStr[0]);
3561 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3562 }
3563 // printf("%s", str"\n") --> puts(str)
3564 if (OperandStr.back() == '\n') {
3565 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3566 return nullptr;
3567 OperandStr = OperandStr.drop_back();
3568 Value *GV = B.CreateGlobalString(Str: OperandStr, Name: "str");
3569 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3570 }
3571 return nullptr;
3572 }
3573
3574 // printf("foo\n") --> puts("foo")
3575 if (FormatStr.back() == '\n' &&
3576 !FormatStr.contains(C: '%')) { // No format characters.
3577 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3578 return nullptr;
3579 // Create a string literal with no \n on it. We expect the constant merge
3580 // pass to be run after this pass, to merge duplicate strings.
3581 FormatStr = FormatStr.drop_back();
3582 Value *GV = B.CreateGlobalString(Str: FormatStr, Name: "str");
3583 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3584 }
3585
3586 // Optimize specific format strings.
3587 // printf("%c", chr) --> putchar(chr)
3588 if (FormatStr == "%c" && CI->arg_size() > 1 &&
3589 CI->getArgOperand(i: 1)->getType()->isIntegerTy()) {
3590 // Convert the argument to the type expected by putchar, i.e., int, which
3591 // need not be 32 bits wide but which is the same as printf's return type.
3592 Value *IntChar = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: IntTy, isSigned: false);
3593 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3594 }
3595
3596 // printf("%s\n", str) --> puts(str)
3597 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3598 CI->getArgOperand(i: 1)->getType()->isPointerTy())
3599 return copyFlags(Old: *CI, New: emitPutS(Str: CI->getArgOperand(i: 1), B, TLI));
3600 return nullptr;
3601}
3602
3603Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3604
3605 Module *M = CI->getModule();
3606 Function *Callee = CI->getCalledFunction();
3607 FunctionType *FT = Callee->getFunctionType();
3608 if (Value *V = optimizePrintFString(CI, B)) {
3609 return V;
3610 }
3611
3612 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3613
3614 // printf(format, ...) -> iprintf(format, ...) if no floating point
3615 // arguments.
3616 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_iprintf) &&
3617 !callHasFloatingPointArgument(CI)) {
3618 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_iprintf, T: FT,
3619 AttributeList: Callee->getAttributes());
3620 CallInst *New = cast<CallInst>(Val: CI->clone());
3621 New->setCalledFunction(IPrintFFn);
3622 B.Insert(I: New);
3623 return New;
3624 }
3625
3626 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3627 // arguments.
3628 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_printf) &&
3629 !callHasFP128Argument(CI)) {
3630 auto SmallPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_printf, T: FT,
3631 AttributeList: Callee->getAttributes());
3632 CallInst *New = cast<CallInst>(Val: CI->clone());
3633 New->setCalledFunction(SmallPrintFFn);
3634 B.Insert(I: New);
3635 return New;
3636 }
3637
3638 return nullptr;
3639}
3640
3641Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3642 IRBuilderBase &B) {
3643 // Check for a fixed format string.
3644 StringRef FormatStr;
3645 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3646 return nullptr;
3647
3648 // If we just have a format string (nothing else crazy) transform it.
3649 Value *Dest = CI->getArgOperand(i: 0);
3650 if (CI->arg_size() == 2) {
3651 // Make sure there's no % in the constant array. We could try to handle
3652 // %% -> % in the future if we cared.
3653 if (FormatStr.contains(C: '%'))
3654 return nullptr; // we found a format specifier, bail out.
3655
3656 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3657 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1),
3658 // Copy the null byte.
3659 Size: TLI->getAsSizeT(V: FormatStr.size() + 1, M: *CI->getModule()));
3660 return ConstantInt::get(Ty: CI->getType(), V: FormatStr.size());
3661 }
3662
3663 // The remaining optimizations require the format string to be "%s" or "%c"
3664 // and have an extra operand.
3665 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3666 return nullptr;
3667
3668 // Decode the second character of the format string.
3669 if (FormatStr[1] == 'c') {
3670 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3671 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3672 return nullptr;
3673 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 2), DestTy: B.getInt8Ty(), Name: "char");
3674 Value *Ptr = Dest;
3675 B.CreateStore(Val: V, Ptr);
3676 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3677 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3678
3679 return ConstantInt::get(Ty: CI->getType(), V: 1);
3680 }
3681
3682 if (FormatStr[1] == 's') {
3683 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3684 // strlen(str)+1)
3685 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3686 return nullptr;
3687
3688 if (CI->use_empty())
3689 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3690 return copyFlags(Old: *CI, New: emitStrCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI));
3691
3692 uint64_t SrcLen = GetStringLength(V: CI->getArgOperand(i: 2));
3693 if (SrcLen) {
3694 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1),
3695 Size: TLI->getAsSizeT(V: SrcLen, M: *CI->getModule()));
3696 // Returns total number of characters written without null-character.
3697 return ConstantInt::get(Ty: CI->getType(), V: SrcLen - 1);
3698 } else if (Value *V = emitStpCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI)) {
3699 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3700 Value *PtrDiff = B.CreatePtrDiff(LHS: V, RHS: Dest);
3701 return B.CreateIntCast(V: PtrDiff, DestTy: CI->getType(), isSigned: false);
3702 }
3703
3704 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3705 QueryType: PGSOQueryType::IRPass))
3706 return nullptr;
3707
3708 Value *Len = emitStrLen(Ptr: CI->getArgOperand(i: 2), B, DL, TLI);
3709 if (!Len)
3710 return nullptr;
3711 Value *IncLen =
3712 B.CreateAdd(LHS: Len, RHS: ConstantInt::get(Ty: Len->getType(), V: 1), Name: "leninc");
3713 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1), Size: IncLen);
3714
3715 // The sprintf result is the unincremented number of bytes in the string.
3716 return B.CreateIntCast(V: Len, DestTy: CI->getType(), isSigned: false);
3717 }
3718 return nullptr;
3719}
3720
3721Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3722 Module *M = CI->getModule();
3723 Function *Callee = CI->getCalledFunction();
3724 FunctionType *FT = Callee->getFunctionType();
3725 if (Value *V = optimizeSPrintFString(CI, B)) {
3726 return V;
3727 }
3728
3729 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
3730
3731 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3732 // point arguments.
3733 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_siprintf) &&
3734 !callHasFloatingPointArgument(CI)) {
3735 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_siprintf,
3736 T: FT, AttributeList: Callee->getAttributes());
3737 CallInst *New = cast<CallInst>(Val: CI->clone());
3738 New->setCalledFunction(SIPrintFFn);
3739 B.Insert(I: New);
3740 return New;
3741 }
3742
3743 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3744 // floating point arguments.
3745 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_sprintf) &&
3746 !callHasFP128Argument(CI)) {
3747 auto SmallSPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_sprintf, T: FT,
3748 AttributeList: Callee->getAttributes());
3749 CallInst *New = cast<CallInst>(Val: CI->clone());
3750 New->setCalledFunction(SmallSPrintFFn);
3751 B.Insert(I: New);
3752 return New;
3753 }
3754
3755 return nullptr;
3756}
3757
3758// Transform an snprintf call CI with the bound N to format the string Str
3759// either to a call to memcpy, or to single character a store, or to nothing,
3760// and fold the result to a constant. A nonnull StrArg refers to the string
3761// argument being formatted. Otherwise the call is one with N < 2 and
3762// the "%c" directive to format a single character.
3763Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3764 StringRef Str, uint64_t N,
3765 IRBuilderBase &B) {
3766 assert(StrArg || (N < 2 && Str.size() == 1));
3767
3768 unsigned IntBits = TLI->getIntSize();
3769 uint64_t IntMax = maxIntN(N: IntBits);
3770 if (Str.size() > IntMax)
3771 // Bail if the string is longer than INT_MAX. POSIX requires
3772 // implementations to set errno to EOVERFLOW in this case, in
3773 // addition to when N is larger than that (checked by the caller).
3774 return nullptr;
3775
3776 Value *StrLen = ConstantInt::get(Ty: CI->getType(), V: Str.size());
3777 if (N == 0)
3778 return StrLen;
3779
3780 // Set to the number of bytes to copy fron StrArg which is also
3781 // the offset of the terinating nul.
3782 uint64_t NCopy;
3783 if (N > Str.size())
3784 // Copy the full string, including the terminating nul (which must
3785 // be present regardless of the bound).
3786 NCopy = Str.size() + 1;
3787 else
3788 NCopy = N - 1;
3789
3790 Value *DstArg = CI->getArgOperand(i: 0);
3791 if (NCopy && StrArg)
3792 // Transform the call to lvm.memcpy(dst, fmt, N).
3793 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst: DstArg, DstAlign: Align(1), Src: StrArg, SrcAlign: Align(1),
3794 Size: TLI->getAsSizeT(V: NCopy, M: *CI->getModule())));
3795
3796 if (N > Str.size())
3797 // Return early when the whole format string, including the final nul,
3798 // has been copied.
3799 return StrLen;
3800
3801 // Otherwise, when truncating the string append a terminating nul.
3802 Type *Int8Ty = B.getInt8Ty();
3803 Value *NulOff = B.getIntN(N: IntBits, C: NCopy);
3804 Value *DstEnd = B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: DstArg, IdxList: NulOff, Name: "endptr");
3805 B.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: 0), Ptr: DstEnd);
3806 return StrLen;
3807}
3808
3809Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3810 IRBuilderBase &B) {
3811 // Check for size
3812 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3813 if (!Size)
3814 return nullptr;
3815
3816 uint64_t N = Size->getZExtValue();
3817 uint64_t IntMax = maxIntN(N: TLI->getIntSize());
3818 if (N > IntMax)
3819 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3820 // to set errno to EOVERFLOW in this case.
3821 return nullptr;
3822
3823 Value *DstArg = CI->getArgOperand(i: 0);
3824 Value *FmtArg = CI->getArgOperand(i: 2);
3825
3826 // Check for a fixed format string.
3827 StringRef FormatStr;
3828 if (!getConstantStringInfo(V: FmtArg, Str&: FormatStr))
3829 return nullptr;
3830
3831 // If we just have a format string (nothing else crazy) transform it.
3832 if (CI->arg_size() == 3) {
3833 if (FormatStr.contains(C: '%'))
3834 // Bail if the format string contains a directive and there are
3835 // no arguments. We could handle "%%" in the future.
3836 return nullptr;
3837
3838 return emitSnPrintfMemCpy(CI, StrArg: FmtArg, Str: FormatStr, N, B);
3839 }
3840
3841 // The remaining optimizations require the format string to be "%s" or "%c"
3842 // and have an extra operand.
3843 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3844 return nullptr;
3845
3846 // Decode the second character of the format string.
3847 if (FormatStr[1] == 'c') {
3848 if (N <= 1) {
3849 // Use an arbitary string of length 1 to transform the call into
3850 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3851 // to one.
3852 StringRef CharStr("*");
3853 return emitSnPrintfMemCpy(CI, StrArg: nullptr, Str: CharStr, N, B);
3854 }
3855
3856 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3857 if (!CI->getArgOperand(i: 3)->getType()->isIntegerTy())
3858 return nullptr;
3859 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 3), DestTy: B.getInt8Ty(), Name: "char");
3860 Value *Ptr = DstArg;
3861 B.CreateStore(Val: V, Ptr);
3862 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3863 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3864 return ConstantInt::get(Ty: CI->getType(), V: 1);
3865 }
3866
3867 if (FormatStr[1] != 's')
3868 return nullptr;
3869
3870 Value *StrArg = CI->getArgOperand(i: 3);
3871 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3872 StringRef Str;
3873 if (!getConstantStringInfo(V: StrArg, Str))
3874 return nullptr;
3875
3876 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3877}
3878
3879Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3880 if (Value *V = optimizeSnPrintFString(CI, B)) {
3881 return V;
3882 }
3883
3884 if (isKnownNonZero(V: CI->getOperand(i_nocapture: 1), Q: DL))
3885 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3886 return nullptr;
3887}
3888
3889Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3890 IRBuilderBase &B) {
3891 optimizeErrorReporting(CI, B, StreamArg: 0);
3892
3893 // All the optimizations depend on the format string.
3894 StringRef FormatStr;
3895 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3896 return nullptr;
3897
3898 // Do not do any of the following transformations if the fprintf return
3899 // value is used, in general the fprintf return value is not compatible
3900 // with fwrite(), fputc() or fputs().
3901 if (!CI->use_empty())
3902 return nullptr;
3903
3904 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3905 if (CI->arg_size() == 2) {
3906 // Could handle %% -> % if we cared.
3907 if (FormatStr.contains(C: '%'))
3908 return nullptr; // We found a format specifier.
3909
3910 return copyFlags(
3911 Old: *CI, New: emitFWrite(Ptr: CI->getArgOperand(i: 1),
3912 Size: TLI->getAsSizeT(V: FormatStr.size(), M: *CI->getModule()),
3913 File: CI->getArgOperand(i: 0), B, DL, TLI));
3914 }
3915
3916 // The remaining optimizations require the format string to be "%s" or "%c"
3917 // and have an extra operand.
3918 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3919 return nullptr;
3920
3921 // Decode the second character of the format string.
3922 if (FormatStr[1] == 'c') {
3923 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3924 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3925 return nullptr;
3926 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3927 Value *V = B.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntTy, /*isSigned*/ true,
3928 Name: "chari");
3929 return copyFlags(Old: *CI, New: emitFPutC(Char: V, File: CI->getArgOperand(i: 0), B, TLI));
3930 }
3931
3932 if (FormatStr[1] == 's') {
3933 // fprintf(F, "%s", str) --> fputs(str, F)
3934 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3935 return nullptr;
3936 return copyFlags(
3937 Old: *CI, New: emitFPutS(Str: CI->getArgOperand(i: 2), File: CI->getArgOperand(i: 0), B, TLI));
3938 }
3939 return nullptr;
3940}
3941
3942Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3943 Module *M = CI->getModule();
3944 Function *Callee = CI->getCalledFunction();
3945 FunctionType *FT = Callee->getFunctionType();
3946 if (Value *V = optimizeFPrintFString(CI, B)) {
3947 return V;
3948 }
3949
3950 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3951 // floating point arguments.
3952 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_fiprintf) &&
3953 !callHasFloatingPointArgument(CI)) {
3954 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_fiprintf,
3955 T: FT, AttributeList: Callee->getAttributes());
3956 CallInst *New = cast<CallInst>(Val: CI->clone());
3957 New->setCalledFunction(FIPrintFFn);
3958 B.Insert(I: New);
3959 return New;
3960 }
3961
3962 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3963 // 128-bit floating point arguments.
3964 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_fprintf) &&
3965 !callHasFP128Argument(CI)) {
3966 auto SmallFPrintFFn =
3967 getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_fprintf, T: FT,
3968 AttributeList: Callee->getAttributes());
3969 CallInst *New = cast<CallInst>(Val: CI->clone());
3970 New->setCalledFunction(SmallFPrintFFn);
3971 B.Insert(I: New);
3972 return New;
3973 }
3974
3975 return nullptr;
3976}
3977
3978Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3979 optimizeErrorReporting(CI, B, StreamArg: 3);
3980
3981 // Get the element size and count.
3982 ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3983 ConstantInt *CountC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
3984 if (SizeC && CountC) {
3985 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3986
3987 // If this is writing zero records, remove the call (it's a noop).
3988 if (Bytes == 0)
3989 return ConstantInt::get(Ty: CI->getType(), V: 0);
3990
3991 // If this is writing one byte, turn it into fputc.
3992 // This optimisation is only valid, if the return value is unused.
3993 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
3994 Value *Char = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0), Name: "char");
3995 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3996 Value *Cast = B.CreateIntCast(V: Char, DestTy: IntTy, /*isSigned*/ true, Name: "chari");
3997 Value *NewCI = emitFPutC(Char: Cast, File: CI->getArgOperand(i: 3), B, TLI);
3998 return NewCI ? ConstantInt::get(Ty: CI->getType(), V: 1) : nullptr;
3999 }
4000 }
4001
4002 return nullptr;
4003}
4004
4005Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
4006 optimizeErrorReporting(CI, B, StreamArg: 1);
4007
4008 // Don't rewrite fputs to fwrite when optimising for size because fwrite
4009 // requires more arguments and thus extra MOVs are required.
4010 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
4011 QueryType: PGSOQueryType::IRPass))
4012 return nullptr;
4013
4014 // We can't optimize if return value is used.
4015 if (!CI->use_empty())
4016 return nullptr;
4017
4018 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
4019 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: 0));
4020 if (!Len)
4021 return nullptr;
4022
4023 // Known to have no uses (see above).
4024 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4025 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4026 return copyFlags(
4027 Old: *CI,
4028 New: emitFWrite(Ptr: CI->getArgOperand(i: 0),
4029 Size: ConstantInt::get(Ty: SizeTTy, V: Len - 1),
4030 File: CI->getArgOperand(i: 1), B, DL, TLI));
4031}
4032
4033Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
4034 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
4035 if (!CI->use_empty())
4036 return nullptr;
4037
4038 // Check for a constant string.
4039 // puts("") -> putchar('\n')
4040 StringRef Str;
4041 if (getConstantStringInfo(V: CI->getArgOperand(i: 0), Str) && Str.empty()) {
4042 // putchar takes an argument of the same type as puts returns, i.e.,
4043 // int, which need not be 32 bits wide.
4044 Type *IntTy = CI->getType();
4045 return copyFlags(Old: *CI, New: emitPutChar(Char: ConstantInt::get(Ty: IntTy, V: '\n'), B, TLI));
4046 }
4047
4048 return nullptr;
4049}
4050
4051Value *LibCallSimplifier::optimizeExit(CallInst *CI) {
4052
4053 // Mark 'exit' as cold if its not exit(0) (success).
4054 const APInt *C;
4055 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
4056 match(V: CI->getArgOperand(i: 0), P: m_APInt(Res&: C)) && !C->isZero()) {
4057 CI->addFnAttr(Kind: Attribute::Cold);
4058 }
4059 return nullptr;
4060}
4061
4062Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
4063 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
4064 return copyFlags(Old: *CI, New: B.CreateMemMove(Dst: CI->getArgOperand(i: 1), DstAlign: Align(1),
4065 Src: CI->getArgOperand(i: 0), SrcAlign: Align(1),
4066 Size: CI->getArgOperand(i: 2)));
4067}
4068
4069bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
4070 SmallString<20> FloatFuncName = FuncName;
4071 FloatFuncName += 'f';
4072 return isLibFuncEmittable(M, TLI, Name: FloatFuncName);
4073}
4074
4075Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
4076 IRBuilderBase &Builder) {
4077 Module *M = CI->getModule();
4078 Function *Callee = CI->getCalledFunction();
4079 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4080
4081 // Check for string/memory library functions.
4082 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4083 // Make sure we never change the calling convention.
4084 assert(
4085 (ignoreCallingConv(Func) ||
4086 TargetLibraryInfoImpl::isCallingConvCCompatible(CI)) &&
4087 "Optimizing string/memory libcall would change the calling convention");
4088 switch (Func) {
4089 case LibFunc_strcat:
4090 return optimizeStrCat(CI, B&: Builder);
4091 case LibFunc_strncat:
4092 return optimizeStrNCat(CI, B&: Builder);
4093 case LibFunc_strchr:
4094 return optimizeStrChr(CI, B&: Builder);
4095 case LibFunc_strrchr:
4096 return optimizeStrRChr(CI, B&: Builder);
4097 case LibFunc_strcmp:
4098 return optimizeStrCmp(CI, B&: Builder);
4099 case LibFunc_strncmp:
4100 return optimizeStrNCmp(CI, B&: Builder);
4101 case LibFunc_strcpy:
4102 return optimizeStrCpy(CI, B&: Builder);
4103 case LibFunc_stpcpy:
4104 return optimizeStpCpy(CI, B&: Builder);
4105 case LibFunc_strlcpy:
4106 return optimizeStrLCpy(CI, B&: Builder);
4107 case LibFunc_stpncpy:
4108 return optimizeStringNCpy(CI, /*RetEnd=*/true, B&: Builder);
4109 case LibFunc_strncpy:
4110 return optimizeStringNCpy(CI, /*RetEnd=*/false, B&: Builder);
4111 case LibFunc_strlen:
4112 return optimizeStrLen(CI, B&: Builder);
4113 case LibFunc_strnlen:
4114 return optimizeStrNLen(CI, B&: Builder);
4115 case LibFunc_strpbrk:
4116 return optimizeStrPBrk(CI, B&: Builder);
4117 case LibFunc_strndup:
4118 return optimizeStrNDup(CI, B&: Builder);
4119 case LibFunc_strtol:
4120 case LibFunc_strtod:
4121 case LibFunc_strtof:
4122 case LibFunc_strtoul:
4123 case LibFunc_strtoll:
4124 case LibFunc_strtold:
4125 case LibFunc_strtoull:
4126 return optimizeStrTo(CI, B&: Builder);
4127 case LibFunc_strspn:
4128 return optimizeStrSpn(CI, B&: Builder);
4129 case LibFunc_strcspn:
4130 return optimizeStrCSpn(CI, B&: Builder);
4131 case LibFunc_strstr:
4132 return optimizeStrStr(CI, B&: Builder);
4133 case LibFunc_memchr:
4134 return optimizeMemChr(CI, B&: Builder);
4135 case LibFunc_memrchr:
4136 return optimizeMemRChr(CI, B&: Builder);
4137 case LibFunc_bcmp:
4138 return optimizeBCmp(CI, B&: Builder);
4139 case LibFunc_memcmp:
4140 return optimizeMemCmp(CI, B&: Builder);
4141 case LibFunc_memcpy:
4142 return optimizeMemCpy(CI, B&: Builder);
4143 case LibFunc_memccpy:
4144 return optimizeMemCCpy(CI, B&: Builder);
4145 case LibFunc_mempcpy:
4146 return optimizeMemPCpy(CI, B&: Builder);
4147 case LibFunc_memmove:
4148 return optimizeMemMove(CI, B&: Builder);
4149 case LibFunc_memset:
4150 return optimizeMemSet(CI, B&: Builder);
4151 case LibFunc_realloc:
4152 return optimizeRealloc(CI, B&: Builder);
4153 case LibFunc_wcslen:
4154 return optimizeWcslen(CI, B&: Builder);
4155 case LibFunc_bcopy:
4156 return optimizeBCopy(CI, B&: Builder);
4157 case LibFunc_Znwm:
4158 case LibFunc_ZnwmRKSt9nothrow_t:
4159 case LibFunc_ZnwmSt11align_val_t:
4160 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
4161 case LibFunc_Znam:
4162 case LibFunc_ZnamRKSt9nothrow_t:
4163 case LibFunc_ZnamSt11align_val_t:
4164 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
4165 case LibFunc_Znwm12__hot_cold_t:
4166 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
4167 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
4168 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4169 case LibFunc_Znam12__hot_cold_t:
4170 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
4171 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
4172 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4173 case LibFunc_size_returning_new:
4174 case LibFunc_size_returning_new_hot_cold:
4175 case LibFunc_size_returning_new_aligned:
4176 case LibFunc_size_returning_new_aligned_hot_cold:
4177 return optimizeNew(CI, B&: Builder, Func);
4178 default:
4179 break;
4180 }
4181 }
4182 return nullptr;
4183}
4184
4185/// Constant folding nan/nanf/nanl.
4186static Value *optimizeNaN(CallInst *CI) {
4187 StringRef CharSeq;
4188 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: CharSeq))
4189 return nullptr;
4190
4191 APInt Fill;
4192 // Treat empty strings as if they were zero.
4193 if (CharSeq.empty())
4194 Fill = APInt(32, 0);
4195 else if (CharSeq.getAsInteger(Radix: 0, Result&: Fill))
4196 return nullptr;
4197
4198 return ConstantFP::getQNaN(Ty: CI->getType(), /*Negative=*/false, Payload: &Fill);
4199}
4200
4201Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
4202 LibFunc Func,
4203 IRBuilderBase &Builder) {
4204 const Module *M = CI->getModule();
4205
4206 // Don't optimize calls that require strict floating point semantics.
4207 if (CI->isStrictFP())
4208 return nullptr;
4209
4210 if (Value *V = optimizeSymmetric(CI, Func, B&: Builder))
4211 return V;
4212
4213 switch (Func) {
4214 case LibFunc_sinpif:
4215 case LibFunc_sinpi:
4216 return optimizeSinCosPi(CI, /*IsSin*/true, B&: Builder);
4217 case LibFunc_cospif:
4218 case LibFunc_cospi:
4219 return optimizeSinCosPi(CI, /*IsSin*/false, B&: Builder);
4220 case LibFunc_sinf:
4221 case LibFunc_sinl:
4222 if (CI->doesNotAccessMemory())
4223 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::sin);
4224 return nullptr;
4225 case LibFunc_cosf:
4226 case LibFunc_cosl:
4227 if (CI->doesNotAccessMemory())
4228 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::cos);
4229 return nullptr;
4230 case LibFunc_powf:
4231 case LibFunc_pow:
4232 case LibFunc_powl:
4233 return optimizePow(Pow: CI, B&: Builder);
4234 case LibFunc_exp2l:
4235 case LibFunc_exp2:
4236 case LibFunc_exp2f:
4237 return optimizeExp2(CI, B&: Builder);
4238 case LibFunc_scalbn:
4239 case LibFunc_scalbnf:
4240 case LibFunc_scalbnl:
4241 // LLVM floating-point types have radix 2, so scalbn is equivalent to
4242 // ldexp. Do not replace a libcall that may set errno.
4243 if (CI->doesNotAccessMemory()) {
4244 Value *NewCall =
4245 Builder.CreateLdexp(Src: CI->getArgOperand(i: 0), Exp: CI->getArgOperand(i: 1), FMFSource: CI);
4246 NewCall->takeName(V: CI);
4247 return copyFlags(Old: *CI, New: NewCall);
4248 }
4249 return nullptr;
4250 case LibFunc_fabsf:
4251 case LibFunc_fabs:
4252 case LibFunc_fabsl:
4253 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::fabs);
4254 case LibFunc_sqrtf:
4255 case LibFunc_sqrt:
4256 case LibFunc_sqrtl:
4257 return optimizeSqrt(CI, B&: Builder);
4258 case LibFunc_fmod:
4259 case LibFunc_fmodf:
4260 case LibFunc_fmodl:
4261 return optimizeFMod(CI, B&: Builder);
4262 case LibFunc_logf:
4263 case LibFunc_log:
4264 case LibFunc_logl:
4265 case LibFunc_log10f:
4266 case LibFunc_log10:
4267 case LibFunc_log10l:
4268 case LibFunc_log1pf:
4269 case LibFunc_log1p:
4270 case LibFunc_log1pl:
4271 case LibFunc_log2f:
4272 case LibFunc_log2:
4273 case LibFunc_log2l:
4274 case LibFunc_logbf:
4275 case LibFunc_logb:
4276 case LibFunc_logbl:
4277 return optimizeLog(Log: CI, B&: Builder);
4278 case LibFunc_tan:
4279 case LibFunc_tanf:
4280 case LibFunc_tanl:
4281 case LibFunc_sinh:
4282 case LibFunc_sinhf:
4283 case LibFunc_sinhl:
4284 case LibFunc_asinh:
4285 case LibFunc_asinhf:
4286 case LibFunc_asinhl:
4287 case LibFunc_cosh:
4288 case LibFunc_coshf:
4289 case LibFunc_coshl:
4290 case LibFunc_atanh:
4291 case LibFunc_atanhf:
4292 case LibFunc_atanhl:
4293 return optimizeTrigInversionPairs(CI, B&: Builder);
4294 case LibFunc_ceil:
4295 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::ceil);
4296 case LibFunc_floor:
4297 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::floor);
4298 case LibFunc_round:
4299 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::round);
4300 case LibFunc_roundeven:
4301 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::roundeven);
4302 case LibFunc_nearbyint:
4303 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::nearbyint);
4304 case LibFunc_rint:
4305 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::rint);
4306 case LibFunc_trunc:
4307 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::trunc);
4308 case LibFunc_sin:
4309 case LibFunc_cos:
4310 if (UnsafeFPShrink &&
4311 hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4312 if (Value *V = optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true))
4313 return V;
4314 if (CI->doesNotAccessMemory())
4315 return replaceUnaryCall(
4316 CI, B&: Builder, IID: Func == LibFunc_sin ? Intrinsic::sin : Intrinsic::cos);
4317 return nullptr;
4318 case LibFunc_acos:
4319 case LibFunc_acosh:
4320 case LibFunc_asin:
4321 case LibFunc_atan:
4322 case LibFunc_cbrt:
4323 case LibFunc_exp:
4324 case LibFunc_exp10:
4325 case LibFunc_expm1:
4326 case LibFunc_tanh:
4327 if (UnsafeFPShrink && hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4328 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true);
4329 return nullptr;
4330 case LibFunc_copysign:
4331 if (hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4332 return optimizeBinaryDoubleFP(CI, B&: Builder, TLI);
4333 return nullptr;
4334 case LibFunc_fdim:
4335 case LibFunc_fdimf:
4336 case LibFunc_fdiml:
4337 return optimizeFdim(CI, B&: Builder);
4338 case LibFunc_fminf:
4339 case LibFunc_fmin:
4340 case LibFunc_fminl:
4341 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::minnum);
4342 case LibFunc_fmaxf:
4343 case LibFunc_fmax:
4344 case LibFunc_fmaxl:
4345 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::maxnum);
4346 case LibFunc_fminimum_numf:
4347 case LibFunc_fminimum_num:
4348 case LibFunc_fminimum_numl:
4349 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::minimumnum);
4350 case LibFunc_fmaximum_numf:
4351 case LibFunc_fmaximum_num:
4352 case LibFunc_fmaximum_numl:
4353 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::maximumnum);
4354 case LibFunc_cabs:
4355 case LibFunc_cabsf:
4356 case LibFunc_cabsl:
4357 return optimizeCAbs(CI, B&: Builder);
4358 case LibFunc_remquo:
4359 case LibFunc_remquof:
4360 case LibFunc_remquol:
4361 return optimizeRemquo(CI, B&: Builder);
4362 case LibFunc_nan:
4363 case LibFunc_nanf:
4364 case LibFunc_nanl:
4365 return optimizeNaN(CI);
4366 default:
4367 return nullptr;
4368 }
4369}
4370
4371Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) {
4372 Module *M = CI->getModule();
4373 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
4374
4375 // TODO: Split out the code below that operates on FP calls so that
4376 // we can all non-FP calls with the StrictFP attribute to be
4377 // optimized.
4378 if (CI->isNoBuiltin()) {
4379 // Optionally update operator new calls.
4380 return maybeOptimizeNoBuiltinOperatorNew(CI, B&: Builder);
4381 }
4382
4383 Function *Callee = CI->getCalledFunction();
4384 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4385 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4386
4387 SmallVector<OperandBundleDef, 2> OpBundles;
4388 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4389
4390 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4391 Builder.setDefaultOperandBundles(OpBundles);
4392
4393 // Command-line parameter overrides instruction attribute.
4394 // This can't be moved to optimizeFloatingPointLibCall() because it may be
4395 // used by the intrinsic optimizations.
4396 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
4397 UnsafeFPShrink = EnableUnsafeFPShrink;
4398 else if (isa<FPMathOperator>(Val: CI) && CI->isFast())
4399 UnsafeFPShrink = true;
4400
4401 // First, check for intrinsics.
4402 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI)) {
4403 if (!IsCallingConvC)
4404 return nullptr;
4405 // The FP intrinsics have corresponding constrained versions so we don't
4406 // need to check for the StrictFP attribute here.
4407 switch (II->getIntrinsicID()) {
4408 case Intrinsic::pow:
4409 return optimizePow(Pow: CI, B&: Builder);
4410 case Intrinsic::exp2:
4411 return optimizeExp2(CI, B&: Builder);
4412 case Intrinsic::log:
4413 case Intrinsic::log2:
4414 case Intrinsic::log10:
4415 return optimizeLog(Log: CI, B&: Builder);
4416 case Intrinsic::sqrt:
4417 return optimizeSqrt(CI, B&: Builder);
4418 case Intrinsic::memset:
4419 return optimizeMemSet(CI, B&: Builder);
4420 case Intrinsic::memcpy:
4421 return optimizeMemCpy(CI, B&: Builder);
4422 case Intrinsic::memmove:
4423 return optimizeMemMove(CI, B&: Builder);
4424 case Intrinsic::sin:
4425 case Intrinsic::cos:
4426 if (UnsafeFPShrink)
4427 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, /*isPrecise=*/true);
4428 return nullptr;
4429 case Intrinsic::sincos:
4430 if (UnsafeFPShrink)
4431 return optimizeSinCosDoubleFP(CI, B&: Builder);
4432 return nullptr;
4433 default:
4434 return nullptr;
4435 }
4436 }
4437
4438 // Also try to simplify calls to fortified library functions.
4439 if (Value *SimplifiedFortifiedCI =
4440 FortifiedSimplifier.optimizeCall(CI, B&: Builder))
4441 return SimplifiedFortifiedCI;
4442
4443 // Then check for known library functions.
4444 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4445 // We never change the calling convention.
4446 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4447 return nullptr;
4448 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
4449 return V;
4450 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
4451 return V;
4452 switch (Func) {
4453 case LibFunc_ffs:
4454 case LibFunc_ffsl:
4455 case LibFunc_ffsll:
4456 return optimizeFFS(CI, B&: Builder);
4457 case LibFunc_fls:
4458 case LibFunc_flsl:
4459 case LibFunc_flsll:
4460 return optimizeFls(CI, B&: Builder);
4461 case LibFunc_abs:
4462 case LibFunc_labs:
4463 case LibFunc_llabs:
4464 return optimizeAbs(CI, B&: Builder);
4465 case LibFunc_isdigit:
4466 return optimizeIsDigit(CI, B&: Builder);
4467 case LibFunc_isascii:
4468 return optimizeIsAscii(CI, B&: Builder);
4469 case LibFunc_toascii:
4470 return optimizeToAscii(CI, B&: Builder);
4471 case LibFunc_atoi:
4472 case LibFunc_atol:
4473 case LibFunc_atoll:
4474 return optimizeAtoi(CI, B&: Builder);
4475 case LibFunc_strtol:
4476 case LibFunc_strtoll:
4477 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/true);
4478 case LibFunc_strtoul:
4479 case LibFunc_strtoull:
4480 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/false);
4481 case LibFunc_printf:
4482 return optimizePrintF(CI, B&: Builder);
4483 case LibFunc_sprintf:
4484 return optimizeSPrintF(CI, B&: Builder);
4485 case LibFunc_snprintf:
4486 return optimizeSnPrintF(CI, B&: Builder);
4487 case LibFunc_fprintf:
4488 return optimizeFPrintF(CI, B&: Builder);
4489 case LibFunc_fwrite:
4490 return optimizeFWrite(CI, B&: Builder);
4491 case LibFunc_fputs:
4492 return optimizeFPuts(CI, B&: Builder);
4493 case LibFunc_puts:
4494 return optimizePuts(CI, B&: Builder);
4495 case LibFunc_perror:
4496 return optimizeErrorReporting(CI, B&: Builder);
4497 case LibFunc_vfprintf:
4498 case LibFunc_fiprintf:
4499 return optimizeErrorReporting(CI, B&: Builder, StreamArg: 0);
4500 case LibFunc_exit:
4501 case LibFunc_Exit:
4502 return optimizeExit(CI);
4503 default:
4504 return nullptr;
4505 }
4506 }
4507 return nullptr;
4508}
4509
4510LibCallSimplifier::LibCallSimplifier(
4511 const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT,
4512 DomConditionCache *DC, AssumptionCache *AC, OptimizationRemarkEmitter &ORE,
4513 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
4514 function_ref<void(Instruction *, Value *)> Replacer,
4515 function_ref<void(Instruction *)> Eraser)
4516 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), DT(DT), DC(DC), AC(AC),
4517 ORE(ORE), BFI(BFI), PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
4518
4519void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
4520 // Indirect through the replacer used in this instance.
4521 Replacer(I, With);
4522}
4523
4524void LibCallSimplifier::eraseFromParent(Instruction *I) {
4525 Eraser(I);
4526}
4527
4528// TODO:
4529// Additional cases that we need to add to this file:
4530//
4531// cbrt:
4532// * cbrt(expN(X)) -> expN(x/3)
4533// * cbrt(sqrt(x)) -> pow(x,1/6)
4534// * cbrt(cbrt(x)) -> pow(x,1/9)
4535//
4536// exp, expf, expl:
4537// * exp(log(x)) -> x
4538//
4539// log, logf, logl:
4540// * log(exp(x)) -> x
4541// * log(exp(y)) -> y*log(e)
4542// * log(exp10(y)) -> y*log(10)
4543// * log(sqrt(x)) -> 0.5*log(x)
4544//
4545// pow, powf, powl:
4546// * pow(sqrt(x),y) -> pow(x,y*0.5)
4547// * pow(pow(x,y),z)-> pow(x,y*z)
4548//
4549// signbit:
4550// * signbit(cnst) -> cnst'
4551// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
4552//
4553// sqrt, sqrtf, sqrtl:
4554// * sqrt(expN(x)) -> expN(x*0.5)
4555// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
4556// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
4557//
4558
4559//===----------------------------------------------------------------------===//
4560// Fortified Library Call Optimizations
4561//===----------------------------------------------------------------------===//
4562
4563bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
4564 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
4565 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
4566 // If this function takes a flag argument, the implementation may use it to
4567 // perform extra checks. Don't fold into the non-checking variant.
4568 if (FlagOp) {
4569 ConstantInt *Flag = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *FlagOp));
4570 if (!Flag || !Flag->isZero())
4571 return false;
4572 }
4573
4574 if (SizeOp && CI->getArgOperand(i: ObjSizeOp) == CI->getArgOperand(i: *SizeOp))
4575 return true;
4576
4577 if (ConstantInt *ObjSizeCI =
4578 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: ObjSizeOp))) {
4579 if (ObjSizeCI->isMinusOne())
4580 return true;
4581 // If the object size wasn't -1 (unknown), bail out if we were asked to.
4582 if (OnlyLowerUnknownSize)
4583 return false;
4584 if (StrOp) {
4585 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: *StrOp));
4586 // If the length is 0 we don't know how long it is and so we can't
4587 // remove the check.
4588 if (Len)
4589 annotateDereferenceableBytes(CI, ArgNos: *StrOp, DereferenceableBytes: Len);
4590 else
4591 return false;
4592 return ObjSizeCI->getZExtValue() >= Len;
4593 }
4594
4595 if (SizeOp) {
4596 if (ConstantInt *SizeCI =
4597 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *SizeOp)))
4598 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4599 }
4600 }
4601 return false;
4602}
4603
4604Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4605 IRBuilderBase &B) {
4606 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4607 CallInst *NewCI =
4608 B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4609 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4610 mergeAttributesAndFlags(NewCI, Old: *CI);
4611 return CI->getArgOperand(i: 0);
4612 }
4613 return nullptr;
4614}
4615
4616Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4617 IRBuilderBase &B) {
4618 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4619 CallInst *NewCI =
4620 B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4621 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4622 mergeAttributesAndFlags(NewCI, Old: *CI);
4623 return CI->getArgOperand(i: 0);
4624 }
4625 return nullptr;
4626}
4627
4628Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4629 IRBuilderBase &B) {
4630 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4631 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
4632 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val,
4633 Size: CI->getArgOperand(i: 2), Align: Align(1));
4634 mergeAttributesAndFlags(NewCI, Old: *CI);
4635 return CI->getArgOperand(i: 0);
4636 }
4637 return nullptr;
4638}
4639
4640Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4641 IRBuilderBase &B) {
4642 const DataLayout &DL = CI->getDataLayout();
4643 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2))
4644 if (Value *Call = emitMemPCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4645 Len: CI->getArgOperand(i: 2), B, DL, TLI)) {
4646 return mergeAttributesAndFlags(NewCI: cast<CallInst>(Val: Call), Old: *CI);
4647 }
4648 return nullptr;
4649}
4650
4651Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4652 IRBuilderBase &B,
4653 LibFunc Func) {
4654 const DataLayout &DL = CI->getDataLayout();
4655 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1),
4656 *ObjSize = CI->getArgOperand(i: 2);
4657
4658 // __stpcpy_chk(x,x,...) -> x+strlen(x)
4659 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4660 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
4661 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
4662 }
4663
4664 // If a) we don't have any length information, or b) we know this will
4665 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4666 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
4667 // TODO: It might be nice to get a maximum length out of the possible
4668 // string lengths for varying.
4669 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: 1)) {
4670 if (Func == LibFunc_strcpy_chk)
4671 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
4672 else
4673 return copyFlags(Old: *CI, New: emitStpCpy(Dst, Src, B, TLI));
4674 }
4675
4676 if (OnlyLowerUnknownSize)
4677 return nullptr;
4678
4679 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4680 uint64_t Len = GetStringLength(V: Src);
4681 if (Len)
4682 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
4683 else
4684 return nullptr;
4685
4686 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4687 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4688 Value *LenV = ConstantInt::get(Ty: SizeTTy, V: Len);
4689 Value *Ret = emitMemCpyChk(Dst, Src, Len: LenV, ObjSize, B, DL, TLI);
4690 // If the function was an __stpcpy_chk, and we were able to fold it into
4691 // a __memcpy_chk, we still need to return the correct end pointer.
4692 if (Ret && Func == LibFunc_stpcpy_chk)
4693 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst,
4694 IdxList: ConstantInt::get(Ty: SizeTTy, V: Len - 1));
4695 return copyFlags(Old: *CI, New: cast<CallInst>(Val: Ret));
4696}
4697
4698Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4699 IRBuilderBase &B) {
4700 if (isFortifiedCallFoldable(CI, ObjSizeOp: 1, SizeOp: std::nullopt, StrOp: 0))
4701 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B,
4702 DL: CI->getDataLayout(), TLI));
4703 return nullptr;
4704}
4705
4706Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4707 IRBuilderBase &B,
4708 LibFunc Func) {
4709 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4710 if (Func == LibFunc_strncpy_chk)
4711 return copyFlags(Old: *CI,
4712 New: emitStrNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4713 Len: CI->getArgOperand(i: 2), B, TLI));
4714 else
4715 return copyFlags(Old: *CI,
4716 New: emitStpNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4717 Len: CI->getArgOperand(i: 2), B, TLI));
4718 }
4719
4720 return nullptr;
4721}
4722
4723Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4724 IRBuilderBase &B) {
4725 if (isFortifiedCallFoldable(CI, ObjSizeOp: 4, SizeOp: 3))
4726 return copyFlags(
4727 Old: *CI, New: emitMemCCpy(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
4728 Val: CI->getArgOperand(i: 2), Len: CI->getArgOperand(i: 3), B, TLI));
4729
4730 return nullptr;
4731}
4732
4733Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4734 IRBuilderBase &B) {
4735 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2)) {
4736 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 5));
4737 return copyFlags(Old: *CI,
4738 New: emitSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4739 Fmt: CI->getArgOperand(i: 4), Args: VariadicArgs, B, TLI));
4740 }
4741
4742 return nullptr;
4743}
4744
4745Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4746 IRBuilderBase &B) {
4747 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1)) {
4748 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 4));
4749 return copyFlags(Old: *CI,
4750 New: emitSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4751 VariadicArgs, B, TLI));
4752 }
4753
4754 return nullptr;
4755}
4756
4757Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4758 IRBuilderBase &B) {
4759 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2))
4760 return copyFlags(
4761 Old: *CI, New: emitStrCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1), B, TLI));
4762
4763 return nullptr;
4764}
4765
4766Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4767 IRBuilderBase &B) {
4768 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4769 return copyFlags(Old: *CI,
4770 New: emitStrLCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4771 Size: CI->getArgOperand(i: 2), B, TLI));
4772
4773 return nullptr;
4774}
4775
4776Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4777 IRBuilderBase &B) {
4778 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4779 return copyFlags(Old: *CI,
4780 New: emitStrNCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4781 Size: CI->getArgOperand(i: 2), B, TLI));
4782
4783 return nullptr;
4784}
4785
4786Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4787 IRBuilderBase &B) {
4788 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4789 return copyFlags(Old: *CI,
4790 New: emitStrLCpy(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4791 Size: CI->getArgOperand(i: 2), B, TLI));
4792
4793 return nullptr;
4794}
4795
4796Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4797 IRBuilderBase &B) {
4798 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2))
4799 return copyFlags(
4800 Old: *CI, New: emitVSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4801 Fmt: CI->getArgOperand(i: 4), VAList: CI->getArgOperand(i: 5), B, TLI));
4802
4803 return nullptr;
4804}
4805
4806Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4807 IRBuilderBase &B) {
4808 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1))
4809 return copyFlags(Old: *CI,
4810 New: emitVSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4811 VAList: CI->getArgOperand(i: 4), B, TLI));
4812
4813 return nullptr;
4814}
4815
4816Value *FortifiedLibCallSimplifier::optimizeCall(CallInst *CI,
4817 IRBuilderBase &Builder) {
4818 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4819 // Some clang users checked for _chk libcall availability using:
4820 // __has_builtin(__builtin___memcpy_chk)
4821 // When compiling with -fno-builtin, this is always true.
4822 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4823 // end up with fortified libcalls, which isn't acceptable in a freestanding
4824 // environment which only provides their non-fortified counterparts.
4825 //
4826 // Until we change clang and/or teach external users to check for availability
4827 // differently, disregard the "nobuiltin" attribute and TLI::has.
4828 //
4829 // PR23093.
4830
4831 Function *Callee = CI->getCalledFunction();
4832 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4833
4834 SmallVector<OperandBundleDef, 2> OpBundles;
4835 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4836
4837 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4838 Builder.setDefaultOperandBundles(OpBundles);
4839
4840 // First, check that this is a known library functions and that the prototype
4841 // is correct.
4842 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4843 if (Func == NotLibFunc)
4844 return nullptr;
4845
4846 // We never change the calling convention.
4847 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4848 return nullptr;
4849
4850 switch (Func) {
4851 case LibFunc_memcpy_chk:
4852 return optimizeMemCpyChk(CI, B&: Builder);
4853 case LibFunc_mempcpy_chk:
4854 return optimizeMemPCpyChk(CI, B&: Builder);
4855 case LibFunc_memmove_chk:
4856 return optimizeMemMoveChk(CI, B&: Builder);
4857 case LibFunc_memset_chk:
4858 return optimizeMemSetChk(CI, B&: Builder);
4859 case LibFunc_stpcpy_chk:
4860 case LibFunc_strcpy_chk:
4861 return optimizeStrpCpyChk(CI, B&: Builder, Func);
4862 case LibFunc_strlen_chk:
4863 return optimizeStrLenChk(CI, B&: Builder);
4864 case LibFunc_stpncpy_chk:
4865 case LibFunc_strncpy_chk:
4866 return optimizeStrpNCpyChk(CI, B&: Builder, Func);
4867 case LibFunc_memccpy_chk:
4868 return optimizeMemCCpyChk(CI, B&: Builder);
4869 case LibFunc_snprintf_chk:
4870 return optimizeSNPrintfChk(CI, B&: Builder);
4871 case LibFunc_sprintf_chk:
4872 return optimizeSPrintfChk(CI, B&: Builder);
4873 case LibFunc_strcat_chk:
4874 return optimizeStrCatChk(CI, B&: Builder);
4875 case LibFunc_strlcat_chk:
4876 return optimizeStrLCat(CI, B&: Builder);
4877 case LibFunc_strncat_chk:
4878 return optimizeStrNCatChk(CI, B&: Builder);
4879 case LibFunc_strlcpy_chk:
4880 return optimizeStrLCpyChk(CI, B&: Builder);
4881 case LibFunc_vsnprintf_chk:
4882 return optimizeVSNPrintfChk(CI, B&: Builder);
4883 case LibFunc_vsprintf_chk:
4884 return optimizeVSPrintfChk(CI, B&: Builder);
4885 default:
4886 break;
4887 }
4888 return nullptr;
4889}
4890
4891FortifiedLibCallSimplifier::FortifiedLibCallSimplifier(
4892 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4893 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
4894