1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/SmallString.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/Analysis/ConstantFolding.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/Utils/Local.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/AttributeMask.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/PatternMatch.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/KnownFPClass.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/TargetParser/Triple.h"
39#include "llvm/Transforms/Utils/BuildLibCalls.h"
40#include "llvm/Transforms/Utils/Local.h"
41#include "llvm/Transforms/Utils/SizeOpts.h"
42
43#include <cmath>
44
45using namespace llvm;
46using namespace PatternMatch;
47
48static cl::opt<bool>
49 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
50 cl::init(Val: false),
51 cl::desc("Enable unsafe double to float "
52 "shrinking for math lib calls"));
53
54// Enable conversion of operator new calls with a MemProf hot or cold hint
55// to an operator new call that takes a hot/cold hint. Off by default since
56// not all allocators currently support this extension.
57static cl::opt<bool>
58 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(Val: false),
59 cl::desc("Enable hot/cold operator new library calls"));
60enum class OptimizeExistingHotColdNewKind {
61 None,
62 Cold,
63 Always,
64};
65static cl::opt<OptimizeExistingHotColdNewKind> OptimizeExistingHotColdNew(
66 "optimize-existing-hot-cold-new", cl::Hidden,
67 cl::desc(
68 "Enable optimization of existing hot/cold operator new library calls"),
69 cl::values(
70 clEnumValN(
71 OptimizeExistingHotColdNewKind::None, "none",
72 "Do not optimize existing hot/cold operator new library calls"),
73 clEnumValN(OptimizeExistingHotColdNewKind::Cold, "cold",
74 "Only optimize existing hot/cold operator new library calls "
75 "if determined to be cold"),
76 clEnumValN(
77 OptimizeExistingHotColdNewKind::Always, "always",
78 "Always optimize existing hot/cold operator new library calls"),
79 clEnumValN(
80 OptimizeExistingHotColdNewKind::Always, "",
81 "Always optimize existing hot/cold operator new library calls")),
82 cl::init(Val: OptimizeExistingHotColdNewKind::None), cl::ValueOptional);
83static cl::opt<bool> OptimizeNoBuiltinHotColdNew(
84 "optimize-nobuiltin-hot-cold-new-new", cl::Hidden, cl::init(Val: false),
85 cl::desc("Enable transformation of nobuiltin operator new library calls"));
86static cl::opt<bool> MinExistingHotColdNewHint(
87 "min-existing-hot-cold-new-hint", cl::Hidden, cl::init(Val: false),
88 cl::desc("Take the minimum of compiler hint and existing hint when "
89 "optimizing existing hot/cold operator new library calls"));
90
91namespace {
92
93// Specialized parser to ensure the hint is an 8 bit value (we can't specify
94// uint8_t to opt<> as that is interpreted to mean that we are passing a char
95// option with a specific set of values.
96struct HotColdHintParser : public cl::parser<unsigned> {
97 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
98
99 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
100 if (Arg.getAsInteger(Radix: 0, Result&: Value))
101 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
102
103 if (Value > 255)
104 return O.error(Message: "'" + Arg + "' value must be in the range [0, 255]!");
105
106 return false;
107 }
108};
109
110} // end anonymous namespace
111
112// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
113// and 255 is the hottest. Default to 1 value away from the coldest and hottest
114// hints, so that the compiler hinted allocations are slightly less strong than
115// manually inserted hints at the two extremes.
116static cl::opt<unsigned, false, HotColdHintParser> ColdNewHintValue(
117 "cold-new-hint-value", cl::Hidden, cl::init(Val: 1),
118 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
119static cl::opt<unsigned, false, HotColdHintParser>
120 NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(Val: 128),
121 cl::desc("Value to pass to hot/cold operator new for "
122 "notcold (warm) allocation"));
123static cl::opt<unsigned, false, HotColdHintParser> HotNewHintValue(
124 "hot-new-hint-value", cl::Hidden, cl::init(Val: 254),
125 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
126static cl::opt<unsigned, false, HotColdHintParser> AmbiguousNewHintValue(
127 "ambiguous-new-hint-value", cl::Hidden, cl::init(Val: 222),
128 cl::desc(
129 "Value to pass to hot/cold operator new for ambiguous allocation"));
130
131//===----------------------------------------------------------------------===//
132// Helper Functions
133//===----------------------------------------------------------------------===//
134
135static bool ignoreCallingConv(LibFunc Func) {
136 return Func == LibFunc_abs || Func == LibFunc_labs ||
137 Func == LibFunc_llabs || Func == LibFunc_strlen;
138}
139
140/// Return true if it is only used in equality comparisons with With.
141static bool isOnlyUsedInEqualityComparison(Value *V, Value *With) {
142 for (User *U : V->users()) {
143 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
144 if (IC->isEquality() && IC->getOperand(i_nocapture: 1) == With)
145 continue;
146 // Unknown instruction.
147 return false;
148 }
149 return true;
150}
151
152static bool callHasFloatingPointArgument(const CallInst *CI) {
153 return any_of(Range: CI->operands(), P: [](const Use &OI) {
154 return OI->getType()->isFloatingPointTy();
155 });
156}
157
158static bool callHasFP128Argument(const CallInst *CI) {
159 return any_of(Range: CI->operands(), P: [](const Use &OI) {
160 return OI->getType()->isFP128Ty();
161 });
162}
163
164// Convert the entire string Str representing an integer in Base, up to
165// the terminating nul if present, to a constant according to the rules
166// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
167// return the result, otherwise null.
168// The function assumes the string is encoded in ASCII and carefully
169// avoids converting sequences (including "") that the corresponding
170// library call might fail and set errno for.
171static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
172 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
173 if (Base < 2 || Base > 36)
174 if (Base != 0)
175 // Fail for an invalid base (required by POSIX).
176 return nullptr;
177
178 // Current offset into the original string to reflect in EndPtr.
179 size_t Offset = 0;
180 // Strip leading whitespace.
181 for ( ; Offset != Str.size(); ++Offset)
182 if (!isSpace(C: (unsigned char)Str[Offset])) {
183 Str = Str.substr(Start: Offset);
184 break;
185 }
186
187 if (Str.empty())
188 // Fail for empty subject sequences (POSIX allows but doesn't require
189 // strtol[l]/strtoul[l] to fail with EINVAL).
190 return nullptr;
191
192 // Strip but remember the sign.
193 bool Negate = Str[0] == '-';
194 if (Str[0] == '-' || Str[0] == '+') {
195 Str = Str.drop_front();
196 if (Str.empty())
197 // Fail for a sign with nothing after it.
198 return nullptr;
199 ++Offset;
200 }
201
202 // Set Max to the absolute value of the minimum (for signed), or
203 // to the maximum (for unsigned) value representable in the type.
204 Type *RetTy = CI->getType();
205 unsigned NBits = RetTy->getPrimitiveSizeInBits();
206 uint64_t Max = AsSigned && Negate ? 1 : 0;
207 Max += AsSigned ? maxIntN(N: NBits) : maxUIntN(N: NBits);
208
209 // Autodetect Base if it's zero and consume the "0x" prefix.
210 if (Str.size() > 1) {
211 if (Str[0] == '0') {
212 if (toUpper(x: (unsigned char)Str[1]) == 'X') {
213 if (Str.size() == 2 || (Base && Base != 16))
214 // Fail if Base doesn't allow the "0x" prefix or for the prefix
215 // alone that implementations like BSD set errno to EINVAL for.
216 return nullptr;
217
218 Str = Str.drop_front(N: 2);
219 Offset += 2;
220 Base = 16;
221 }
222 else if (Base == 0)
223 Base = 8;
224 } else if (Base == 0)
225 Base = 10;
226 }
227 else if (Base == 0)
228 Base = 10;
229
230 // Convert the rest of the subject sequence, not including the sign,
231 // to its uint64_t representation (this assumes the source character
232 // set is ASCII).
233 uint64_t Result = 0;
234 for (unsigned i = 0; i != Str.size(); ++i) {
235 unsigned char DigVal = Str[i];
236 if (isDigit(C: DigVal))
237 DigVal = DigVal - '0';
238 else {
239 DigVal = toUpper(x: DigVal);
240 if (isAlpha(C: DigVal))
241 DigVal = DigVal - 'A' + 10;
242 else
243 return nullptr;
244 }
245
246 if (DigVal >= Base)
247 // Fail if the digit is not valid in the Base.
248 return nullptr;
249
250 // Add the digit and fail if the result is not representable in
251 // the (unsigned form of the) destination type.
252 bool VFlow;
253 Result = SaturatingMultiplyAdd(X: Result, Y: Base, A: (uint64_t)DigVal, ResultOverflowed: &VFlow);
254 if (VFlow || Result > Max)
255 return nullptr;
256 }
257
258 if (EndPtr) {
259 // Store the pointer to the end.
260 Value *Off = B.getInt64(C: Offset + Str.size());
261 Value *StrBeg = CI->getArgOperand(i: 0);
262 Value *StrEnd = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: StrBeg, IdxList: Off, Name: "endptr");
263 B.CreateStore(Val: StrEnd, Ptr: EndPtr);
264 }
265
266 if (Negate) {
267 // Unsigned negation doesn't overflow.
268 Result = -Result;
269 // For unsigned numbers, discard sign bits.
270 if (!AsSigned)
271 Result &= maxUIntN(N: NBits);
272 }
273
274 return ConstantInt::get(Ty: RetTy, V: Result, IsSigned: AsSigned);
275}
276
277static bool isOnlyUsedInComparisonWithZero(Value *V) {
278 for (User *U : V->users()) {
279 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
280 if (Constant *C = dyn_cast<Constant>(Val: IC->getOperand(i_nocapture: 1)))
281 if (C->isNullValue())
282 continue;
283 // Unknown instruction.
284 return false;
285 }
286 return true;
287}
288
289static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
290 const SimplifyQuery &SQ) {
291 if (!isOnlyUsedInComparisonWithZero(V: CI))
292 return false;
293
294 if (!isDereferenceablePointer(V: Str, Size: APInt(64, Len), Q: SQ))
295 return false;
296
297 if (CI->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemory))
298 return false;
299
300 return true;
301}
302
303static void annotateDereferenceableBytes(CallInst *CI,
304 ArrayRef<unsigned> ArgNos,
305 uint64_t DereferenceableBytes) {
306 const Function *F = CI->getCaller();
307 if (!F)
308 return;
309 for (unsigned ArgNo : ArgNos) {
310 uint64_t DerefBytes = DereferenceableBytes;
311 unsigned AS = CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
312 if (!llvm::NullPointerIsDefined(F, AS) ||
313 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
314 DerefBytes = std::max(a: CI->getParamDereferenceableOrNullBytes(i: ArgNo),
315 b: DereferenceableBytes);
316
317 if (CI->getParamDereferenceableBytes(i: ArgNo) < DerefBytes) {
318 CI->removeParamAttr(ArgNo, Kind: Attribute::Dereferenceable);
319 if (!llvm::NullPointerIsDefined(F, AS) ||
320 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
321 CI->removeParamAttr(ArgNo, Kind: Attribute::DereferenceableOrNull);
322 CI->addParamAttr(ArgNo, Attr: Attribute::getWithDereferenceableBytes(
323 Context&: CI->getContext(), Bytes: DerefBytes));
324 }
325 }
326}
327
328static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI,
329 ArrayRef<unsigned> ArgNos) {
330 Function *F = CI->getCaller();
331 if (!F)
332 return;
333
334 for (unsigned ArgNo : ArgNos) {
335 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NoUndef))
336 CI->addParamAttr(ArgNo, Kind: Attribute::NoUndef);
337
338 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull)) {
339 unsigned AS =
340 CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
341 if (llvm::NullPointerIsDefined(F, AS))
342 continue;
343 CI->addParamAttr(ArgNo, Kind: Attribute::NonNull);
344 }
345
346 annotateDereferenceableBytes(CI, ArgNos: ArgNo, DereferenceableBytes: 1);
347 }
348}
349
350static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> ArgNos,
351 Value *Size, const DataLayout &DL) {
352 if (ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size)) {
353 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
354 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: LenC->getZExtValue());
355 } else if (isKnownNonZero(V: Size, Q: DL)) {
356 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
357 uint64_t X, Y;
358 uint64_t DerefMin = 1;
359 if (match(V: Size, P: m_Select(C: m_Value(), L: m_ConstantInt(V&: X), R: m_ConstantInt(V&: Y)))) {
360 DerefMin = std::min(a: X, b: Y);
361 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: DerefMin);
362 }
363 }
364}
365
366// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
367// easier chaining. Calls to emit* and B.createCall should probably be wrapped
368// in this function when New is created to replace Old. Callers should take
369// care to check Old.isMustTailCall() if they aren't replacing Old directly
370// with New.
371static Value *copyFlags(const CallInst &Old, Value *New) {
372 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
373 assert(!Old.isNoTailCall() && "do not copy notail call flags");
374 if (auto *NewCI = dyn_cast_or_null<CallInst>(Val: New))
375 NewCI->setTailCallKind(Old.getTailCallKind());
376 return New;
377}
378
379static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
380 NewCI->setAttributes(AttributeList::get(
381 C&: NewCI->getContext(), Attrs: {NewCI->getAttributes(), Old.getAttributes()}));
382 NewCI->removeRetAttrs(AttrsToRemove: AttributeFuncs::typeIncompatible(
383 Ty: NewCI->getType(), AS: NewCI->getRetAttributes()));
384 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
385 NewCI->removeParamAttrs(
386 ArgNo: I, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewCI->getArgOperand(i: I)->getType(),
387 AS: NewCI->getParamAttributes(ArgNo: I)));
388
389 return copyFlags(Old, New: NewCI);
390}
391
392// Helper to avoid truncating the length if size_t is 32-bits.
393static StringRef substr(StringRef Str, uint64_t Len) {
394 return Len >= Str.size() ? Str : Str.substr(Start: 0, N: Len);
395}
396
397//===----------------------------------------------------------------------===//
398// String and Memory Library Call Optimizations
399//===----------------------------------------------------------------------===//
400
401Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
402 // Extract some information from the instruction
403 Value *Dst = CI->getArgOperand(i: 0);
404 Value *Src = CI->getArgOperand(i: 1);
405 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
406
407 // See if we can get the length of the input string.
408 uint64_t Len = GetStringLength(V: Src);
409 if (Len)
410 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
411 else
412 return nullptr;
413 --Len; // Unbias length.
414
415 // Handle the simple, do-nothing case: strcat(x, "") -> x
416 if (Len == 0)
417 return Dst;
418
419 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len, B));
420}
421
422Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
423 IRBuilderBase &B) {
424 // We need to find the end of the destination string. That's where the
425 // memory is to be moved to. We just generate a call to strlen.
426 Value *DstLen = emitStrLen(Ptr: Dst, B, DL, TLI);
427 if (!DstLen)
428 return nullptr;
429
430 // Now that we have the destination's length, we must index into the
431 // destination's pointer to get the actual memcpy destination (end of
432 // the string .. we're concatenating).
433 Value *CpyDst = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: DstLen, Name: "endptr");
434
435 // We have enough information to now generate the memcpy call to do the
436 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
437 B.CreateMemCpy(Dst: CpyDst, DstAlign: Align(1), Src, SrcAlign: Align(1),
438 Size: TLI->getAsSizeT(V: Len + 1, M: *B.GetInsertBlock()->getModule()));
439 return Dst;
440}
441
442Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
443 // Extract some information from the instruction.
444 Value *Dst = CI->getArgOperand(i: 0);
445 Value *Src = CI->getArgOperand(i: 1);
446 Value *Size = CI->getArgOperand(i: 2);
447 uint64_t Len;
448 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
449 if (isKnownNonZero(V: Size, Q: DL))
450 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
451
452 // We don't do anything if length is not constant.
453 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size);
454 if (LengthArg) {
455 Len = LengthArg->getZExtValue();
456 // strncat(x, c, 0) -> x
457 if (!Len)
458 return Dst;
459 } else {
460 return nullptr;
461 }
462
463 // See if we can get the length of the input string.
464 uint64_t SrcLen = GetStringLength(V: Src);
465 if (SrcLen) {
466 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
467 --SrcLen; // Unbias length.
468 } else {
469 return nullptr;
470 }
471
472 // strncat(x, "", c) -> x
473 if (SrcLen == 0)
474 return Dst;
475
476 // We don't optimize this case.
477 if (Len < SrcLen)
478 return nullptr;
479
480 // strncat(x, s, c) -> strcat(x, s)
481 // s is constant so the strcat can be optimized further.
482 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len: SrcLen, B));
483}
484
485// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
486// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
487// is that either S is dereferenceable or the value of N is nonzero.
488static Value* memChrToCharCompare(CallInst *CI, Value *NBytes,
489 IRBuilderBase &B, const DataLayout &DL)
490{
491 Value *Src = CI->getArgOperand(i: 0);
492 Value *CharVal = CI->getArgOperand(i: 1);
493
494 // Fold memchr(A, C, N) == A to N && *A == C.
495 Type *CharTy = B.getInt8Ty();
496 Value *Char0 = B.CreateLoad(Ty: CharTy, Ptr: Src);
497 CharVal = B.CreateTrunc(V: CharVal, DestTy: CharTy);
498 Value *Cmp = B.CreateICmpEQ(LHS: Char0, RHS: CharVal, Name: "char0cmp");
499
500 if (NBytes) {
501 Value *Zero = ConstantInt::get(Ty: NBytes->getType(), V: 0);
502 Value *And = B.CreateICmpNE(LHS: NBytes, RHS: Zero);
503 Cmp = B.CreateLogicalAnd(Cond1: And, Cond2: Cmp);
504 }
505
506 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
507 return B.CreateSelect(C: Cmp, True: Src, False: NullPtr);
508}
509
510Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
511 Value *SrcStr = CI->getArgOperand(i: 0);
512 Value *CharVal = CI->getArgOperand(i: 1);
513 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
514
515 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
516 return memChrToCharCompare(CI, NBytes: nullptr, B, DL);
517
518 // If the second operand is non-constant, see if we can compute the length
519 // of the input string and turn this into memchr.
520 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
521 if (!CharC) {
522 uint64_t Len = GetStringLength(V: SrcStr);
523 if (Len)
524 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len);
525 else
526 return nullptr;
527
528 Function *Callee = CI->getCalledFunction();
529 FunctionType *FT = Callee->getFunctionType();
530 unsigned IntBits = TLI->getIntSize();
531 if (!FT->getParamType(i: 1)->isIntegerTy(BitWidth: IntBits)) // memchr needs 'int'.
532 return nullptr;
533
534 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
535 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
536 return copyFlags(Old: *CI,
537 New: emitMemChr(Ptr: SrcStr, Val: CharVal, // include nul.
538 Len: ConstantInt::get(Ty: SizeTTy, V: Len), B,
539 DL, TLI));
540 }
541
542 if (CharC->isZero()) {
543 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
544 if (isOnlyUsedInEqualityComparison(V: CI, With: NullPtr))
545 // Pre-empt the transformation to strlen below and fold
546 // strchr(A, '\0') == null to false.
547 return B.CreateIntToPtr(V: B.getTrue(), DestTy: CI->getType());
548 }
549
550 // Otherwise, the character is a constant, see if the first argument is
551 // a string literal. If so, we can constant fold.
552 StringRef Str;
553 if (!getConstantStringInfo(V: SrcStr, Str)) {
554 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
555 if (Value *StrLen = emitStrLen(Ptr: SrcStr, B, DL, TLI))
556 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: StrLen, Name: "strchr");
557 return nullptr;
558 }
559
560 // Compute the offset, make sure to handle the case when we're searching for
561 // zero (a weird way to spell strlen).
562 size_t I = (0xFF & CharC->getSExtValue()) == 0
563 ? Str.size()
564 : Str.find(C: CharC->getSExtValue());
565 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
566 return Constant::getNullValue(Ty: CI->getType());
567
568 // strchr(s+n,c) -> gep(s+n+i,c)
569 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: I), Name: "strchr");
570}
571
572Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
573 Value *SrcStr = CI->getArgOperand(i: 0);
574 Value *CharVal = CI->getArgOperand(i: 1);
575 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
576 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
577
578 StringRef Str;
579 if (!getConstantStringInfo(V: SrcStr, Str)) {
580 // strrchr(s, 0) -> strchr(s, 0)
581 if (CharC && CharC->isZero())
582 return copyFlags(Old: *CI, New: emitStrChr(Ptr: SrcStr, C: '\0', B, TLI));
583 return nullptr;
584 }
585
586 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
587 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
588
589 // Try to expand strrchr to the memrchr nonstandard extension if it's
590 // available, or simply fail otherwise.
591 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
592 Value *Size = ConstantInt::get(Ty: SizeTTy, V: NBytes);
593 return copyFlags(Old: *CI, New: emitMemRChr(Ptr: SrcStr, Val: CharVal, Len: Size, B, DL, TLI));
594}
595
596Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
597 Value *Str1P = CI->getArgOperand(i: 0), *Str2P = CI->getArgOperand(i: 1);
598 if (Str1P == Str2P) // strcmp(x,x) -> 0
599 return ConstantInt::get(Ty: CI->getType(), V: 0);
600
601 StringRef Str1, Str2;
602 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
603 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
604
605 // strcmp(x, y) -> cnst (if both x and y are constant strings)
606 if (HasStr1 && HasStr2)
607 return ConstantInt::getSigned(Ty: CI->getType(),
608 V: std::clamp(val: Str1.compare(RHS: Str2), lo: -1, hi: 1));
609
610 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
611 return B.CreateNeg(V: B.CreateZExt(
612 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
613
614 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
615 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
616 DestTy: CI->getType());
617
618 // strcmp(P, "x") -> memcmp(P, "x", 2)
619 uint64_t Len1 = GetStringLength(V: Str1P);
620 if (Len1)
621 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
622 uint64_t Len2 = GetStringLength(V: Str2P);
623 if (Len2)
624 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
625
626 if (Len1 && Len2) {
627 return copyFlags(
628 Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
629 Len: TLI->getAsSizeT(V: std::min(a: Len1, b: Len2), M: *CI->getModule()),
630 B, DL, TLI));
631 }
632
633 // strcmp to memcmp
634 SimplifyQuery SQ(DL, TLI, DT, AC, CI);
635 if (!HasStr1 && HasStr2) {
636 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ))
637 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
638 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
639 B, DL, TLI));
640 } else if (HasStr1 && !HasStr2) {
641 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ))
642 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
643 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
644 B, DL, TLI));
645 }
646
647 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
648 return nullptr;
649}
650
651// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
652// arrays LHS and RHS and nonconstant Size.
653static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
654 Value *Size, bool StrNCmp,
655 IRBuilderBase &B, const DataLayout &DL);
656
657Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
658 Value *Str1P = CI->getArgOperand(i: 0);
659 Value *Str2P = CI->getArgOperand(i: 1);
660 Value *Size = CI->getArgOperand(i: 2);
661 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
662 return ConstantInt::get(Ty: CI->getType(), V: 0);
663
664 if (isKnownNonZero(V: Size, Q: DL))
665 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
666 // Get the length argument if it is constant.
667 uint64_t Length;
668 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size))
669 Length = LengthArg->getZExtValue();
670 else
671 return optimizeMemCmpVarSize(CI, LHS: Str1P, RHS: Str2P, Size, StrNCmp: true, B, DL);
672
673 if (Length == 0) // strncmp(x,y,0) -> 0
674 return ConstantInt::get(Ty: CI->getType(), V: 0);
675
676 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
677 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P, Len: Size, B, DL, TLI));
678
679 StringRef Str1, Str2;
680 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
681 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
682
683 // strncmp(x, y) -> cnst (if both x and y are constant strings)
684 if (HasStr1 && HasStr2) {
685 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
686 StringRef SubStr1 = substr(Str: Str1, Len: Length);
687 StringRef SubStr2 = substr(Str: Str2, Len: Length);
688 return ConstantInt::getSigned(Ty: CI->getType(),
689 V: std::clamp(val: SubStr1.compare(RHS: SubStr2), lo: -1, hi: 1));
690 }
691
692 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
693 return B.CreateNeg(V: B.CreateZExt(
694 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
695
696 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
697 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
698 DestTy: CI->getType());
699
700 uint64_t Len1 = GetStringLength(V: Str1P);
701 if (Len1)
702 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
703 uint64_t Len2 = GetStringLength(V: Str2P);
704 if (Len2)
705 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
706
707 // strncmp to memcmp
708 if (!HasStr1 && HasStr2) {
709 Len2 = std::min(a: Len2, b: Length);
710 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ: DL))
711 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
712 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
713 B, DL, TLI));
714 } else if (HasStr1 && !HasStr2) {
715 Len1 = std::min(a: Len1, b: Length);
716 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ: DL))
717 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
718 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
719 B, DL, TLI));
720 }
721
722 return nullptr;
723}
724
725Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
726 Value *Src = CI->getArgOperand(i: 0);
727 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
728 uint64_t SrcLen = GetStringLength(V: Src);
729 if (SrcLen && Size) {
730 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: SrcLen);
731 if (SrcLen <= Size->getZExtValue() + 1)
732 return copyFlags(Old: *CI, New: emitStrDup(Ptr: Src, B, TLI));
733 }
734
735 return nullptr;
736}
737
738Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
739 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
740 if (Dst == Src) // strcpy(x,x) -> x
741 return Src;
742
743 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
744 // See if we can get the length of the input string.
745 uint64_t Len = GetStringLength(V: Src);
746 if (Len)
747 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
748 else
749 return nullptr;
750
751 // We have enough information to now generate the memcpy call to do the
752 // copy for us. Make a memcpy to copy the nul byte with align = 1.
753 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
754 Size: TLI->getAsSizeT(V: Len, M: *CI->getModule()));
755 mergeAttributesAndFlags(NewCI, Old: *CI);
756 return Dst;
757}
758
759Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
760 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
761
762 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
763 if (CI->use_empty())
764 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
765
766 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
767 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
768 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
769 }
770
771 // See if we can get the length of the input string.
772 uint64_t Len = GetStringLength(V: Src);
773 if (Len)
774 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
775 else
776 return nullptr;
777
778 Value *LenV = TLI->getAsSizeT(V: Len, M: *CI->getModule());
779 Value *DstEnd = B.CreateInBoundsGEP(
780 Ty: B.getInt8Ty(), Ptr: Dst, IdxList: TLI->getAsSizeT(V: Len - 1, M: *CI->getModule()));
781
782 // We have enough information to now generate the memcpy call to do the
783 // copy for us. Make a memcpy to copy the nul byte with align = 1.
784 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: LenV);
785 mergeAttributesAndFlags(NewCI, Old: *CI);
786 return DstEnd;
787}
788
789// Optimize a call to size_t strlcpy(char*, const char*, size_t).
790
791Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
792 Value *Size = CI->getArgOperand(i: 2);
793 if (isKnownNonZero(V: Size, Q: DL))
794 // Like snprintf, the function stores into the destination only when
795 // the size argument is nonzero.
796 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
797 // The function reads the source argument regardless of Size (it returns
798 // its length).
799 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
800
801 uint64_t NBytes;
802 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
803 NBytes = SizeC->getZExtValue();
804 else
805 return nullptr;
806
807 Value *Dst = CI->getArgOperand(i: 0);
808 Value *Src = CI->getArgOperand(i: 1);
809 if (NBytes <= 1) {
810 if (NBytes == 1)
811 // For a call to strlcpy(D, S, 1) first store a nul in *D.
812 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
813
814 // Transform strlcpy(D, S, 0) to a call to strlen(S).
815 return copyFlags(Old: *CI, New: emitStrLen(Ptr: Src, B, DL, TLI));
816 }
817
818 // Try to determine the length of the source, substituting its size
819 // when it's not nul-terminated (as it's required to be) to avoid
820 // reading past its end.
821 StringRef Str;
822 if (!getConstantStringInfo(V: Src, Str, /*TrimAtNul=*/false))
823 return nullptr;
824
825 uint64_t SrcLen = Str.find(C: '\0');
826 // Set if the terminating nul should be copied by the call to memcpy
827 // below.
828 bool NulTerm = SrcLen < NBytes;
829
830 if (NulTerm)
831 // Overwrite NBytes with the number of bytes to copy, including
832 // the terminating nul.
833 NBytes = SrcLen + 1;
834 else {
835 // Set the length of the source for the function to return to its
836 // size, and cap NBytes at the same.
837 SrcLen = std::min(a: SrcLen, b: uint64_t(Str.size()));
838 NBytes = std::min(a: NBytes - 1, b: SrcLen);
839 }
840
841 if (SrcLen == 0) {
842 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
843 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
844 return ConstantInt::get(Ty: CI->getType(), V: 0);
845 }
846
847 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
848 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
849 // D[N' - 1] if necessary.
850 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
851 Size: TLI->getAsSizeT(V: NBytes, M: *CI->getModule()));
852 mergeAttributesAndFlags(NewCI, Old: *CI);
853
854 if (!NulTerm) {
855 Value *EndOff = ConstantInt::get(Ty: CI->getType(), V: NBytes);
856 Value *EndPtr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: EndOff);
857 B.CreateStore(Val: B.getInt8(C: 0), Ptr: EndPtr);
858 }
859
860 // Like snprintf, strlcpy returns the number of nonzero bytes that would
861 // have been copied if the bound had been sufficiently big (which in this
862 // case is strlen(Src)).
863 return ConstantInt::get(Ty: CI->getType(), V: SrcLen);
864}
865
866// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
867// otherwise.
868Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
869 IRBuilderBase &B) {
870 Value *Dst = CI->getArgOperand(i: 0);
871 Value *Src = CI->getArgOperand(i: 1);
872 Value *Size = CI->getArgOperand(i: 2);
873
874 if (isKnownNonZero(V: Size, Q: DL)) {
875 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
876 // only when N is nonzero.
877 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
878 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
879 }
880
881 // If the "bound" argument is known set N to it. Otherwise set it to
882 // UINT64_MAX and handle it later.
883 uint64_t N = UINT64_MAX;
884 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
885 N = SizeC->getZExtValue();
886
887 if (N == 0)
888 // Fold st{p,r}ncpy(D, S, 0) to D.
889 return Dst;
890
891 if (N == 1) {
892 Type *CharTy = B.getInt8Ty();
893 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "stxncpy.char0");
894 B.CreateStore(Val: CharVal, Ptr: Dst);
895 if (!RetEnd)
896 // Transform strncpy(D, S, 1) to return (*D = *S), D.
897 return Dst;
898
899 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
900 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
901 Value *Cmp = B.CreateICmpEQ(LHS: CharVal, RHS: ZeroChar, Name: "stpncpy.char0cmp");
902
903 Value *Off1 = B.getInt32(C: 1);
904 Value *EndPtr = B.CreateInBoundsGEP(Ty: CharTy, Ptr: Dst, IdxList: Off1, Name: "stpncpy.end");
905 return B.CreateSelect(C: Cmp, True: Dst, False: EndPtr, Name: "stpncpy.sel");
906 }
907
908 // If the length of the input string is known set SrcLen to it.
909 uint64_t SrcLen = GetStringLength(V: Src);
910 if (SrcLen)
911 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
912 else
913 return nullptr;
914
915 --SrcLen; // Unbias length.
916
917 if (SrcLen == 0) {
918 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
919 Align MemSetAlign =
920 CI->getAttributes().getParamAttrs(ArgNo: 0).getAlignment().valueOrOne();
921 CallInst *NewCI = B.CreateMemSet(Ptr: Dst, Val: B.getInt8(C: '\0'), Size, Align: MemSetAlign);
922 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(ArgNo: 0));
923 NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
924 C&: CI->getContext(), ArgNo: 0, B: ArgAttrs));
925 copyFlags(Old: *CI, New: NewCI);
926 return Dst;
927 }
928
929 if (N > SrcLen + 1) {
930 if (N > 128)
931 // Bail if N is large or unknown.
932 return nullptr;
933
934 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
935 StringRef Str;
936 if (!getConstantStringInfo(V: Src, Str))
937 return nullptr;
938 std::string SrcStr = Str.str();
939 // Create a bigger, nul-padded array with the same length, SrcLen,
940 // as the original string.
941 SrcStr.resize(n: N, c: '\0');
942 Src = B.CreateGlobalString(Str: SrcStr, Name: "str", /*AddressSpace=*/0,
943 /*M=*/nullptr, /*AddNull=*/false);
944 }
945
946 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
947 // S and N are constant.
948 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
949 Size: TLI->getAsSizeT(V: N, M: *CI->getModule()));
950 mergeAttributesAndFlags(NewCI, Old: *CI);
951 if (!RetEnd)
952 return Dst;
953
954 // stpncpy(D, S, N) returns the address of the first null in D if it writes
955 // one, otherwise D + N.
956 Value *Off = B.getInt64(C: std::min(a: SrcLen, b: N));
957 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: Off, Name: "endptr");
958}
959
960Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
961 unsigned CharSize,
962 Value *Bound) {
963 Value *Src = CI->getArgOperand(i: 0);
964 Type *CharTy = B.getIntNTy(N: CharSize);
965
966 if (isOnlyUsedInZeroEqualityComparison(CxtI: CI) &&
967 (!Bound || isKnownNonZero(V: Bound, Q: DL))) {
968 // Fold strlen:
969 // strlen(x) != 0 --> *x != 0
970 // strlen(x) == 0 --> *x == 0
971 // and likewise strnlen with constant N > 0:
972 // strnlen(x, N) != 0 --> *x != 0
973 // strnlen(x, N) == 0 --> *x == 0
974 return B.CreateZExt(V: B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "char0"),
975 DestTy: CI->getType());
976 }
977
978 if (Bound) {
979 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Val: Bound)) {
980 if (BoundCst->isZero())
981 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
982 return ConstantInt::get(Ty: CI->getType(), V: 0);
983
984 if (BoundCst->isOne()) {
985 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
986 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "strnlen.char0");
987 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
988 Value *Cmp = B.CreateICmpNE(LHS: CharVal, RHS: ZeroChar, Name: "strnlen.char0cmp");
989 return B.CreateZExt(V: Cmp, DestTy: CI->getType());
990 }
991 }
992 }
993
994 if (uint64_t Len = GetStringLength(V: Src, CharSize)) {
995 Value *LenC = ConstantInt::get(Ty: CI->getType(), V: Len - 1);
996 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
997 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
998 if (Bound)
999 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: LenC, RHS: Bound);
1000 return LenC;
1001 }
1002
1003 if (Bound)
1004 // Punt for strnlen for now.
1005 return nullptr;
1006
1007 // If s is a constant pointer pointing to a string literal, we can fold
1008 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
1009 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
1010 // We only try to simplify strlen when the pointer s points to an array
1011 // of CharSize elements. Otherwise, we would need to scale the offset x before
1012 // doing the subtraction. This will make the optimization more complex, and
1013 // it's not very useful because calling strlen for a pointer of other types is
1014 // very uncommon.
1015 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: Src)) {
1016 unsigned BW = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1017 SmallMapVector<Value *, APInt, 4> VarOffsets;
1018 APInt ConstOffset(BW, 0);
1019 assert(CharSize % 8 == 0 && "Expected a multiple of 8 sized CharSize");
1020 // Check the gep is a single variable offset.
1021 if (!GEP->collectOffset(DL, BitWidth: BW, VariableOffsets&: VarOffsets, ConstantOffset&: ConstOffset) ||
1022 VarOffsets.size() != 1 || ConstOffset != 0 ||
1023 VarOffsets.begin()->second != CharSize / 8)
1024 return nullptr;
1025
1026 ConstantDataArraySlice Slice;
1027 if (getConstantDataArrayInfo(V: GEP->getOperand(i_nocapture: 0), Slice, ElementSize: CharSize)) {
1028 uint64_t NullTermIdx;
1029 if (Slice.Array == nullptr) {
1030 NullTermIdx = 0;
1031 } else {
1032 NullTermIdx = ~((uint64_t)0);
1033 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
1034 if (Slice.Array->getElementAsInteger(i: I + Slice.Offset) == 0) {
1035 NullTermIdx = I;
1036 break;
1037 }
1038 }
1039 // If the string does not have '\0', leave it to strlen to compute
1040 // its length.
1041 if (NullTermIdx == ~((uint64_t)0))
1042 return nullptr;
1043 }
1044
1045 Value *Offset = VarOffsets.begin()->first;
1046 KnownBits Known = computeKnownBits(V: Offset, DL, AC: nullptr, CxtI: CI, DT: nullptr);
1047
1048 // If Offset is not provably in the range [0, NullTermIdx], we can still
1049 // optimize if we can prove that the program has undefined behavior when
1050 // Offset is outside that range. That is the case when GEP->getOperand(0)
1051 // is a pointer to an object whose memory extent is NullTermIdx+1.
1052 if ((Known.isNonNegative() && Known.getMaxValue().ule(RHS: NullTermIdx)) ||
1053 (isa<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)) &&
1054 NullTermIdx == Slice.Length - 1)) {
1055 Offset = B.CreateSExtOrTrunc(V: Offset, DestTy: CI->getType());
1056 return B.CreateSub(LHS: ConstantInt::get(Ty: CI->getType(), V: NullTermIdx),
1057 RHS: Offset);
1058 }
1059 }
1060 }
1061
1062 // strlen(x?"foo":"bars") --> x ? 3 : 4
1063 if (SelectInst *SI = dyn_cast<SelectInst>(Val: Src)) {
1064 uint64_t LenTrue = GetStringLength(V: SI->getTrueValue(), CharSize);
1065 uint64_t LenFalse = GetStringLength(V: SI->getFalseValue(), CharSize);
1066 if (LenTrue && LenFalse) {
1067 ORE.emit(RemarkBuilder: [&]() {
1068 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1069 << "folded strlen(select) to select of constants";
1070 });
1071 return B.CreateSelect(C: SI->getCondition(),
1072 True: ConstantInt::get(Ty: CI->getType(), V: LenTrue - 1),
1073 False: ConstantInt::get(Ty: CI->getType(), V: LenFalse - 1));
1074 }
1075 }
1076
1077 return nullptr;
1078}
1079
1080Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1081 if (Value *V = optimizeStringLength(CI, B, CharSize: 8))
1082 return V;
1083 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1084 return nullptr;
1085}
1086
1087Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1088 Value *Bound = CI->getArgOperand(i: 1);
1089 if (Value *V = optimizeStringLength(CI, B, CharSize: 8, Bound))
1090 return V;
1091
1092 if (isKnownNonZero(V: Bound, Q: DL))
1093 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1094 return nullptr;
1095}
1096
1097Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1098 Module &M = *CI->getModule();
1099 unsigned WCharSize = TLI->getWCharSize(M) * 8;
1100 // We cannot perform this optimization without wchar_size metadata.
1101 if (WCharSize == 0)
1102 return nullptr;
1103
1104 return optimizeStringLength(CI, B, CharSize: WCharSize);
1105}
1106
1107Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1108 StringRef S1, S2;
1109 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1110 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1111
1112 // strpbrk(s, "") -> nullptr
1113 // strpbrk("", s) -> nullptr
1114 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1115 return Constant::getNullValue(Ty: CI->getType());
1116
1117 // Constant folding.
1118 if (HasS1 && HasS2) {
1119 size_t I = S1.find_first_of(Chars: S2);
1120 if (I == StringRef::npos) // No match.
1121 return Constant::getNullValue(Ty: CI->getType());
1122
1123 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1124 IdxList: B.getInt64(C: I), Name: "strpbrk");
1125 }
1126
1127 // strpbrk(s, "a") -> strchr(s, 'a')
1128 if (HasS2 && S2.size() == 1)
1129 return copyFlags(Old: *CI, New: emitStrChr(Ptr: CI->getArgOperand(i: 0), C: S2[0], B, TLI));
1130
1131 return nullptr;
1132}
1133
1134Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1135 Value *EndPtr = CI->getArgOperand(i: 1);
1136 if (isa<ConstantPointerNull>(Val: EndPtr)) {
1137 // With a null EndPtr, this function won't capture the main argument.
1138 // It would be readonly too, except that it still may write to errno.
1139 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
1140 CI: CaptureInfo::none()));
1141 }
1142
1143 return nullptr;
1144}
1145
1146Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1147 StringRef S1, S2;
1148 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1149 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1150
1151 // strspn(s, "") -> 0
1152 // strspn("", s) -> 0
1153 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1154 return Constant::getNullValue(Ty: CI->getType());
1155
1156 // Constant folding.
1157 if (HasS1 && HasS2) {
1158 size_t Pos = S1.find_first_not_of(Chars: S2);
1159 if (Pos == StringRef::npos)
1160 Pos = S1.size();
1161 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1162 }
1163
1164 return nullptr;
1165}
1166
1167Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1168 StringRef S1, S2;
1169 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1170 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1171
1172 // strcspn("", s) -> 0
1173 if (HasS1 && S1.empty())
1174 return Constant::getNullValue(Ty: CI->getType());
1175
1176 // Constant folding.
1177 if (HasS1 && HasS2) {
1178 size_t Pos = S1.find_first_of(Chars: S2);
1179 if (Pos == StringRef::npos)
1180 Pos = S1.size();
1181 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1182 }
1183
1184 // strcspn(s, "") -> strlen(s)
1185 if (HasS2 && S2.empty())
1186 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B, DL, TLI));
1187
1188 return nullptr;
1189}
1190
1191Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1192 // fold strstr(x, x) -> x.
1193 if (CI->getArgOperand(i: 0) == CI->getArgOperand(i: 1))
1194 return CI->getArgOperand(i: 0);
1195
1196 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1197 if (isOnlyUsedInEqualityComparison(V: CI, With: CI->getArgOperand(i: 0))) {
1198 Value *StrLen = emitStrLen(Ptr: CI->getArgOperand(i: 1), B, DL, TLI);
1199 if (!StrLen)
1200 return nullptr;
1201 Value *StrNCmp = emitStrNCmp(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
1202 Len: StrLen, B, DL, TLI);
1203 if (!StrNCmp)
1204 return nullptr;
1205 for (User *U : llvm::make_early_inc_range(Range: CI->users())) {
1206 ICmpInst *Old = cast<ICmpInst>(Val: U);
1207 Value *Cmp =
1208 B.CreateICmp(P: Old->getPredicate(), LHS: StrNCmp,
1209 RHS: ConstantInt::getNullValue(Ty: StrNCmp->getType()), Name: "cmp");
1210 replaceAllUsesWith(I: Old, With: Cmp);
1211 }
1212 return CI;
1213 }
1214
1215 // See if either input string is a constant string.
1216 StringRef SearchStr, ToFindStr;
1217 bool HasStr1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: SearchStr);
1218 bool HasStr2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: ToFindStr);
1219
1220 // fold strstr(x, "") -> x.
1221 if (HasStr2 && ToFindStr.empty())
1222 return CI->getArgOperand(i: 0);
1223
1224 // If both strings are known, constant fold it.
1225 if (HasStr1 && HasStr2) {
1226 size_t Offset = SearchStr.find(Str: ToFindStr);
1227
1228 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1229 return Constant::getNullValue(Ty: CI->getType());
1230
1231 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1232 return B.CreateConstInBoundsGEP1_64(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1233 Idx0: Offset, Name: "strstr");
1234 }
1235
1236 // fold strstr(x, "y") -> strchr(x, 'y').
1237 if (HasStr2 && ToFindStr.size() == 1) {
1238 return emitStrChr(Ptr: CI->getArgOperand(i: 0), C: ToFindStr[0], B, TLI);
1239 }
1240
1241 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
1242 return nullptr;
1243}
1244
1245Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1246 Value *SrcStr = CI->getArgOperand(i: 0);
1247 Value *Size = CI->getArgOperand(i: 2);
1248 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1249 Value *CharVal = CI->getArgOperand(i: 1);
1250 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1251 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1252
1253 if (LenC) {
1254 if (LenC->isZero())
1255 // Fold memrchr(x, y, 0) --> null.
1256 return NullPtr;
1257
1258 if (LenC->isOne()) {
1259 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1260 // constant or otherwise.
1261 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memrchr.char0");
1262 // Slice off the character's high end bits.
1263 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1264 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memrchr.char0cmp");
1265 return B.CreateSelect(C: Cmp, True: SrcStr, False: NullPtr, Name: "memrchr.sel");
1266 }
1267 }
1268
1269 StringRef Str;
1270 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1271 return nullptr;
1272
1273 if (Str.size() == 0)
1274 // If the array is empty fold memrchr(A, C, N) to null for any value
1275 // of C and N on the basis that the only valid value of N is zero
1276 // (otherwise the call is undefined).
1277 return NullPtr;
1278
1279 uint64_t EndOff = UINT64_MAX;
1280 if (LenC) {
1281 EndOff = LenC->getZExtValue();
1282 if (Str.size() < EndOff)
1283 // Punt out-of-bounds accesses to sanitizers and/or libc.
1284 return nullptr;
1285 }
1286
1287 if (ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal)) {
1288 // Fold memrchr(S, C, N) for a constant C.
1289 size_t Pos = Str.rfind(C: CharC->getZExtValue(), From: EndOff);
1290 if (Pos == StringRef::npos)
1291 // When the character is not in the source array fold the result
1292 // to null regardless of Size.
1293 return NullPtr;
1294
1295 if (LenC)
1296 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1297 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos));
1298
1299 if (Str.find(C: Str[Pos]) == Pos) {
1300 // When there is just a single occurrence of C in S, i.e., the one
1301 // in Str[Pos], fold
1302 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1303 // for nonconstant N.
1304 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1305 Name: "memrchr.cmp");
1306 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr,
1307 IdxList: B.getInt64(C: Pos), Name: "memrchr.ptr_plus");
1308 return B.CreateSelect(C: Cmp, True: NullPtr, False: SrcPlus, Name: "memrchr.sel");
1309 }
1310 }
1311
1312 // Truncate the string to search at most EndOff characters.
1313 Str = Str.substr(Start: 0, N: EndOff);
1314 if (Str.find_first_not_of(C: Str[0]) != StringRef::npos)
1315 return nullptr;
1316
1317 // If the source array consists of all equal characters, then for any
1318 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1319 // N != 0 && *S == C ? S + N - 1 : null
1320 Type *SizeTy = Size->getType();
1321 Type *Int8Ty = B.getInt8Ty();
1322 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1323 // Slice off the sought character's high end bits.
1324 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1325 Value *CEqS0 = B.CreateICmpEQ(LHS: ConstantInt::get(Ty: Int8Ty, V: Str[0]), RHS: CharVal);
1326 Value *And = B.CreateLogicalAnd(Cond1: NNeZ, Cond2: CEqS0);
1327 Value *SizeM1 = B.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
1328 Value *SrcPlus =
1329 B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: SrcStr, IdxList: SizeM1, Name: "memrchr.ptr_plus");
1330 return B.CreateSelect(C: And, True: SrcPlus, False: NullPtr, Name: "memrchr.sel");
1331}
1332
1333Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1334 Value *SrcStr = CI->getArgOperand(i: 0);
1335 Value *Size = CI->getArgOperand(i: 2);
1336
1337 if (isKnownNonZero(V: Size, Q: DL)) {
1338 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1339 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1340 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1341 }
1342
1343 Value *CharVal = CI->getArgOperand(i: 1);
1344 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
1345 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1346 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1347
1348 // memchr(x, y, 0) -> null
1349 if (LenC) {
1350 if (LenC->isZero())
1351 return NullPtr;
1352
1353 if (LenC->isOne()) {
1354 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1355 // constant or otherwise.
1356 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memchr.char0");
1357 // Slice off the character's high end bits.
1358 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1359 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memchr.char0cmp");
1360 return B.CreateSelect(C: Cmp, True: SrcStr, False: NullPtr, Name: "memchr.sel");
1361 }
1362 }
1363
1364 StringRef Str;
1365 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1366 return nullptr;
1367
1368 if (CharC) {
1369 size_t Pos = Str.find(C: CharC->getZExtValue());
1370 if (Pos == StringRef::npos)
1371 // When the character is not in the source array fold the result
1372 // to null regardless of Size.
1373 return NullPtr;
1374
1375 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1376 // When the constant Size is less than or equal to the character
1377 // position also fold the result to null.
1378 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1379 Name: "memchr.cmp");
1380 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos),
1381 Name: "memchr.ptr");
1382 return B.CreateSelect(C: Cmp, True: NullPtr, False: SrcPlus);
1383 }
1384
1385 if (Str.size() == 0)
1386 // If the array is empty fold memchr(A, C, N) to null for any value
1387 // of C and N on the basis that the only valid value of N is zero
1388 // (otherwise the call is undefined).
1389 return NullPtr;
1390
1391 if (LenC)
1392 Str = substr(Str, Len: LenC->getZExtValue());
1393
1394 size_t Pos = Str.find_first_not_of(C: Str[0]);
1395 if (Pos == StringRef::npos
1396 || Str.find_first_not_of(C: Str[Pos], From: Pos) == StringRef::npos) {
1397 // If the source array consists of at most two consecutive sequences
1398 // of the same characters, then for any C and N (whether in bounds or
1399 // not), fold memchr(S, C, N) to
1400 // N != 0 && *S == C ? S : null
1401 // or for the two sequences to:
1402 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1403 // ^Sel2 ^Sel1 are denoted above.
1404 // The latter makes it also possible to fold strchr() calls with strings
1405 // of the same characters.
1406 Type *SizeTy = Size->getType();
1407 Type *Int8Ty = B.getInt8Ty();
1408
1409 // Slice off the sought character's high end bits.
1410 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1411
1412 Value *Sel1 = NullPtr;
1413 if (Pos != StringRef::npos) {
1414 // Handle two consecutive sequences of the same characters.
1415 Value *PosVal = ConstantInt::get(Ty: SizeTy, V: Pos);
1416 Value *StrPos = ConstantInt::get(Ty: Int8Ty, V: Str[Pos]);
1417 Value *CEqSPos = B.CreateICmpEQ(LHS: CharVal, RHS: StrPos);
1418 Value *NGtPos = B.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: Size, RHS: PosVal);
1419 Value *And = B.CreateAnd(LHS: CEqSPos, RHS: NGtPos);
1420 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: PosVal);
1421 Sel1 = B.CreateSelect(C: And, True: SrcPlus, False: NullPtr, Name: "memchr.sel1");
1422 }
1423
1424 Value *Str0 = ConstantInt::get(Ty: Int8Ty, V: Str[0]);
1425 Value *CEqS0 = B.CreateICmpEQ(LHS: Str0, RHS: CharVal);
1426 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1427 Value *And = B.CreateAnd(LHS: NNeZ, RHS: CEqS0);
1428 return B.CreateSelect(C: And, True: SrcStr, False: Sel1, Name: "memchr.sel2");
1429 }
1430
1431 if (!LenC) {
1432 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1433 // S is dereferenceable so it's safe to load from it and fold
1434 // memchr(S, C, N) == S to N && *S == C for any C and N.
1435 // TODO: This is safe even for nonconstant S.
1436 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1437
1438 // From now on we need a constant length and constant array.
1439 return nullptr;
1440 }
1441
1442 bool OptForSize = llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
1443 QueryType: PGSOQueryType::IRPass);
1444
1445 // If the char is variable but the input str and length are not we can turn
1446 // this memchr call into a simple bit field test. Of course this only works
1447 // when the return value is only checked against null.
1448 //
1449 // It would be really nice to reuse switch lowering here but we can't change
1450 // the CFG at this point.
1451 //
1452 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1453 // != 0
1454 // after bounds check.
1455 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CxtI: CI))
1456 return nullptr;
1457
1458 unsigned char Max =
1459 *std::max_element(first: reinterpret_cast<const unsigned char *>(Str.begin()),
1460 last: reinterpret_cast<const unsigned char *>(Str.end()));
1461
1462 // Make sure the bit field we're about to create fits in a register on the
1463 // target.
1464 // FIXME: On a 64 bit architecture this prevents us from using the
1465 // interesting range of alpha ascii chars. We could do better by emitting
1466 // two bitfields or shifting the range by 64 if no lower chars are used.
1467 if (!DL.fitsInLegalInteger(Width: Max + 1)) {
1468 // Build chain of ORs
1469 // Transform:
1470 // memchr("abcd", C, 4) != nullptr
1471 // to:
1472 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1473 std::string SortedStr = Str.str();
1474 llvm::sort(C&: SortedStr);
1475 // Compute the number of of non-contiguous ranges.
1476 unsigned NonContRanges = 1;
1477 for (size_t i = 1; i < SortedStr.size(); ++i) {
1478 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1479 NonContRanges++;
1480 }
1481 }
1482
1483 // Restrict this optimization to profitable cases with one or two range
1484 // checks.
1485 if (NonContRanges > 2)
1486 return nullptr;
1487
1488 // Slice off the character's high end bits.
1489 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1490
1491 SmallVector<Value *> CharCompares;
1492 for (unsigned char C : SortedStr)
1493 CharCompares.push_back(Elt: B.CreateICmpEQ(LHS: CharVal, RHS: B.getInt8(C)));
1494
1495 return B.CreateIntToPtr(V: B.CreateOr(Ops: CharCompares), DestTy: CI->getType());
1496 }
1497
1498 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1499 // creating unnecessary illegal types.
1500 unsigned char Width = NextPowerOf2(A: std::max(a: (unsigned char)7, b: Max));
1501
1502 // Now build the bit field.
1503 APInt Bitfield(Width, 0);
1504 for (char C : Str)
1505 Bitfield.setBit((unsigned char)C);
1506 Value *BitfieldC = B.getInt(AI: Bitfield);
1507
1508 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1509 Value *C = B.CreateZExtOrTrunc(V: CharVal, DestTy: BitfieldC->getType());
1510 C = B.CreateAnd(LHS: C, RHS: B.getIntN(N: Width, C: 0xFF));
1511
1512 // First check that the bit field access is within bounds.
1513 Value *Bounds = B.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: C, RHS: B.getIntN(N: Width, C: Width),
1514 Name: "memchr.bounds");
1515
1516 // Create code that checks if the given bit is set in the field.
1517 Value *Shl = B.CreateShl(LHS: B.getIntN(N: Width, C: 1ULL), RHS: C);
1518 Value *Bits = B.CreateIsNotNull(Arg: B.CreateAnd(LHS: Shl, RHS: BitfieldC), Name: "memchr.bits");
1519
1520 // Finally merge both checks and cast to pointer type. The inttoptr
1521 // implicitly zexts the i1 to intptr type.
1522 return B.CreateIntToPtr(V: B.CreateLogicalAnd(Cond1: Bounds, Cond2: Bits, Name: "memchr"),
1523 DestTy: CI->getType());
1524}
1525
1526// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1527// arrays LHS and RHS and nonconstant Size.
1528static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
1529 Value *Size, bool StrNCmp,
1530 IRBuilderBase &B, const DataLayout &DL) {
1531 if (LHS == RHS) // memcmp(s,s,x) -> 0
1532 return Constant::getNullValue(Ty: CI->getType());
1533
1534 StringRef LStr, RStr;
1535 if (!getConstantStringInfo(V: LHS, Str&: LStr, /*TrimAtNul=*/false) ||
1536 !getConstantStringInfo(V: RHS, Str&: RStr, /*TrimAtNul=*/false))
1537 return nullptr;
1538
1539 // If the contents of both constant arrays are known, fold a call to
1540 // memcmp(A, B, N) to
1541 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1542 // where Pos is the first mismatch between A and B, determined below.
1543
1544 uint64_t Pos = 0;
1545 Value *Zero = ConstantInt::get(Ty: CI->getType(), V: 0);
1546 for (uint64_t MinSize = std::min(a: LStr.size(), b: RStr.size()); ; ++Pos) {
1547 if (Pos == MinSize ||
1548 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1549 // One array is a leading part of the other of equal or greater
1550 // size, or for strncmp, the arrays are equal strings.
1551 // Fold the result to zero. Size is assumed to be in bounds, since
1552 // otherwise the call would be undefined.
1553 return Zero;
1554 }
1555
1556 if (LStr[Pos] != RStr[Pos])
1557 break;
1558 }
1559
1560 // Normalize the result.
1561 typedef unsigned char UChar;
1562 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1563 Value *MaxSize = ConstantInt::get(Ty: Size->getType(), V: Pos);
1564 Value *Cmp = B.CreateICmp(P: ICmpInst::ICMP_ULE, LHS: Size, RHS: MaxSize);
1565 Value *Res = ConstantInt::getSigned(Ty: CI->getType(), V: IRes);
1566 return B.CreateSelect(C: Cmp, True: Zero, False: Res);
1567}
1568
1569// Optimize a memcmp call CI with constant size Len.
1570static Value *optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS,
1571 uint64_t Len, IRBuilderBase &B,
1572 const DataLayout &DL) {
1573 if (Len == 0) // memcmp(s1,s2,0) -> 0
1574 return Constant::getNullValue(Ty: CI->getType());
1575
1576 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1577 if (Len == 1) {
1578 Value *LHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: LHS, Name: "lhsc"),
1579 DestTy: CI->getType(), Name: "lhsv");
1580 Value *RHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: RHS, Name: "rhsc"),
1581 DestTy: CI->getType(), Name: "rhsv");
1582 return B.CreateSub(LHS: LHSV, RHS: RHSV, Name: "chardiff");
1583 }
1584
1585 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1586 // TODO: The case where both inputs are constants does not need to be limited
1587 // to legal integers or equality comparison. See block below this.
1588 if (DL.isLegalInteger(Width: Len * 8) && isOnlyUsedInZeroEqualityComparison(CxtI: CI)) {
1589 IntegerType *IntType = IntegerType::get(C&: CI->getContext(), NumBits: Len * 8);
1590 Align PrefAlignment = DL.getPrefTypeAlign(Ty: IntType);
1591
1592 // First, see if we can fold either argument to a constant.
1593 Value *LHSV = nullptr;
1594 if (auto *LHSC = dyn_cast<Constant>(Val: LHS))
1595 LHSV = ConstantFoldLoadFromConstPtr(C: LHSC, Ty: IntType, DL);
1596
1597 Value *RHSV = nullptr;
1598 if (auto *RHSC = dyn_cast<Constant>(Val: RHS))
1599 RHSV = ConstantFoldLoadFromConstPtr(C: RHSC, Ty: IntType, DL);
1600
1601 // Don't generate unaligned loads. If either source is constant data,
1602 // alignment doesn't matter for that source because there is no load.
1603 if ((LHSV || getKnownAlignment(V: LHS, DL, CxtI: CI) >= PrefAlignment) &&
1604 (RHSV || getKnownAlignment(V: RHS, DL, CxtI: CI) >= PrefAlignment)) {
1605 if (!LHSV)
1606 LHSV = B.CreateLoad(Ty: IntType, Ptr: LHS, Name: "lhsv");
1607 if (!RHSV)
1608 RHSV = B.CreateLoad(Ty: IntType, Ptr: RHS, Name: "rhsv");
1609 return B.CreateZExt(V: B.CreateICmpNE(LHS: LHSV, RHS: RHSV), DestTy: CI->getType(), Name: "memcmp");
1610 }
1611 }
1612
1613 return nullptr;
1614}
1615
1616// Most simplifications for memcmp also apply to bcmp.
1617Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1618 IRBuilderBase &B) {
1619 Value *LHS = CI->getArgOperand(i: 0), *RHS = CI->getArgOperand(i: 1);
1620 Value *Size = CI->getArgOperand(i: 2);
1621
1622 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1623
1624 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, StrNCmp: false, B, DL))
1625 return Res;
1626
1627 // Handle constant Size.
1628 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1629 if (!LenC)
1630 return nullptr;
1631
1632 return optimizeMemCmpConstantSize(CI, LHS, RHS, Len: LenC->getZExtValue(), B, DL);
1633}
1634
1635Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1636 Module *M = CI->getModule();
1637 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1638 return V;
1639
1640 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1641 // bcmp can be more efficient than memcmp because it only has to know that
1642 // there is a difference, not how different one is to the other.
1643 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_bcmp) &&
1644 isOnlyUsedInZeroEqualityComparison(CxtI: CI)) {
1645 Value *LHS = CI->getArgOperand(i: 0);
1646 Value *RHS = CI->getArgOperand(i: 1);
1647 Value *Size = CI->getArgOperand(i: 2);
1648 return copyFlags(Old: *CI, New: emitBCmp(Ptr1: LHS, Ptr2: RHS, Len: Size, B, DL, TLI));
1649 }
1650
1651 return nullptr;
1652}
1653
1654Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1655 return optimizeMemCmpBCmpCommon(CI, B);
1656}
1657
1658Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1659 Value *Size = CI->getArgOperand(i: 2);
1660 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1661 if (isa<IntrinsicInst>(Val: CI))
1662 return nullptr;
1663
1664 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1665 CallInst *NewCI = B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1666 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1667 mergeAttributesAndFlags(NewCI, Old: *CI);
1668 return CI->getArgOperand(i: 0);
1669}
1670
1671Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1672 Value *Dst = CI->getArgOperand(i: 0);
1673 Value *Src = CI->getArgOperand(i: 1);
1674 ConstantInt *StopChar = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
1675 ConstantInt *N = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
1676 StringRef SrcStr;
1677 if (CI->use_empty() && Dst == Src)
1678 return Dst;
1679 // memccpy(d, s, c, 0) -> nullptr
1680 if (N) {
1681 if (N->isNullValue())
1682 return Constant::getNullValue(Ty: CI->getType());
1683 if (!getConstantStringInfo(V: Src, Str&: SrcStr, /*TrimAtNul=*/false) ||
1684 // TODO: Handle zeroinitializer.
1685 !StopChar)
1686 return nullptr;
1687 } else {
1688 return nullptr;
1689 }
1690
1691 // Wrap arg 'c' of type int to char
1692 size_t Pos = SrcStr.find(C: StopChar->getSExtValue() & 0xFF);
1693 if (Pos == StringRef::npos) {
1694 if (N->getZExtValue() <= SrcStr.size()) {
1695 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
1696 Size: CI->getArgOperand(i: 3)));
1697 return Constant::getNullValue(Ty: CI->getType());
1698 }
1699 return nullptr;
1700 }
1701
1702 Value *NewN =
1703 ConstantInt::get(Ty: N->getType(), V: std::min(a: uint64_t(Pos + 1), b: N->getZExtValue()));
1704 // memccpy -> llvm.memcpy
1705 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: NewN));
1706 return Pos + 1 <= N->getZExtValue()
1707 ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: NewN)
1708 : Constant::getNullValue(Ty: CI->getType());
1709}
1710
1711Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1712 Value *Dst = CI->getArgOperand(i: 0);
1713 Value *N = CI->getArgOperand(i: 2);
1714 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1715 CallInst *NewCI =
1716 B.CreateMemCpy(Dst, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size: N);
1717 // Propagate attributes, but memcpy has no return value, so make sure that
1718 // any return attributes are compliant.
1719 // TODO: Attach return value attributes to the 1st operand to preserve them?
1720 mergeAttributesAndFlags(NewCI, Old: *CI);
1721 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: N);
1722}
1723
1724Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1725 Value *Size = CI->getArgOperand(i: 2);
1726 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1727 if (isa<IntrinsicInst>(Val: CI))
1728 return nullptr;
1729
1730 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1731 CallInst *NewCI = B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1732 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1733 mergeAttributesAndFlags(NewCI, Old: *CI);
1734 return CI->getArgOperand(i: 0);
1735}
1736
1737Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1738 Value *Size = CI->getArgOperand(i: 2);
1739 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1740 if (isa<IntrinsicInst>(Val: CI))
1741 return nullptr;
1742
1743 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1744 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
1745 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val, Size, Align: Align(1));
1746 mergeAttributesAndFlags(NewCI, Old: *CI);
1747 return CI->getArgOperand(i: 0);
1748}
1749
1750Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1751 if (isa<ConstantPointerNull>(Val: CI->getArgOperand(i: 0))) {
1752 Value *Malloc = emitMalloc(Num: CI->getArgOperand(i: 1), B, DL, TLI);
1753 if (auto *MallocCI = dyn_cast_or_null<CallInst>(Val: Malloc))
1754 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_alloc_token))
1755 MallocCI->setMetadata(KindID: LLVMContext::MD_alloc_token, Node: MD);
1756 return copyFlags(Old: *CI, New: Malloc);
1757 }
1758
1759 return nullptr;
1760}
1761
1762// Optionally allow optimization of nobuiltin calls to operator new and its
1763// variants.
1764Value *LibCallSimplifier::maybeOptimizeNoBuiltinOperatorNew(CallInst *CI,
1765 IRBuilderBase &B) {
1766 if (!OptimizeHotColdNew)
1767 return nullptr;
1768 Function *Callee = CI->getCalledFunction();
1769 if (!Callee)
1770 return nullptr;
1771 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
1772 if (Func == NotLibFunc)
1773 return nullptr;
1774 switch (Func) {
1775 case LibFunc_Znwm:
1776 case LibFunc_ZnwmRKSt9nothrow_t:
1777 case LibFunc_ZnwmSt11align_val_t:
1778 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1779 case LibFunc_Znam:
1780 case LibFunc_ZnamRKSt9nothrow_t:
1781 case LibFunc_ZnamSt11align_val_t:
1782 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1783 case LibFunc_size_returning_new:
1784 case LibFunc_size_returning_new_aligned:
1785 // By default normal operator new calls (not already passing a hot_cold_t
1786 // parameter) are not mutated if the call is not marked builtin. Optionally
1787 // enable that in cases where it is known to be safe.
1788 if (!OptimizeNoBuiltinHotColdNew)
1789 return nullptr;
1790 break;
1791 case LibFunc_Znwm12__hot_cold_t:
1792 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1793 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1794 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1795 case LibFunc_Znam12__hot_cold_t:
1796 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1797 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1798 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1799 case LibFunc_size_returning_new_hot_cold:
1800 case LibFunc_size_returning_new_aligned_hot_cold:
1801 // If the nobuiltin call already passes a hot_cold_t parameter, allow update
1802 // of that parameter when enabled.
1803 if (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::None)
1804 return nullptr;
1805 break;
1806 default:
1807 return nullptr;
1808 }
1809 return optimizeNew(CI, B, Func);
1810}
1811
1812// When enabled, replace operator new() calls marked with a hot or cold memprof
1813// attribute with an operator new() call that takes a __hot_cold_t parameter.
1814// Currently this is supported by the open source version of tcmalloc, see:
1815// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1816Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1817 LibFunc &Func) {
1818 if (!OptimizeHotColdNew)
1819 return nullptr;
1820
1821 uint8_t HotCold;
1822 bool IsCold = false;
1823 if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold") {
1824 HotCold = ColdNewHintValue;
1825 IsCold = true;
1826 } else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1827 "notcold")
1828 HotCold = NotColdNewHintValue;
1829 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "hot")
1830 HotCold = HotNewHintValue;
1831 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1832 "ambiguous")
1833 HotCold = AmbiguousNewHintValue;
1834 else
1835 return nullptr;
1836
1837 bool ShouldOptimizeExistingHotColdNew =
1838 OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Always ||
1839 (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Cold &&
1840 IsCold);
1841
1842 Value *HotColdVal = B.getInt8(C: HotCold);
1843 auto getHotColdHintForExisting = [&](uint8_t HotCold) -> Value * {
1844 // If not taking the minimum, simply use the compiler hint value.
1845 if (!MinExistingHotColdNewHint)
1846 return HotColdVal;
1847 Value *ExistingHint = CI->getArgOperand(i: CI->arg_size() - 1);
1848 if (ExistingHint->getType() != B.getInt8Ty())
1849 ExistingHint = B.CreateTruncOrBitCast(V: ExistingHint, DestTy: B.getInt8Ty());
1850 // Emit a umin intrinsic to take the minimum of the existing hint and the
1851 // compiler hint. When the existing hint is a compile-time constant, the
1852 // IRBuilder folder will automatically constant-fold this into a constant.
1853 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: ExistingHint, RHS: HotColdVal);
1854 };
1855
1856 // For calls that already pass a hot/cold hint, only update the hint if
1857 // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
1858 // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
1859 // Note that in cases where we decide it is "notcold", it might be slightly
1860 // better to replace the hinted call with a non hinted call, to avoid the
1861 // extra parameter and the if condition check of the hint value in the
1862 // allocator. This can be considered in the future.
1863 Value *NewCall = nullptr;
1864 switch (Func) {
1865 case LibFunc_Znwm12__hot_cold_t:
1866 if (ShouldOptimizeExistingHotColdNew)
1867 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1868 NewFunc: LibFunc_Znwm12__hot_cold_t,
1869 HotCold: getHotColdHintForExisting(HotCold));
1870 break;
1871 case LibFunc_Znwm:
1872 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1873 NewFunc: LibFunc_Znwm12__hot_cold_t, HotCold: HotColdVal);
1874 break;
1875 case LibFunc_Znam12__hot_cold_t:
1876 if (ShouldOptimizeExistingHotColdNew)
1877 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1878 NewFunc: LibFunc_Znam12__hot_cold_t,
1879 HotCold: getHotColdHintForExisting(HotCold));
1880 break;
1881 case LibFunc_Znam:
1882 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1883 NewFunc: LibFunc_Znam12__hot_cold_t, HotCold: HotColdVal);
1884 break;
1885 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1886 if (ShouldOptimizeExistingHotColdNew)
1887 NewCall =
1888 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1889 TLI, NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
1890 HotCold: getHotColdHintForExisting(HotCold));
1891 break;
1892 case LibFunc_ZnwmRKSt9nothrow_t:
1893 NewCall = emitHotColdNewNoThrow(
1894 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1895 NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1896 break;
1897 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1898 if (ShouldOptimizeExistingHotColdNew)
1899 NewCall =
1900 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1901 TLI, NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
1902 HotCold: getHotColdHintForExisting(HotCold));
1903 break;
1904 case LibFunc_ZnamRKSt9nothrow_t:
1905 NewCall = emitHotColdNewNoThrow(
1906 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1907 NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1908 break;
1909 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1910 if (ShouldOptimizeExistingHotColdNew)
1911 NewCall =
1912 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1913 TLI, NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
1914 HotCold: getHotColdHintForExisting(HotCold));
1915 break;
1916 case LibFunc_ZnwmSt11align_val_t:
1917 NewCall = emitHotColdNewAligned(
1918 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1919 NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1920 break;
1921 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1922 if (ShouldOptimizeExistingHotColdNew)
1923 NewCall =
1924 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1925 TLI, NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t,
1926 HotCold: getHotColdHintForExisting(HotCold));
1927 break;
1928 case LibFunc_ZnamSt11align_val_t:
1929 NewCall = emitHotColdNewAligned(
1930 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1931 NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1932 break;
1933 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1934 if (ShouldOptimizeExistingHotColdNew)
1935 NewCall = emitHotColdNewAlignedNoThrow(
1936 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1937 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1938 HotCold: getHotColdHintForExisting(HotCold));
1939 break;
1940 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1941 NewCall = emitHotColdNewAlignedNoThrow(
1942 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1943 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1944 HotCold: HotColdVal);
1945 break;
1946 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1947 if (ShouldOptimizeExistingHotColdNew)
1948 NewCall = emitHotColdNewAlignedNoThrow(
1949 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1950 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1951 HotCold: getHotColdHintForExisting(HotCold));
1952 break;
1953 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1954 NewCall = emitHotColdNewAlignedNoThrow(
1955 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1956 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1957 HotCold: HotColdVal);
1958 break;
1959 case LibFunc_size_returning_new:
1960 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
1961 NewFunc: LibFunc_size_returning_new_hot_cold,
1962 HotCold: HotColdVal);
1963 break;
1964 case LibFunc_size_returning_new_hot_cold:
1965 if (ShouldOptimizeExistingHotColdNew)
1966 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
1967 NewFunc: LibFunc_size_returning_new_hot_cold,
1968 HotCold: getHotColdHintForExisting(HotCold));
1969 break;
1970 case LibFunc_size_returning_new_aligned:
1971 NewCall = emitHotColdSizeReturningNewAligned(
1972 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1973 NewFunc: LibFunc_size_returning_new_aligned_hot_cold, HotCold: HotColdVal);
1974 break;
1975 case LibFunc_size_returning_new_aligned_hot_cold:
1976 if (ShouldOptimizeExistingHotColdNew)
1977 NewCall = emitHotColdSizeReturningNewAligned(
1978 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1979 NewFunc: LibFunc_size_returning_new_aligned_hot_cold,
1980 HotCold: getHotColdHintForExisting(HotCold));
1981 break;
1982 default:
1983 return nullptr;
1984 }
1985
1986 if (auto *NewCI = dyn_cast_or_null<Instruction>(Val: NewCall))
1987 NewCI->copyMetadata(SrcInst: *CI);
1988
1989 return NewCall;
1990}
1991
1992//===----------------------------------------------------------------------===//
1993// Math Library Optimizations
1994//===----------------------------------------------------------------------===//
1995
1996// Replace a libcall \p CI with a call to intrinsic \p IID
1997static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
1998 Intrinsic::ID IID) {
1999 Value *NewCall = B.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0), FMFSource: CI);
2000 NewCall->takeName(V: CI);
2001 return copyFlags(Old: *CI, New: NewCall);
2002}
2003
2004static Value *replaceBinaryCall(CallInst *CI, IRBuilderBase &B,
2005 Intrinsic::ID IID) {
2006 Value *NewCall = B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2007 RHS: CI->getArgOperand(i: 1), FMFSource: CI);
2008 NewCall->takeName(V: CI);
2009 return copyFlags(Old: *CI, New: NewCall);
2010}
2011
2012/// Return a variant of Val with float type.
2013/// Currently this works in two cases: If Val is an FPExtension of a float
2014/// value to something bigger, simply return the operand.
2015/// If Val is a ConstantFP but can be converted to a float ConstantFP without
2016/// loss of precision do so.
2017static Value *valueHasFloatPrecision(Value *Val) {
2018 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
2019 Value *Op = Cast->getOperand(i_nocapture: 0);
2020 if (Op->getType()->isFloatTy())
2021 return Op;
2022 }
2023 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
2024 APFloat F = Const->getValueAPF();
2025 bool losesInfo;
2026 (void)F.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
2027 losesInfo: &losesInfo);
2028 if (!losesInfo)
2029 return ConstantFP::get(Context&: Const->getContext(), V: F);
2030 }
2031 return nullptr;
2032}
2033
2034/// Shrink double -> float functions.
2035static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
2036 bool isBinary, const TargetLibraryInfo *TLI,
2037 bool isPrecise = false) {
2038 Function *CalleeFn = CI->getCalledFunction();
2039 if (!CI->getType()->isDoubleTy() || !CalleeFn)
2040 return nullptr;
2041
2042 // If not all the uses of the function are converted to float, then bail out.
2043 // This matters if the precision of the result is more important than the
2044 // precision of the arguments.
2045 if (isPrecise)
2046 for (User *U : CI->users()) {
2047 FPTruncInst *Cast = dyn_cast<FPTruncInst>(Val: U);
2048 if (!Cast || !Cast->getType()->isFloatTy())
2049 return nullptr;
2050 }
2051
2052 // If this is something like 'g((double) float)', convert to 'gf(float)'.
2053 Value *V[2];
2054 V[0] = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2055 V[1] = isBinary ? valueHasFloatPrecision(Val: CI->getArgOperand(i: 1)) : nullptr;
2056 if (!V[0] || (isBinary && !V[1]))
2057 return nullptr;
2058
2059 // If call isn't an intrinsic, check that it isn't within a function with the
2060 // same name as the float version of this call, otherwise the result is an
2061 // infinite loop. For example, from MinGW-w64:
2062 //
2063 // float expf(float val) { return (float) exp((double) val); }
2064 StringRef CalleeName = CalleeFn->getName();
2065 bool IsIntrinsic = CalleeFn->isIntrinsic();
2066 if (!IsIntrinsic) {
2067 StringRef CallerName = CI->getFunction()->getName();
2068 if (CallerName.ends_with(Suffix: 'f') &&
2069 CallerName.size() == (CalleeName.size() + 1) &&
2070 CallerName.starts_with(Prefix: CalleeName))
2071 return nullptr;
2072 }
2073
2074 // Propagate the math semantics from the current function to the new function.
2075 IRBuilderBase::FastMathFlagGuard Guard(B);
2076 B.setFastMathFlags(CI->getFastMathFlags());
2077
2078 // g((double) float) -> (double) gf(float)
2079 Value *R;
2080 if (IsIntrinsic) {
2081 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
2082 R = isBinary ? B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V)
2083 : B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V[0]);
2084 } else {
2085 AttributeList CallsiteAttrs = CI->getAttributes();
2086 R = isBinary
2087 ? emitBinaryFloatFnCall(Op1: V[0], Op2: V[1], TLI, Name: CalleeName, B,
2088 Attrs: CallsiteAttrs)
2089 : emitUnaryFloatFnCall(Op: V[0], TLI, Name: CalleeName, B, Attrs: CallsiteAttrs);
2090 }
2091 return B.CreateFPExt(V: R, DestTy: B.getDoubleTy());
2092}
2093
2094/// Shrink double -> float for unary functions.
2095static Value *optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2096 const TargetLibraryInfo *TLI,
2097 bool isPrecise = false) {
2098 return optimizeDoubleFP(CI, B, isBinary: false, TLI, isPrecise);
2099}
2100
2101/// Shrink double -> float for binary functions.
2102static Value *optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2103 const TargetLibraryInfo *TLI,
2104 bool isPrecise = false) {
2105 return optimizeDoubleFP(CI, B, isBinary: true, TLI, isPrecise);
2106}
2107
2108/// Shrink double -> float for llvm.sincos.
2109static Value *optimizeSinCosDoubleFP(CallInst *CI, IRBuilderBase &B) {
2110 auto *RetTy = dyn_cast<StructType>(Val: CI->getType());
2111 if (!RetTy || RetTy->getNumElements() != 2 ||
2112 !RetTy->getElementType(N: 0)->getScalarType()->isDoubleTy())
2113 return nullptr;
2114
2115 Value *X = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2116 if (!X)
2117 if (auto *Ext = dyn_cast<FPExtInst>(Val: CI->getArgOperand(i: 0)))
2118 if (Ext->getOperand(i_nocapture: 0)->getType()->getScalarType()->isFloatTy())
2119 X = Ext->getOperand(i_nocapture: 0);
2120 if (!X)
2121 return nullptr;
2122
2123 for (User *U : CI->users()) {
2124 auto *EV = dyn_cast<ExtractValueInst>(Val: U);
2125 if (!EV)
2126 return nullptr;
2127 for (User *EVU : EV->users()) {
2128 auto *Cast = dyn_cast<FPTruncInst>(Val: EVU);
2129 if (!Cast || !Cast->getType()->getScalarType()->isFloatTy())
2130 return nullptr;
2131 }
2132 }
2133
2134 IRBuilderBase::FastMathFlagGuard Guard(B);
2135 B.setFastMathFlags(CI->getFastMathFlags());
2136
2137 Value *NewCall = B.CreateIntrinsic(ID: Intrinsic::sincos, OverloadTypes: X->getType(), Args: X);
2138 cast<Instruction>(Val: NewCall)->setMetadata(
2139 KindID: LLVMContext::MD_fpmath, Node: CI->getMetadata(KindID: LLVMContext::MD_fpmath));
2140 Value *Res = PoisonValue::get(T: RetTy);
2141 for (unsigned I = 0; I != 2; ++I) {
2142 Value *Ext = B.CreateFPExt(V: B.CreateExtractValue(Agg: NewCall, Idxs: I),
2143 DestTy: RetTy->getElementType(N: I));
2144 Res = B.CreateInsertValue(Agg: Res, Val: Ext, Idxs: I);
2145 }
2146 return Res;
2147}
2148
2149// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
2150Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
2151 Value *Real, *Imag;
2152
2153 if (CI->arg_size() == 1) {
2154
2155 if (!CI->isFast())
2156 return nullptr;
2157
2158 Value *Op = CI->getArgOperand(i: 0);
2159 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
2160
2161 Real = B.CreateExtractValue(Agg: Op, Idxs: 0, Name: "real");
2162 Imag = B.CreateExtractValue(Agg: Op, Idxs: 1, Name: "imag");
2163
2164 } else {
2165 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
2166
2167 Real = CI->getArgOperand(i: 0);
2168 Imag = CI->getArgOperand(i: 1);
2169
2170 // if real or imaginary part is zero, simplify to abs(cimag(z))
2171 // or abs(creal(z))
2172 Value *AbsOp = nullptr;
2173 if (ConstantFP *ConstReal = dyn_cast<ConstantFP>(Val: Real)) {
2174 if (ConstReal->isZero())
2175 AbsOp = Imag;
2176
2177 } else if (ConstantFP *ConstImag = dyn_cast<ConstantFP>(Val: Imag)) {
2178 if (ConstImag->isZero())
2179 AbsOp = Real;
2180 }
2181
2182 if (AbsOp)
2183 return copyFlags(Old: *CI, New: B.CreateFAbs(V: AbsOp, FMFSource: CI, Name: "cabs"));
2184
2185 if (!CI->isFast())
2186 return nullptr;
2187 }
2188
2189 // Propagate fast-math flags from the existing call to new instructions.
2190 Value *RealReal = B.CreateFMulFMF(L: Real, R: Real, FMFSource: CI);
2191 Value *ImagImag = B.CreateFMulFMF(L: Imag, R: Imag, FMFSource: CI);
2192 return copyFlags(
2193 Old: *CI, New: B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt,
2194 Op: B.CreateFAddFMF(L: RealReal, R: ImagImag, FMFSource: CI), FMFSource: CI,
2195 Name: "cabs"));
2196}
2197
2198// Return a properly extended integer (DstWidth bits wide) if the operation is
2199// an itofp.
2200static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
2201 if (isa<SIToFPInst>(Val: I2F) || isa<UIToFPInst>(Val: I2F)) {
2202 Value *Op = cast<Instruction>(Val: I2F)->getOperand(i: 0);
2203 // Make sure that the exponent fits inside an "int" of size DstWidth,
2204 // thus avoiding any range issues that FP has not.
2205 unsigned BitWidth = Op->getType()->getScalarSizeInBits();
2206 if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(Val: I2F))) {
2207 Type *IntTy = Op->getType()->getWithNewBitWidth(NewBitWidth: DstWidth);
2208 return isa<SIToFPInst>(Val: I2F) ? B.CreateSExt(V: Op, DestTy: IntTy)
2209 : B.CreateZExt(V: Op, DestTy: IntTy);
2210 }
2211 }
2212
2213 return nullptr;
2214}
2215
2216/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
2217/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
2218/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
2219Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
2220 Module *M = Pow->getModule();
2221 Value *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2222 Type *Ty = Pow->getType();
2223 bool Ignored;
2224
2225 // Evaluate special cases related to a nested function as the base.
2226
2227 // pow(exp(x), y) -> exp(x * y)
2228 // pow(exp2(x), y) -> exp2(x * y)
2229 // If exp{,2}() is used only once, it is better to fold two transcendental
2230 // math functions into one. If used again, exp{,2}() would still have to be
2231 // called with the original argument, then keep both original transcendental
2232 // functions. However, this transformation is only safe with fully relaxed
2233 // math semantics, since, besides rounding differences, it changes overflow
2234 // and underflow behavior quite dramatically. For example:
2235 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
2236 // Whereas:
2237 // exp(1000 * 0.001) = exp(1)
2238 // TODO: Loosen the requirement for fully relaxed math semantics.
2239 // TODO: Handle exp10() when more targets have it available.
2240 CallInst *BaseFn = dyn_cast<CallInst>(Val: Base);
2241 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
2242 Function *CalleeFn = BaseFn->getCalledFunction();
2243 LibFunc LibFn =
2244 CalleeFn ? TLI->getLibFunc(funcName: CalleeFn->getName()) : NotLibFunc;
2245 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFn)) {
2246 StringRef ExpName;
2247 Intrinsic::ID ID;
2248 Value *ExpFn;
2249 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2250
2251 switch (LibFn) {
2252 default:
2253 return nullptr;
2254 case LibFunc_expf:
2255 case LibFunc_exp:
2256 case LibFunc_expl:
2257 ExpName = TLI->getName(F: LibFunc_exp);
2258 ID = Intrinsic::exp;
2259 LibFnFloat = LibFunc_expf;
2260 LibFnDouble = LibFunc_exp;
2261 LibFnLongDouble = LibFunc_expl;
2262 break;
2263 case LibFunc_exp2f:
2264 case LibFunc_exp2:
2265 case LibFunc_exp2l:
2266 ExpName = TLI->getName(F: LibFunc_exp2);
2267 ID = Intrinsic::exp2;
2268 LibFnFloat = LibFunc_exp2f;
2269 LibFnDouble = LibFunc_exp2;
2270 LibFnLongDouble = LibFunc_exp2l;
2271 break;
2272 }
2273
2274 // Create new exp{,2}() with the product as its argument.
2275 Value *FMul = B.CreateFMul(L: BaseFn->getArgOperand(i: 0), R: Expo, Name: "mul");
2276 ExpFn = BaseFn->doesNotAccessMemory()
2277 ? B.CreateUnaryIntrinsic(ID, Op: FMul, FMFSource: nullptr, Name: ExpName)
2278 : emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFnDouble, FloatFn: LibFnFloat,
2279 LongDoubleFn: LibFnLongDouble, B,
2280 Attrs: BaseFn->getAttributes());
2281
2282 // Since the new exp{,2}() is different from the original one, dead code
2283 // elimination cannot be trusted to remove it, since it may have side
2284 // effects (e.g., errno). When the only consumer for the original
2285 // exp{,2}() is pow(), then it has to be explicitly erased.
2286 substituteInParent(I: BaseFn, With: ExpFn);
2287 return ExpFn;
2288 }
2289 }
2290
2291 // Evaluate special cases related to a constant base.
2292
2293 const APFloat *BaseF;
2294 if (!match(V: Base, P: m_APFloat(Res&: BaseF)))
2295 return nullptr;
2296
2297 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2298
2299 const bool UseIntrinsic = Pow->doesNotAccessMemory();
2300
2301 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2302 if ((UseIntrinsic || !Ty->isVectorTy()) && BaseF->isExactlyValue(V: 2.0) &&
2303 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo)) &&
2304 (UseIntrinsic ||
2305 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2306
2307 // TODO: Shouldn't really need to depend on getIntToFPVal for intrinsic. Can
2308 // just directly use the original integer type.
2309 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize())) {
2310 Constant *One = ConstantFP::get(Ty, V: 1.0);
2311
2312 if (UseIntrinsic) {
2313 return copyFlags(Old: *Pow, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2314 OverloadTypes: {Ty, ExpoI->getType()},
2315 Args: {One, ExpoI}, FMFSource: Pow, Name: "exp2"));
2316 }
2317
2318 return copyFlags(Old: *Pow, New: emitBinaryFloatFnCall(
2319 Op1: One, Op2: ExpoI, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2320 LongDoubleFn: LibFunc_ldexpl, B, Attrs: NoAttrs));
2321 }
2322 }
2323
2324 // pow(2.0 ** n, x) -> exp2(n * x)
2325 if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f, LongDoubleFn: LibFunc_exp2l)) {
2326 APFloat BaseR = APFloat(1.0);
2327 BaseR.convert(ToSemantics: BaseF->getSemantics(), RM: APFloat::rmTowardZero, losesInfo: &Ignored);
2328 BaseR = BaseR / *BaseF;
2329 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2330 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2331 APSInt NI(64, false);
2332 if ((IsInteger || IsReciprocal) &&
2333 NF->convertToInteger(Result&: NI, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2334 APFloat::opOK &&
2335 NI > 1 && NI.isPowerOf2()) {
2336 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2337 Value *FMul = B.CreateFMul(L: Expo, R: ConstantFP::get(Ty, V: N), Name: "mul");
2338 if (Pow->doesNotAccessMemory())
2339 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2340 FMFSource: nullptr, Name: "exp2"));
2341 else
2342 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2343 FloatFn: LibFunc_exp2f,
2344 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2345 }
2346 }
2347
2348 // pow(10.0, x) -> exp10(x)
2349 if (BaseF->isExactlyValue(V: 10.0) &&
2350 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp10, FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l)) {
2351
2352 if (Pow->doesNotAccessMemory()) {
2353 return B.CreateIntrinsic(ID: Intrinsic::exp10, OverloadTypes: {Ty}, Args: {Expo}, FMFSource: Pow, Name: "exp10", OpBundles: {},
2354 SetFn: [Pow](CallInst *CI) { CI->copyIRFlags(V: Pow); });
2355 }
2356
2357 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: Expo, TLI, DoubleFn: LibFunc_exp10,
2358 FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l,
2359 B, Attrs: NoAttrs));
2360 }
2361
2362 // pow(x, y) -> exp2(log2(x) * y)
2363 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2364 !BaseF->isNegative()) {
2365 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2366 // Luckily optimizePow has already handled the x == 1 case.
2367 assert(!match(Base, m_FPOne()) &&
2368 "pow(1.0, y) should have been simplified earlier!");
2369
2370 Value *Log = nullptr;
2371 if (Ty->isFloatTy())
2372 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToFloat()));
2373 else if (Ty->isDoubleTy())
2374 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToDouble()));
2375
2376 if (Log) {
2377 Value *FMul = B.CreateFMul(L: Log, R: Expo, Name: "mul");
2378 if (Pow->doesNotAccessMemory())
2379 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2380 FMFSource: nullptr, Name: "exp2"));
2381 else if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f,
2382 LongDoubleFn: LibFunc_exp2l))
2383 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2384 FloatFn: LibFunc_exp2f,
2385 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2386 }
2387 }
2388
2389 return nullptr;
2390}
2391
2392static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2393 Module *M, IRBuilderBase &B,
2394 const TargetLibraryInfo *TLI) {
2395 // If errno is never set, then use the intrinsic for sqrt().
2396 if (NoErrno)
2397 return B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: V, FMFSource: nullptr, Name: "sqrt");
2398
2399 // Otherwise, use the libcall for sqrt().
2400 if (hasFloatFn(M, TLI, Ty: V->getType(), DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2401 LongDoubleFn: LibFunc_sqrtl))
2402 // TODO: We also should check that the target can in fact lower the sqrt()
2403 // libcall. We currently have no way to ask this question, so we ask if
2404 // the target has a sqrt() libcall, which is not exactly the same.
2405 return emitUnaryFloatFnCall(Op: V, TLI, DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2406 LongDoubleFn: LibFunc_sqrtl, B, Attrs);
2407
2408 return nullptr;
2409}
2410
2411/// Use square root in place of pow(x, +/-0.5).
2412Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2413 Value *Sqrt, *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2414 Module *Mod = Pow->getModule();
2415 Type *Ty = Pow->getType();
2416
2417 const APFloat *ExpoF;
2418 if (!match(V: Expo, P: m_APFloat(Res&: ExpoF)) ||
2419 (!ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)))
2420 return nullptr;
2421
2422 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2423 // so that requires fast-math-flags (afn or reassoc).
2424 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2425 return nullptr;
2426
2427 // If we have a pow() library call (accesses memory) and we can't guarantee
2428 // that the base is not an infinity, give up:
2429 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2430 // errno), but sqrt(-Inf) is required by various standards to set errno.
2431 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2432 !isKnownNeverInfinity(
2433 V: Base, SQ: SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
2434 return nullptr;
2435
2436 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M: Mod, B,
2437 TLI);
2438 if (!Sqrt)
2439 return nullptr;
2440
2441 // Handle signed zero base by expanding to fabs(sqrt(x)).
2442 if (!Pow->hasNoSignedZeros())
2443 Sqrt = B.CreateFAbs(V: Sqrt, FMFSource: nullptr, Name: "abs");
2444
2445 Sqrt = copyFlags(Old: *Pow, New: Sqrt);
2446
2447 // Handle non finite base by expanding to
2448 // (x == -infinity ? +infinity : sqrt(x)).
2449 if (!Pow->hasNoInfs()) {
2450 Value *PosInf = ConstantFP::getInfinity(Ty),
2451 *NegInf = ConstantFP::getInfinity(Ty, Negative: true);
2452 Value *FCmp = B.CreateFCmpOEQ(LHS: Base, RHS: NegInf, Name: "isinf");
2453 Sqrt = B.CreateSelect(C: FCmp, True: PosInf, False: Sqrt);
2454 }
2455
2456 // If the exponent is negative, then get the reciprocal.
2457 if (ExpoF->isNegative())
2458 Sqrt = B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Sqrt, Name: "reciprocal");
2459
2460 return Sqrt;
2461}
2462
2463static Value *createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M,
2464 IRBuilderBase &B) {
2465 Value *Args[] = {Base, Expo};
2466 Type *Types[] = {Base->getType(), Expo->getType()};
2467 return B.CreateIntrinsic(ID: Intrinsic::powi, OverloadTypes: Types, Args);
2468}
2469
2470Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2471 Value *Base = Pow->getArgOperand(i: 0);
2472 Value *Expo = Pow->getArgOperand(i: 1);
2473 Function *Callee = Pow->getCalledFunction();
2474 StringRef Name = Callee->getName();
2475 Type *Ty = Pow->getType();
2476 Module *M = Pow->getModule();
2477 bool AllowApprox = Pow->hasApproxFunc();
2478 bool Ignored;
2479
2480 // Propagate the math semantics from the call to any created instructions.
2481 IRBuilderBase::FastMathFlagGuard Guard(B);
2482 B.setFastMathFlags(Pow->getFastMathFlags());
2483 // Evaluate special cases related to the base.
2484
2485 // pow(1.0, x) -> 1.0
2486 if (match(V: Base, P: m_FPOne()))
2487 return Base;
2488
2489 if (Value *Exp = replacePowWithExp(Pow, B))
2490 return Exp;
2491
2492 // Evaluate special cases related to the exponent.
2493
2494 // pow(x, -1.0) -> 1.0 / x
2495 if (match(V: Expo, P: m_SpecificFP(V: -1.0)))
2496 return B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Base, Name: "reciprocal");
2497
2498 // pow(x, +/-0.0) -> 1.0
2499 if (match(V: Expo, P: m_AnyZeroFP()))
2500 return ConstantFP::get(Ty, V: 1.0);
2501
2502 // pow(x, 1.0) -> x
2503 if (match(V: Expo, P: m_FPOne()))
2504 return Base;
2505
2506 // pow(x, 2.0) -> x * x
2507 if (match(V: Expo, P: m_SpecificFP(V: 2.0)) && Pow->doesNotAccessMemory())
2508 return B.CreateFMul(L: Base, R: Base, Name: "square");
2509
2510 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2511 return Sqrt;
2512
2513 // If we can approximate pow:
2514 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2515 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2516 const APFloat *ExpoF;
2517 if (AllowApprox && match(V: Expo, P: m_APFloat(Res&: ExpoF)) &&
2518 !ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)) {
2519 APFloat ExpoA(abs(X: *ExpoF));
2520 APFloat ExpoI(*ExpoF);
2521 Value *Sqrt = nullptr;
2522 if (!ExpoA.isInteger()) {
2523 APFloat Expo2 = ExpoA;
2524 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2525 // is no floating point exception and the result is an integer, then
2526 // ExpoA == integer + 0.5
2527 if (Expo2.add(RHS: ExpoA, RM: APFloat::rmNearestTiesToEven) != APFloat::opOK)
2528 return nullptr;
2529
2530 if (!Expo2.isInteger())
2531 return nullptr;
2532
2533 if (ExpoI.roundToIntegral(RM: APFloat::rmTowardNegative) !=
2534 APFloat::opInexact)
2535 return nullptr;
2536 if (!ExpoI.isInteger())
2537 return nullptr;
2538 ExpoF = &ExpoI;
2539
2540 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M,
2541 B, TLI);
2542 if (!Sqrt)
2543 return nullptr;
2544 }
2545
2546 // 0.5 fraction is now optionally handled.
2547 // Do pow -> powi for remaining integer exponent
2548 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2549 if (ExpoF->isInteger() &&
2550 ExpoF->convertToInteger(Result&: IntExpo, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2551 APFloat::opOK) {
2552 Value *PowI = copyFlags(
2553 Old: *Pow,
2554 New: createPowWithIntegerExponent(
2555 Base, Expo: ConstantInt::get(Ty: B.getIntNTy(N: TLI->getIntSize()), V: IntExpo),
2556 M, B));
2557
2558 if (PowI && Sqrt)
2559 return B.CreateFMul(L: PowI, R: Sqrt);
2560
2561 return PowI;
2562 }
2563 }
2564
2565 // powf(x, itofp(y)) -> powi(x, y)
2566 // The powi exponent must be a scalar integer, so a vector y is not usable.
2567 if (AllowApprox && !Expo->getType()->isVectorTy() &&
2568 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo))) {
2569 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize()))
2570 return copyFlags(Old: *Pow, New: createPowWithIntegerExponent(Base, Expo: ExpoI, M, B));
2571 }
2572
2573 // Shrink pow() to powf() if the arguments are single precision,
2574 // unless the result is expected to be double precision.
2575 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_pow) &&
2576 hasFloatVersion(M, FuncName: Name)) {
2577 if (Value *Shrunk = optimizeBinaryDoubleFP(CI: Pow, B, TLI, isPrecise: true))
2578 return Shrunk;
2579 }
2580
2581 return nullptr;
2582}
2583
2584Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2585 Module *M = CI->getModule();
2586 Function *Callee = CI->getCalledFunction();
2587 StringRef Name = Callee->getName();
2588 Value *Ret = nullptr;
2589 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_exp2) &&
2590 hasFloatVersion(M, FuncName: Name))
2591 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2592
2593 // If we have an llvm.exp2 intrinsic, emit the llvm.ldexp intrinsic. If we
2594 // have the libcall, emit the libcall.
2595 //
2596 // TODO: In principle we should be able to just always use the intrinsic for
2597 // any doesNotAccessMemory callsite.
2598
2599 const bool UseIntrinsic = Callee->isIntrinsic();
2600 // Bail out for vectors because the code below only expects scalars.
2601 Type *Ty = CI->getType();
2602 if (!UseIntrinsic && Ty->isVectorTy())
2603 return Ret;
2604
2605 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2606 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2607 Value *Op = CI->getArgOperand(i: 0);
2608 if ((isa<SIToFPInst>(Val: Op) || isa<UIToFPInst>(Val: Op)) &&
2609 (UseIntrinsic ||
2610 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2611 if (Value *Exp = getIntToFPVal(I2F: Op, B, DstWidth: TLI->getIntSize())) {
2612 Constant *One = ConstantFP::get(Ty, V: 1.0);
2613
2614 if (UseIntrinsic) {
2615 return copyFlags(Old: *CI, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2616 OverloadTypes: {Ty, Exp->getType()},
2617 Args: {One, Exp}, FMFSource: CI));
2618 }
2619
2620 IRBuilderBase::FastMathFlagGuard Guard(B);
2621 B.setFastMathFlags(CI->getFastMathFlags());
2622 return copyFlags(Old: *CI, New: emitBinaryFloatFnCall(
2623 Op1: One, Op2: Exp, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2624 LongDoubleFn: LibFunc_ldexpl, B, Attrs: AttributeList()));
2625 }
2626 }
2627
2628 return Ret;
2629}
2630
2631Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B,
2632 Intrinsic::ID IID) {
2633 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2634 // the intrinsics for improved optimization (for example, vectorization).
2635 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2636 // From the C standard draft WG14/N1256:
2637 // "Ideally, fmax would be sensitive to the sign of zero, for example
2638 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2639 // might be impractical."
2640 FastMathFlags FMF = CI->getFastMathFlags();
2641 FMF.setNoSignedZeros();
2642 return copyFlags(Old: *CI, New: B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2643 RHS: CI->getArgOperand(i: 1), FMFSource: FMF));
2644}
2645
2646Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2647 Function *LogFn = Log->getCalledFunction();
2648 StringRef LogNm = LogFn->getName();
2649 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2650 Module *Mod = Log->getModule();
2651 Type *Ty = Log->getType();
2652
2653 if (UnsafeFPShrink && hasFloatVersion(M: Mod, FuncName: LogNm))
2654 if (Value *Ret = optimizeUnaryDoubleFP(CI: Log, B, TLI, isPrecise: true))
2655 return Ret;
2656
2657 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2658
2659 // This is only applicable to log(), log2(), log10().
2660 LogLb = TLI->getLibFunc(funcName: LogNm);
2661 if (LogLb != NotLibFunc) {
2662 switch (LogLb) {
2663 case LibFunc_logf:
2664 LogID = Intrinsic::log;
2665 ExpLb = LibFunc_expf;
2666 Exp2Lb = LibFunc_exp2f;
2667 Exp10Lb = LibFunc_exp10f;
2668 PowLb = LibFunc_powf;
2669 break;
2670 case LibFunc_log:
2671 LogID = Intrinsic::log;
2672 ExpLb = LibFunc_exp;
2673 Exp2Lb = LibFunc_exp2;
2674 Exp10Lb = LibFunc_exp10;
2675 PowLb = LibFunc_pow;
2676 break;
2677 case LibFunc_logl:
2678 LogID = Intrinsic::log;
2679 ExpLb = LibFunc_expl;
2680 Exp2Lb = LibFunc_exp2l;
2681 Exp10Lb = LibFunc_exp10l;
2682 PowLb = LibFunc_powl;
2683 break;
2684 case LibFunc_log2f:
2685 LogID = Intrinsic::log2;
2686 ExpLb = LibFunc_expf;
2687 Exp2Lb = LibFunc_exp2f;
2688 Exp10Lb = LibFunc_exp10f;
2689 PowLb = LibFunc_powf;
2690 break;
2691 case LibFunc_log2:
2692 LogID = Intrinsic::log2;
2693 ExpLb = LibFunc_exp;
2694 Exp2Lb = LibFunc_exp2;
2695 Exp10Lb = LibFunc_exp10;
2696 PowLb = LibFunc_pow;
2697 break;
2698 case LibFunc_log2l:
2699 LogID = Intrinsic::log2;
2700 ExpLb = LibFunc_expl;
2701 Exp2Lb = LibFunc_exp2l;
2702 Exp10Lb = LibFunc_exp10l;
2703 PowLb = LibFunc_powl;
2704 break;
2705 case LibFunc_log10f:
2706 LogID = Intrinsic::log10;
2707 ExpLb = LibFunc_expf;
2708 Exp2Lb = LibFunc_exp2f;
2709 Exp10Lb = LibFunc_exp10f;
2710 PowLb = LibFunc_powf;
2711 break;
2712 case LibFunc_log10:
2713 LogID = Intrinsic::log10;
2714 ExpLb = LibFunc_exp;
2715 Exp2Lb = LibFunc_exp2;
2716 Exp10Lb = LibFunc_exp10;
2717 PowLb = LibFunc_pow;
2718 break;
2719 case LibFunc_log10l:
2720 LogID = Intrinsic::log10;
2721 ExpLb = LibFunc_expl;
2722 Exp2Lb = LibFunc_exp2l;
2723 Exp10Lb = LibFunc_exp10l;
2724 PowLb = LibFunc_powl;
2725 break;
2726 default:
2727 return nullptr;
2728 }
2729
2730 // Convert libcall to intrinsic if the value is known > 0.
2731 bool IsKnownNoErrno = Log->hasNoNaNs() && Log->hasNoInfs();
2732 if (!IsKnownNoErrno) {
2733 SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
2734 KnownFPClass Known = computeKnownFPClass(
2735 V: Log->getOperand(i_nocapture: 0),
2736 InterestedClasses: KnownFPClass::OrderedLessThanZeroMask | fcSubnormal, SQ);
2737 Function *F = Log->getParent()->getParent();
2738 const fltSemantics &FltSem = Ty->getScalarType()->getFltSemantics();
2739 IsKnownNoErrno =
2740 Known.cannotBeOrderedLessThanZero() &&
2741 Known.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
2742 }
2743 if (IsKnownNoErrno) {
2744 Value *NewLog = B.CreateUnaryIntrinsic(ID: LogID, Op: Log->getArgOperand(i: 0), FMFSource: Log);
2745 if (auto *I = dyn_cast<Instruction>(Val: NewLog)) {
2746 I->copyMetadata(SrcInst: *Log);
2747 return copyFlags(Old: *Log, New: I);
2748 }
2749 return NewLog;
2750 }
2751 } else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2752 LogID == Intrinsic::log10) {
2753 if (Ty->getScalarType()->isFloatTy()) {
2754 ExpLb = LibFunc_expf;
2755 Exp2Lb = LibFunc_exp2f;
2756 Exp10Lb = LibFunc_exp10f;
2757 PowLb = LibFunc_powf;
2758 } else if (Ty->getScalarType()->isDoubleTy()) {
2759 ExpLb = LibFunc_exp;
2760 Exp2Lb = LibFunc_exp2;
2761 Exp10Lb = LibFunc_exp10;
2762 PowLb = LibFunc_pow;
2763 } else
2764 return nullptr;
2765 } else
2766 return nullptr;
2767
2768 // The earlier call must also be 'fast' in order to do these transforms.
2769 CallInst *Arg = dyn_cast<CallInst>(Val: Log->getArgOperand(i: 0));
2770 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2771 return nullptr;
2772
2773 IRBuilderBase::FastMathFlagGuard Guard(B);
2774 B.setFastMathFlags(FastMathFlags::getFast());
2775
2776 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2777 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2778
2779 // log(pow(x,y)) -> y*log(x)
2780 AttributeList NoAttrs;
2781 if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2782 Value *LogX =
2783 Log->doesNotAccessMemory()
2784 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Arg->getOperand(i_nocapture: 0), FMFSource: nullptr, Name: "log")
2785 : emitUnaryFloatFnCall(Op: Arg->getOperand(i_nocapture: 0), TLI, Name: LogNm, B, Attrs: NoAttrs);
2786 Value *Y = Arg->getArgOperand(i: 1);
2787 // Cast exponent to FP if integer.
2788 if (ArgID == Intrinsic::powi)
2789 Y = B.CreateSIToFP(V: Y, DestTy: Ty, Name: "cast");
2790 Value *MulY = B.CreateFMul(L: Y, R: LogX, Name: "mul");
2791 // Since pow() may have side effects, e.g. errno,
2792 // dead code elimination may not be trusted to remove it.
2793 substituteInParent(I: Arg, With: MulY);
2794 return MulY;
2795 }
2796
2797 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2798 // TODO: There is no exp10() intrinsic yet.
2799 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2800 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
2801 Constant *Eul;
2802 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2803 // FIXME: Add more precise value of e for long double.
2804 Eul = ConstantFP::get(Ty: Log->getType(), V: numbers::e);
2805 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2806 Eul = ConstantFP::get(Ty: Log->getType(), V: 2.0);
2807 else
2808 Eul = ConstantFP::get(Ty: Log->getType(), V: 10.0);
2809 Value *LogE = Log->doesNotAccessMemory()
2810 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Eul, FMFSource: nullptr, Name: "log")
2811 : emitUnaryFloatFnCall(Op: Eul, TLI, Name: LogNm, B, Attrs: NoAttrs);
2812 Value *MulY = B.CreateFMul(L: Arg->getArgOperand(i: 0), R: LogE, Name: "mul");
2813 // Since exp() may have side effects, e.g. errno,
2814 // dead code elimination may not be trusted to remove it.
2815 substituteInParent(I: Arg, With: MulY);
2816 return MulY;
2817 }
2818
2819 return nullptr;
2820}
2821
2822// sqrt(exp(X)) -> exp(X * 0.5)
2823Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2824 if (!CI->hasAllowReassoc())
2825 return nullptr;
2826
2827 Function *SqrtFn = CI->getCalledFunction();
2828 CallInst *Arg = dyn_cast<CallInst>(Val: CI->getArgOperand(i: 0));
2829 if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2830 return nullptr;
2831 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2832 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2833
2834 LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2835
2836 SqrtLb = TLI->getLibFunc(funcName: SqrtFn->getName());
2837 if (SqrtLb != NotLibFunc)
2838 switch (SqrtLb) {
2839 case LibFunc_sqrtf:
2840 ExpLb = LibFunc_expf;
2841 Exp2Lb = LibFunc_exp2f;
2842 Exp10Lb = LibFunc_exp10f;
2843 break;
2844 case LibFunc_sqrt:
2845 ExpLb = LibFunc_exp;
2846 Exp2Lb = LibFunc_exp2;
2847 Exp10Lb = LibFunc_exp10;
2848 break;
2849 case LibFunc_sqrtl:
2850 ExpLb = LibFunc_expl;
2851 Exp2Lb = LibFunc_exp2l;
2852 Exp10Lb = LibFunc_exp10l;
2853 break;
2854 default:
2855 return nullptr;
2856 }
2857 else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2858 if (CI->getType()->getScalarType()->isFloatTy()) {
2859 ExpLb = LibFunc_expf;
2860 Exp2Lb = LibFunc_exp2f;
2861 Exp10Lb = LibFunc_exp10f;
2862 } else if (CI->getType()->getScalarType()->isDoubleTy()) {
2863 ExpLb = LibFunc_exp;
2864 Exp2Lb = LibFunc_exp2;
2865 Exp10Lb = LibFunc_exp10;
2866 } else
2867 return nullptr;
2868 } else
2869 return nullptr;
2870
2871 if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2872 ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2873 return nullptr;
2874
2875 IRBuilderBase::InsertPointGuard Guard(B);
2876 B.SetInsertPoint(Arg);
2877 auto *ExpOperand = Arg->getOperand(i_nocapture: 0);
2878 auto *FMul =
2879 B.CreateFMulFMF(L: ExpOperand, R: ConstantFP::get(Ty: ExpOperand->getType(), V: 0.5),
2880 FMFSource: CI, Name: "merged.sqrt");
2881
2882 Arg->setOperand(i_nocapture: 0, Val_nocapture: FMul);
2883 return Arg;
2884}
2885
2886Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2887 Module *M = CI->getModule();
2888 Function *Callee = CI->getCalledFunction();
2889 Value *Ret = nullptr;
2890 // TODO: Once we have a way (other than checking for the existince of the
2891 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2892 // condition below.
2893 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_sqrtf) &&
2894 (Callee->getName() == "sqrt" ||
2895 Callee->getIntrinsicID() == Intrinsic::sqrt))
2896 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2897
2898 if (Value *Opt = mergeSqrtToExp(CI, B))
2899 return Opt;
2900
2901 if (!CI->isFast())
2902 return Ret;
2903
2904 Instruction *I = dyn_cast<Instruction>(Val: CI->getArgOperand(i: 0));
2905 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2906 return Ret;
2907
2908 // We're looking for a repeated factor in a multiplication tree,
2909 // so we can do this fold: sqrt(x * x) -> fabs(x);
2910 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2911 Value *Op0 = I->getOperand(i: 0);
2912 Value *Op1 = I->getOperand(i: 1);
2913 Value *RepeatOp = nullptr;
2914 Value *OtherOp = nullptr;
2915 if (Op0 == Op1) {
2916 // Simple match: the operands of the multiply are identical.
2917 RepeatOp = Op0;
2918 } else {
2919 // Look for a more complicated pattern: one of the operands is itself
2920 // a multiply, so search for a common factor in that multiply.
2921 // Note: We don't bother looking any deeper than this first level or for
2922 // variations of this pattern because instcombine's visitFMUL and/or the
2923 // reassociation pass should give us this form.
2924 Value *MulOp;
2925 if (match(V: Op0, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2926 cast<Instruction>(Val: Op0)->isFast()) {
2927 // Pattern: sqrt((x * x) * z)
2928 RepeatOp = MulOp;
2929 OtherOp = Op1;
2930 } else if (match(V: Op1, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2931 cast<Instruction>(Val: Op1)->isFast()) {
2932 // Pattern: sqrt(z * (x * x))
2933 RepeatOp = MulOp;
2934 OtherOp = Op0;
2935 }
2936 }
2937 if (!RepeatOp)
2938 return Ret;
2939
2940 // Fast math flags for any created instructions should match the sqrt
2941 // and multiply.
2942
2943 // If we found a repeated factor, hoist it out of the square root and
2944 // replace it with the fabs of that factor.
2945 Value *FabsCall = B.CreateFAbs(V: RepeatOp, FMFSource: I, Name: "fabs");
2946 if (OtherOp) {
2947 // If we found a non-repeated factor, we still need to get its square
2948 // root. We then multiply that by the value that was simplified out
2949 // of the square root calculation.
2950 Value *SqrtCall =
2951 B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: OtherOp, FMFSource: I, Name: "sqrt");
2952 return copyFlags(Old: *CI, New: B.CreateFMulFMF(L: FabsCall, R: SqrtCall, FMFSource: I));
2953 }
2954 return copyFlags(Old: *CI, New: FabsCall);
2955}
2956
2957Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
2958
2959 // fmod(x,y) sets errno if y == 0 or x == +/-inf. frem does not set errno,
2960 // so the fold is valid only when we can prove fmod wouldn't either.
2961 bool IsNoErrno = CI->hasNoNaNs();
2962 if (!IsNoErrno) {
2963 SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
2964 KnownFPClass Known0 = computeKnownFPClass(V: CI->getOperand(i_nocapture: 0), InterestedClasses: fcInf, SQ);
2965 if (Known0.isKnownNeverInfinity()) {
2966 KnownFPClass Known1 =
2967 computeKnownFPClass(V: CI->getOperand(i_nocapture: 1), InterestedClasses: fcZero | fcSubnormal, SQ);
2968 Function *F = CI->getParent()->getParent();
2969 const fltSemantics &FltSem =
2970 CI->getType()->getScalarType()->getFltSemantics();
2971 IsNoErrno = Known1.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
2972 }
2973 }
2974
2975 if (IsNoErrno)
2976 return B.CreateFRemFMF(L: CI->getOperand(i_nocapture: 0), R: CI->getOperand(i_nocapture: 1), FMFSource: CI);
2977 return nullptr;
2978}
2979
2980Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
2981 IRBuilderBase &B) {
2982 Module *M = CI->getModule();
2983 Function *Callee = CI->getCalledFunction();
2984 Value *Ret = nullptr;
2985 StringRef Name = Callee->getName();
2986 if (UnsafeFPShrink &&
2987 (Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
2988 Name == "asinh") &&
2989 hasFloatVersion(M, FuncName: Name))
2990 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2991
2992 Value *Op1 = CI->getArgOperand(i: 0);
2993 auto *OpC = dyn_cast<CallInst>(Val: Op1);
2994 if (!OpC)
2995 return Ret;
2996
2997 // Both calls must be 'fast' in order to remove them.
2998 if (!CI->isFast() || !OpC->isFast())
2999 return Ret;
3000
3001 // tan(atan(x)) -> x
3002 // atanh(tanh(x)) -> x
3003 // sinh(asinh(x)) -> x
3004 // asinh(sinh(x)) -> x
3005 // cosh(acosh(x)) -> x
3006 Function *F = OpC->getCalledFunction();
3007 LibFunc Func = F ? TLI->getLibFunc(funcName: F->getName()) : NotLibFunc;
3008 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3009 LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
3010 .Case(S: "tan", Value: LibFunc_atan)
3011 .Case(S: "atanh", Value: LibFunc_tanh)
3012 .Case(S: "sinh", Value: LibFunc_asinh)
3013 .Case(S: "cosh", Value: LibFunc_acosh)
3014 .Case(S: "tanf", Value: LibFunc_atanf)
3015 .Case(S: "atanhf", Value: LibFunc_tanhf)
3016 .Case(S: "sinhf", Value: LibFunc_asinhf)
3017 .Case(S: "coshf", Value: LibFunc_acoshf)
3018 .Case(S: "tanl", Value: LibFunc_atanl)
3019 .Case(S: "atanhl", Value: LibFunc_tanhl)
3020 .Case(S: "sinhl", Value: LibFunc_asinhl)
3021 .Case(S: "coshl", Value: LibFunc_acoshl)
3022 .Case(S: "asinh", Value: LibFunc_sinh)
3023 .Case(S: "asinhf", Value: LibFunc_sinhf)
3024 .Case(S: "asinhl", Value: LibFunc_sinhl)
3025 .Default(Value: NotLibFunc); // Used as error value
3026 if (Func == inverseFunc)
3027 Ret = OpC->getArgOperand(i: 0);
3028 }
3029 return Ret;
3030}
3031
3032static bool isTrigLibCall(CallInst *CI) {
3033 // We can only hope to do anything useful if we can ignore things like errno
3034 // and floating-point exceptions.
3035 // We already checked the prototype.
3036 return CI->doesNotThrow() && CI->doesNotAccessMemory();
3037}
3038
3039static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
3040 bool UseFloat, Value *&Sin, Value *&Cos,
3041 Value *&SinCos, const TargetLibraryInfo *TLI) {
3042 Module *M = OrigCallee->getParent();
3043 Type *ArgTy = Arg->getType();
3044 Type *ResTy;
3045 StringRef Name;
3046
3047 Triple T(OrigCallee->getParent()->getTargetTriple());
3048 if (UseFloat) {
3049 Name = "__sincospif_stret";
3050
3051 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
3052 // x86_64 can't use {float, float} since that would be returned in both
3053 // xmm0 and xmm1, which isn't what a real struct would do.
3054 ResTy = T.getArch() == Triple::x86_64
3055 ? static_cast<Type *>(FixedVectorType::get(ElementType: ArgTy, NumElts: 2))
3056 : static_cast<Type *>(StructType::get(elt1: ArgTy, elts: ArgTy));
3057 } else {
3058 Name = "__sincospi_stret";
3059 ResTy = StructType::get(elt1: ArgTy, elts: ArgTy);
3060 }
3061
3062 if (!isLibFuncEmittable(M, TLI, Name))
3063 return false;
3064 LibFunc TheLibFunc = TLI->getLibFunc(funcName: Name);
3065 FunctionCallee Callee = getOrInsertLibFunc(
3066 M, TLI: *TLI, TheLibFunc, AttributeList: OrigCallee->getAttributes(), RetTy: ResTy, Args: ArgTy);
3067
3068 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
3069 // If the argument is an instruction, it must dominate all uses so put our
3070 // sincos call there.
3071 B.SetInsertPoint(TheBB: ArgInst->getParent(), IP: ++ArgInst->getIterator());
3072 } else {
3073 // Otherwise (e.g. for a constant) the beginning of the function is as
3074 // good a place as any.
3075 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
3076 B.SetInsertPoint(TheBB: &EntryBB, IP: EntryBB.begin());
3077 }
3078
3079 SinCos = B.CreateCall(Callee, Args: Arg, Name: "sincospi");
3080
3081 if (SinCos->getType()->isStructTy()) {
3082 Sin = B.CreateExtractValue(Agg: SinCos, Idxs: 0, Name: "sinpi");
3083 Cos = B.CreateExtractValue(Agg: SinCos, Idxs: 1, Name: "cospi");
3084 } else {
3085 Sin = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{0}, Name: "sinpi");
3086 Cos = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{1}, Name: "cospi");
3087 }
3088
3089 return true;
3090}
3091
3092static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
3093 IRBuilderBase &B) {
3094 Value *X;
3095 Value *Src = CI->getArgOperand(i: 0);
3096
3097 if (match(V: Src, P: m_OneUse(SubPattern: m_FNeg(X: m_Value(V&: X))))) {
3098 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3099 auto *CallInst = copyFlags(Old: *CI, New: Call);
3100 if (IsEven) {
3101 // Even function: f(-x) = f(x)
3102 return CallInst;
3103 }
3104 // Odd function: f(-x) = -f(x)
3105 return B.CreateFNegFMF(V: CallInst, FMFSource: CI);
3106 }
3107
3108 // Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
3109 if (IsEven && (match(V: Src, P: m_FAbs(Op0: m_Value(V&: X))) ||
3110 match(V: Src, P: m_CopySign(Op0: m_Value(V&: X), Op1: m_Value())))) {
3111 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3112 return copyFlags(Old: *CI, New: Call);
3113 }
3114
3115 return nullptr;
3116}
3117
3118Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
3119 IRBuilderBase &B) {
3120 switch (Func) {
3121 case LibFunc_cos:
3122 case LibFunc_cosf:
3123 case LibFunc_cosl:
3124
3125 case LibFunc_cosh:
3126 case LibFunc_coshf:
3127 case LibFunc_coshl:
3128 return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
3129
3130 case LibFunc_sin:
3131 case LibFunc_sinf:
3132 case LibFunc_sinl:
3133
3134 case LibFunc_sinh:
3135 case LibFunc_sinhf:
3136 case LibFunc_sinhl:
3137
3138 case LibFunc_tan:
3139 case LibFunc_tanf:
3140 case LibFunc_tanl:
3141
3142 case LibFunc_tanh:
3143 case LibFunc_tanhf:
3144 case LibFunc_tanhl:
3145
3146 case LibFunc_erf:
3147 case LibFunc_erff:
3148 case LibFunc_erfl:
3149 return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
3150
3151 default:
3152 return nullptr;
3153 }
3154}
3155
3156Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
3157 // Make sure the prototype is as expected, otherwise the rest of the
3158 // function is probably invalid and likely to abort.
3159 if (!isTrigLibCall(CI))
3160 return nullptr;
3161
3162 Value *Arg = CI->getArgOperand(i: 0);
3163 if (isa<ConstantData>(Val: Arg))
3164 return nullptr;
3165
3166 SmallVector<CallInst *, 1> SinCalls;
3167 SmallVector<CallInst *, 1> CosCalls;
3168 SmallVector<CallInst *, 1> SinCosCalls;
3169
3170 bool IsFloat = Arg->getType()->isFloatTy();
3171
3172 // Look for all compatible sinpi, cospi and sincospi calls with the same
3173 // argument. If there are enough (in some sense) we can make the
3174 // substitution.
3175 Function *F = CI->getFunction();
3176 for (User *U : Arg->users())
3177 classifyArgUse(Val: U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
3178
3179 // It's only worthwhile if both sinpi and cospi are actually used.
3180 if (SinCalls.empty() || CosCalls.empty())
3181 return nullptr;
3182
3183 Value *Sin, *Cos, *SinCos;
3184 if (!insertSinCosCall(B, OrigCallee: CI->getCalledFunction(), Arg, UseFloat: IsFloat, Sin, Cos,
3185 SinCos, TLI))
3186 return nullptr;
3187
3188 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
3189 Value *Res) {
3190 for (CallInst *C : Calls)
3191 replaceAllUsesWith(I: C, With: Res);
3192 };
3193
3194 replaceTrigInsts(SinCalls, Sin);
3195 replaceTrigInsts(CosCalls, Cos);
3196 replaceTrigInsts(SinCosCalls, SinCos);
3197
3198 return IsSin ? Sin : Cos;
3199}
3200
3201void LibCallSimplifier::classifyArgUse(
3202 Value *Val, Function *F, bool IsFloat,
3203 SmallVectorImpl<CallInst *> &SinCalls,
3204 SmallVectorImpl<CallInst *> &CosCalls,
3205 SmallVectorImpl<CallInst *> &SinCosCalls) {
3206 auto *CI = dyn_cast<CallInst>(Val);
3207 if (!CI || CI->use_empty())
3208 return;
3209
3210 // Don't consider calls in other functions.
3211 if (CI->getFunction() != F)
3212 return;
3213
3214 Module *M = CI->getModule();
3215 Function *Callee = CI->getCalledFunction();
3216 LibFunc Func = Callee ? TLI->getLibFunc(FDecl: *Callee) : NotLibFunc;
3217 if (!isLibFuncEmittable(M, TLI, TheLibFunc: Func) || !isTrigLibCall(CI))
3218 return;
3219
3220 if (IsFloat) {
3221 if (Func == LibFunc_sinpif)
3222 SinCalls.push_back(Elt: CI);
3223 else if (Func == LibFunc_cospif)
3224 CosCalls.push_back(Elt: CI);
3225 else if (Func == LibFunc_sincospif_stret)
3226 SinCosCalls.push_back(Elt: CI);
3227 } else {
3228 if (Func == LibFunc_sinpi)
3229 SinCalls.push_back(Elt: CI);
3230 else if (Func == LibFunc_cospi)
3231 CosCalls.push_back(Elt: CI);
3232 else if (Func == LibFunc_sincospi_stret)
3233 SinCosCalls.push_back(Elt: CI);
3234 }
3235}
3236
3237/// Constant folds remquo
3238Value *LibCallSimplifier::optimizeRemquo(CallInst *CI, IRBuilderBase &B) {
3239 const APFloat *X, *Y;
3240 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3241 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3242 return nullptr;
3243
3244 APFloat::opStatus Status;
3245 APFloat Quot = *X;
3246 Status = Quot.divide(RHS: *Y, RM: APFloat::rmNearestTiesToEven);
3247 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3248 return nullptr;
3249 APFloat Rem = *X;
3250 if (Rem.remainder(RHS: *Y) != APFloat::opOK)
3251 return nullptr;
3252
3253 // TODO: We can only keep at least the three of the last bits of x/y
3254 unsigned IntBW = TLI->getIntSize();
3255 APSInt QuotInt(IntBW, /*isUnsigned=*/false);
3256 bool IsExact;
3257 Status =
3258 Quot.convertToInteger(Result&: QuotInt, RM: APFloat::rmNearestTiesToEven, IsExact: &IsExact);
3259 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3260 return nullptr;
3261
3262 B.CreateAlignedStore(
3263 Val: ConstantInt::getSigned(Ty: B.getIntNTy(N: IntBW), V: QuotInt.getExtValue()),
3264 Ptr: CI->getArgOperand(i: 2), Align: CI->getParamAlign(ArgNo: 2));
3265 return ConstantFP::get(Ty: CI->getType(), V: Rem);
3266}
3267
3268/// Constant folds fdim
3269Value *LibCallSimplifier::optimizeFdim(CallInst *CI, IRBuilderBase &B) {
3270 // Cannot perform the fold unless the call has attribute memory(none)
3271 if (!CI->doesNotAccessMemory())
3272 return nullptr;
3273
3274 // TODO : Handle undef values
3275 // Propagate poison if any
3276 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 0)))
3277 return CI->getArgOperand(i: 0);
3278 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 1)))
3279 return CI->getArgOperand(i: 1);
3280
3281 const APFloat *X, *Y;
3282 // Check if both values are constants
3283 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3284 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3285 return nullptr;
3286
3287 // C99 fdim(x, y) = (x > y) ? x - y : +0.
3288 if (X->compare(RHS: *Y) != APFloat::cmpGreaterThan && !X->isNaN() && !Y->isNaN())
3289 return ConstantFP::getZero(Ty: CI->getType());
3290 APFloat Difference = *X;
3291 Difference.subtract(RHS: *Y, RM: RoundingMode::NearestTiesToEven);
3292 return ConstantFP::get(Ty: CI->getType(), V: Difference);
3293}
3294
3295//===----------------------------------------------------------------------===//
3296// Integer Library Call Optimizations
3297//===----------------------------------------------------------------------===//
3298
3299Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
3300 // All variants of ffs return int which need not be 32 bits wide.
3301 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
3302 Type *RetType = CI->getType();
3303 Value *Op = CI->getArgOperand(i: 0);
3304 Type *ArgType = Op->getType();
3305 Value *V = B.CreateIntrinsic(ID: Intrinsic::cttz, OverloadTypes: {ArgType}, Args: {Op, B.getTrue()},
3306 FMFSource: nullptr, Name: "cttz");
3307 V = B.CreateAdd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 1));
3308 V = B.CreateIntCast(V, DestTy: RetType, isSigned: false);
3309
3310 Value *Cond = B.CreateICmpNE(LHS: Op, RHS: Constant::getNullValue(Ty: ArgType));
3311 return B.CreateSelect(C: Cond, True: V, False: ConstantInt::get(Ty: RetType, V: 0));
3312}
3313
3314Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
3315 // All variants of fls return int which need not be 32 bits wide.
3316 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
3317 Value *Op = CI->getArgOperand(i: 0);
3318 Type *ArgType = Op->getType();
3319 Value *V = B.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {ArgType}, Args: {Op, B.getFalse()},
3320 FMFSource: nullptr, Name: "ctlz");
3321 V = B.CreateSub(LHS: ConstantInt::get(Ty: V->getType(), V: ArgType->getIntegerBitWidth()),
3322 RHS: V);
3323 return B.CreateIntCast(V, DestTy: CI->getType(), isSigned: false);
3324}
3325
3326Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
3327 // abs(x) -> x <s 0 ? -x : x
3328 // The negation has 'nsw' because abs of INT_MIN is undefined.
3329 Value *X = CI->getArgOperand(i: 0);
3330 Value *IsNeg = B.CreateIsNeg(Arg: X);
3331 Value *NegX = B.CreateNSWNeg(V: X, Name: "neg");
3332 return B.CreateSelect(C: IsNeg, True: NegX, False: X);
3333}
3334
3335Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
3336 // isdigit(c) -> (c-'0') <u 10
3337 Value *Op = CI->getArgOperand(i: 0);
3338 Type *ArgType = Op->getType();
3339 Op = B.CreateSub(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: '0'), Name: "isdigittmp");
3340 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 10), Name: "isdigit");
3341 return B.CreateZExt(V: Op, DestTy: CI->getType());
3342}
3343
3344Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
3345 // isascii(c) -> c <u 128
3346 Value *Op = CI->getArgOperand(i: 0);
3347 Type *ArgType = Op->getType();
3348 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 128), Name: "isascii");
3349 return B.CreateZExt(V: Op, DestTy: CI->getType());
3350}
3351
3352Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
3353 // toascii(c) -> c & 0x7f
3354 return B.CreateAnd(LHS: CI->getArgOperand(i: 0),
3355 RHS: ConstantInt::get(Ty: CI->getType(), V: 0x7F));
3356}
3357
3358// Fold calls to atoi, atol, and atoll.
3359Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
3360 StringRef Str;
3361 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3362 return nullptr;
3363
3364 return convertStrToInt(CI, Str, EndPtr: nullptr, Base: 10, /*AsSigned=*/true, B);
3365}
3366
3367// Fold calls to strtol, strtoll, strtoul, and strtoull.
3368Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
3369 bool AsSigned) {
3370 Value *EndPtr = CI->getArgOperand(i: 1);
3371 if (isa<ConstantPointerNull>(Val: EndPtr)) {
3372 // With a null EndPtr, this function won't capture the main argument.
3373 // It would be readonly too, except that it still may write to errno.
3374 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
3375 CI: CaptureInfo::none()));
3376 EndPtr = nullptr;
3377 } else if (!isKnownNonZero(V: EndPtr, Q: DL))
3378 return nullptr;
3379
3380 StringRef Str;
3381 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3382 return nullptr;
3383
3384 if (ConstantInt *CInt = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2))) {
3385 return convertStrToInt(CI, Str, EndPtr, Base: CInt->getSExtValue(), AsSigned, B);
3386 }
3387
3388 return nullptr;
3389}
3390
3391//===----------------------------------------------------------------------===//
3392// Formatting and IO Library Call Optimizations
3393//===----------------------------------------------------------------------===//
3394
3395static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
3396
3397Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3398 int StreamArg) {
3399 Function *Callee = CI->getCalledFunction();
3400 // Error reporting calls should be cold, mark them as such.
3401 // This applies even to non-builtin calls: it is only a hint and applies to
3402 // functions that the frontend might not understand as builtins.
3403
3404 // This heuristic was suggested in:
3405 // Improving Static Branch Prediction in a Compiler
3406 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3407 // Proceedings of PACT'98, Oct. 1998, IEEE
3408 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
3409 isReportingError(Callee, CI, StreamArg)) {
3410 CI->addFnAttr(Kind: Attribute::Cold);
3411 }
3412
3413 return nullptr;
3414}
3415
3416static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3417 if (!Callee || !Callee->isDeclaration())
3418 return false;
3419
3420 if (StreamArg < 0)
3421 return true;
3422
3423 // These functions might be considered cold, but only if their stream
3424 // argument is stderr.
3425
3426 if (StreamArg >= (int)CI->arg_size())
3427 return false;
3428 LoadInst *LI = dyn_cast<LoadInst>(Val: CI->getArgOperand(i: StreamArg));
3429 if (!LI)
3430 return false;
3431 GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: LI->getPointerOperand());
3432 if (!GV || !GV->isDeclaration())
3433 return false;
3434 return GV->getName() == "stderr";
3435}
3436
3437Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3438 // Check for a fixed format string.
3439 StringRef FormatStr;
3440 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: FormatStr))
3441 return nullptr;
3442
3443 // Empty format string -> noop.
3444 if (FormatStr.empty()) // Tolerate printf's declared void.
3445 return CI->use_empty() ? (Value *)CI : ConstantInt::get(Ty: CI->getType(), V: 0);
3446
3447 // Do not do any of the following transformations if the printf return value
3448 // is used, in general the printf return value is not compatible with either
3449 // putchar() or puts().
3450 if (!CI->use_empty())
3451 return nullptr;
3452
3453 Type *IntTy = CI->getType();
3454 // printf("x") -> putchar('x'), even for "%" and "%%".
3455 if (FormatStr.size() == 1 || FormatStr == "%%") {
3456 // Convert the character to unsigned char before passing it to putchar
3457 // to avoid host-specific sign extension in the IR. Putchar converts
3458 // it to unsigned char regardless.
3459 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)FormatStr[0]);
3460 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3461 }
3462
3463 // Try to remove call or emit putchar/puts.
3464 if (FormatStr == "%s" && CI->arg_size() > 1) {
3465 StringRef OperandStr;
3466 if (!getConstantStringInfo(V: CI->getOperand(i_nocapture: 1), Str&: OperandStr))
3467 return nullptr;
3468 // printf("%s", "") --> NOP
3469 if (OperandStr.empty())
3470 return (Value *)CI;
3471 // printf("%s", "a") --> putchar('a')
3472 if (OperandStr.size() == 1) {
3473 // Convert the character to unsigned char before passing it to putchar
3474 // to avoid host-specific sign extension in the IR. Putchar converts
3475 // it to unsigned char regardless.
3476 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)OperandStr[0]);
3477 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3478 }
3479 // printf("%s", str"\n") --> puts(str)
3480 if (OperandStr.back() == '\n') {
3481 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3482 return nullptr;
3483 OperandStr = OperandStr.drop_back();
3484 Value *GV = B.CreateGlobalString(Str: OperandStr, Name: "str");
3485 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3486 }
3487 return nullptr;
3488 }
3489
3490 // printf("foo\n") --> puts("foo")
3491 if (FormatStr.back() == '\n' &&
3492 !FormatStr.contains(C: '%')) { // No format characters.
3493 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3494 return nullptr;
3495 // Create a string literal with no \n on it. We expect the constant merge
3496 // pass to be run after this pass, to merge duplicate strings.
3497 FormatStr = FormatStr.drop_back();
3498 Value *GV = B.CreateGlobalString(Str: FormatStr, Name: "str");
3499 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3500 }
3501
3502 // Optimize specific format strings.
3503 // printf("%c", chr) --> putchar(chr)
3504 if (FormatStr == "%c" && CI->arg_size() > 1 &&
3505 CI->getArgOperand(i: 1)->getType()->isIntegerTy()) {
3506 // Convert the argument to the type expected by putchar, i.e., int, which
3507 // need not be 32 bits wide but which is the same as printf's return type.
3508 Value *IntChar = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: IntTy, isSigned: false);
3509 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3510 }
3511
3512 // printf("%s\n", str) --> puts(str)
3513 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3514 CI->getArgOperand(i: 1)->getType()->isPointerTy())
3515 return copyFlags(Old: *CI, New: emitPutS(Str: CI->getArgOperand(i: 1), B, TLI));
3516 return nullptr;
3517}
3518
3519Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3520
3521 Module *M = CI->getModule();
3522 Function *Callee = CI->getCalledFunction();
3523 FunctionType *FT = Callee->getFunctionType();
3524 if (Value *V = optimizePrintFString(CI, B)) {
3525 return V;
3526 }
3527
3528 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3529
3530 // printf(format, ...) -> iprintf(format, ...) if no floating point
3531 // arguments.
3532 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_iprintf) &&
3533 !callHasFloatingPointArgument(CI)) {
3534 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_iprintf, T: FT,
3535 AttributeList: Callee->getAttributes());
3536 CallInst *New = cast<CallInst>(Val: CI->clone());
3537 New->setCalledFunction(IPrintFFn);
3538 B.Insert(I: New);
3539 return New;
3540 }
3541
3542 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3543 // arguments.
3544 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_printf) &&
3545 !callHasFP128Argument(CI)) {
3546 auto SmallPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_printf, T: FT,
3547 AttributeList: Callee->getAttributes());
3548 CallInst *New = cast<CallInst>(Val: CI->clone());
3549 New->setCalledFunction(SmallPrintFFn);
3550 B.Insert(I: New);
3551 return New;
3552 }
3553
3554 return nullptr;
3555}
3556
3557Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3558 IRBuilderBase &B) {
3559 // Check for a fixed format string.
3560 StringRef FormatStr;
3561 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3562 return nullptr;
3563
3564 // If we just have a format string (nothing else crazy) transform it.
3565 Value *Dest = CI->getArgOperand(i: 0);
3566 if (CI->arg_size() == 2) {
3567 // Make sure there's no % in the constant array. We could try to handle
3568 // %% -> % in the future if we cared.
3569 if (FormatStr.contains(C: '%'))
3570 return nullptr; // we found a format specifier, bail out.
3571
3572 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3573 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1),
3574 // Copy the null byte.
3575 Size: TLI->getAsSizeT(V: FormatStr.size() + 1, M: *CI->getModule()));
3576 return ConstantInt::get(Ty: CI->getType(), V: FormatStr.size());
3577 }
3578
3579 // The remaining optimizations require the format string to be "%s" or "%c"
3580 // and have an extra operand.
3581 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3582 return nullptr;
3583
3584 // Decode the second character of the format string.
3585 if (FormatStr[1] == 'c') {
3586 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3587 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3588 return nullptr;
3589 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 2), DestTy: B.getInt8Ty(), Name: "char");
3590 Value *Ptr = Dest;
3591 B.CreateStore(Val: V, Ptr);
3592 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3593 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3594
3595 return ConstantInt::get(Ty: CI->getType(), V: 1);
3596 }
3597
3598 if (FormatStr[1] == 's') {
3599 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3600 // strlen(str)+1)
3601 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3602 return nullptr;
3603
3604 if (CI->use_empty())
3605 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3606 return copyFlags(Old: *CI, New: emitStrCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI));
3607
3608 uint64_t SrcLen = GetStringLength(V: CI->getArgOperand(i: 2));
3609 if (SrcLen) {
3610 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1),
3611 Size: TLI->getAsSizeT(V: SrcLen, M: *CI->getModule()));
3612 // Returns total number of characters written without null-character.
3613 return ConstantInt::get(Ty: CI->getType(), V: SrcLen - 1);
3614 } else if (Value *V = emitStpCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI)) {
3615 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3616 Value *PtrDiff = B.CreatePtrDiff(LHS: V, RHS: Dest);
3617 return B.CreateIntCast(V: PtrDiff, DestTy: CI->getType(), isSigned: false);
3618 }
3619
3620 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3621 QueryType: PGSOQueryType::IRPass))
3622 return nullptr;
3623
3624 Value *Len = emitStrLen(Ptr: CI->getArgOperand(i: 2), B, DL, TLI);
3625 if (!Len)
3626 return nullptr;
3627 Value *IncLen =
3628 B.CreateAdd(LHS: Len, RHS: ConstantInt::get(Ty: Len->getType(), V: 1), Name: "leninc");
3629 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1), Size: IncLen);
3630
3631 // The sprintf result is the unincremented number of bytes in the string.
3632 return B.CreateIntCast(V: Len, DestTy: CI->getType(), isSigned: false);
3633 }
3634 return nullptr;
3635}
3636
3637Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3638 Module *M = CI->getModule();
3639 Function *Callee = CI->getCalledFunction();
3640 FunctionType *FT = Callee->getFunctionType();
3641 if (Value *V = optimizeSPrintFString(CI, B)) {
3642 return V;
3643 }
3644
3645 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
3646
3647 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3648 // point arguments.
3649 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_siprintf) &&
3650 !callHasFloatingPointArgument(CI)) {
3651 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_siprintf,
3652 T: FT, AttributeList: Callee->getAttributes());
3653 CallInst *New = cast<CallInst>(Val: CI->clone());
3654 New->setCalledFunction(SIPrintFFn);
3655 B.Insert(I: New);
3656 return New;
3657 }
3658
3659 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3660 // floating point arguments.
3661 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_sprintf) &&
3662 !callHasFP128Argument(CI)) {
3663 auto SmallSPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_sprintf, T: FT,
3664 AttributeList: Callee->getAttributes());
3665 CallInst *New = cast<CallInst>(Val: CI->clone());
3666 New->setCalledFunction(SmallSPrintFFn);
3667 B.Insert(I: New);
3668 return New;
3669 }
3670
3671 return nullptr;
3672}
3673
3674// Transform an snprintf call CI with the bound N to format the string Str
3675// either to a call to memcpy, or to single character a store, or to nothing,
3676// and fold the result to a constant. A nonnull StrArg refers to the string
3677// argument being formatted. Otherwise the call is one with N < 2 and
3678// the "%c" directive to format a single character.
3679Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3680 StringRef Str, uint64_t N,
3681 IRBuilderBase &B) {
3682 assert(StrArg || (N < 2 && Str.size() == 1));
3683
3684 unsigned IntBits = TLI->getIntSize();
3685 uint64_t IntMax = maxIntN(N: IntBits);
3686 if (Str.size() > IntMax)
3687 // Bail if the string is longer than INT_MAX. POSIX requires
3688 // implementations to set errno to EOVERFLOW in this case, in
3689 // addition to when N is larger than that (checked by the caller).
3690 return nullptr;
3691
3692 Value *StrLen = ConstantInt::get(Ty: CI->getType(), V: Str.size());
3693 if (N == 0)
3694 return StrLen;
3695
3696 // Set to the number of bytes to copy fron StrArg which is also
3697 // the offset of the terinating nul.
3698 uint64_t NCopy;
3699 if (N > Str.size())
3700 // Copy the full string, including the terminating nul (which must
3701 // be present regardless of the bound).
3702 NCopy = Str.size() + 1;
3703 else
3704 NCopy = N - 1;
3705
3706 Value *DstArg = CI->getArgOperand(i: 0);
3707 if (NCopy && StrArg)
3708 // Transform the call to lvm.memcpy(dst, fmt, N).
3709 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst: DstArg, DstAlign: Align(1), Src: StrArg, SrcAlign: Align(1),
3710 Size: TLI->getAsSizeT(V: NCopy, M: *CI->getModule())));
3711
3712 if (N > Str.size())
3713 // Return early when the whole format string, including the final nul,
3714 // has been copied.
3715 return StrLen;
3716
3717 // Otherwise, when truncating the string append a terminating nul.
3718 Type *Int8Ty = B.getInt8Ty();
3719 Value *NulOff = B.getIntN(N: IntBits, C: NCopy);
3720 Value *DstEnd = B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: DstArg, IdxList: NulOff, Name: "endptr");
3721 B.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: 0), Ptr: DstEnd);
3722 return StrLen;
3723}
3724
3725Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3726 IRBuilderBase &B) {
3727 // Check for size
3728 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3729 if (!Size)
3730 return nullptr;
3731
3732 uint64_t N = Size->getZExtValue();
3733 uint64_t IntMax = maxIntN(N: TLI->getIntSize());
3734 if (N > IntMax)
3735 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3736 // to set errno to EOVERFLOW in this case.
3737 return nullptr;
3738
3739 Value *DstArg = CI->getArgOperand(i: 0);
3740 Value *FmtArg = CI->getArgOperand(i: 2);
3741
3742 // Check for a fixed format string.
3743 StringRef FormatStr;
3744 if (!getConstantStringInfo(V: FmtArg, Str&: FormatStr))
3745 return nullptr;
3746
3747 // If we just have a format string (nothing else crazy) transform it.
3748 if (CI->arg_size() == 3) {
3749 if (FormatStr.contains(C: '%'))
3750 // Bail if the format string contains a directive and there are
3751 // no arguments. We could handle "%%" in the future.
3752 return nullptr;
3753
3754 return emitSnPrintfMemCpy(CI, StrArg: FmtArg, Str: FormatStr, N, B);
3755 }
3756
3757 // The remaining optimizations require the format string to be "%s" or "%c"
3758 // and have an extra operand.
3759 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3760 return nullptr;
3761
3762 // Decode the second character of the format string.
3763 if (FormatStr[1] == 'c') {
3764 if (N <= 1) {
3765 // Use an arbitary string of length 1 to transform the call into
3766 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3767 // to one.
3768 StringRef CharStr("*");
3769 return emitSnPrintfMemCpy(CI, StrArg: nullptr, Str: CharStr, N, B);
3770 }
3771
3772 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3773 if (!CI->getArgOperand(i: 3)->getType()->isIntegerTy())
3774 return nullptr;
3775 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 3), DestTy: B.getInt8Ty(), Name: "char");
3776 Value *Ptr = DstArg;
3777 B.CreateStore(Val: V, Ptr);
3778 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3779 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3780 return ConstantInt::get(Ty: CI->getType(), V: 1);
3781 }
3782
3783 if (FormatStr[1] != 's')
3784 return nullptr;
3785
3786 Value *StrArg = CI->getArgOperand(i: 3);
3787 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3788 StringRef Str;
3789 if (!getConstantStringInfo(V: StrArg, Str))
3790 return nullptr;
3791
3792 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3793}
3794
3795Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3796 if (Value *V = optimizeSnPrintFString(CI, B)) {
3797 return V;
3798 }
3799
3800 if (isKnownNonZero(V: CI->getOperand(i_nocapture: 1), Q: DL))
3801 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3802 return nullptr;
3803}
3804
3805Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3806 IRBuilderBase &B) {
3807 optimizeErrorReporting(CI, B, StreamArg: 0);
3808
3809 // All the optimizations depend on the format string.
3810 StringRef FormatStr;
3811 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3812 return nullptr;
3813
3814 // Do not do any of the following transformations if the fprintf return
3815 // value is used, in general the fprintf return value is not compatible
3816 // with fwrite(), fputc() or fputs().
3817 if (!CI->use_empty())
3818 return nullptr;
3819
3820 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3821 if (CI->arg_size() == 2) {
3822 // Could handle %% -> % if we cared.
3823 if (FormatStr.contains(C: '%'))
3824 return nullptr; // We found a format specifier.
3825
3826 return copyFlags(
3827 Old: *CI, New: emitFWrite(Ptr: CI->getArgOperand(i: 1),
3828 Size: TLI->getAsSizeT(V: FormatStr.size(), M: *CI->getModule()),
3829 File: CI->getArgOperand(i: 0), B, DL, TLI));
3830 }
3831
3832 // The remaining optimizations require the format string to be "%s" or "%c"
3833 // and have an extra operand.
3834 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3835 return nullptr;
3836
3837 // Decode the second character of the format string.
3838 if (FormatStr[1] == 'c') {
3839 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3840 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3841 return nullptr;
3842 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3843 Value *V = B.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntTy, /*isSigned*/ true,
3844 Name: "chari");
3845 return copyFlags(Old: *CI, New: emitFPutC(Char: V, File: CI->getArgOperand(i: 0), B, TLI));
3846 }
3847
3848 if (FormatStr[1] == 's') {
3849 // fprintf(F, "%s", str) --> fputs(str, F)
3850 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3851 return nullptr;
3852 return copyFlags(
3853 Old: *CI, New: emitFPutS(Str: CI->getArgOperand(i: 2), File: CI->getArgOperand(i: 0), B, TLI));
3854 }
3855 return nullptr;
3856}
3857
3858Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3859 Module *M = CI->getModule();
3860 Function *Callee = CI->getCalledFunction();
3861 FunctionType *FT = Callee->getFunctionType();
3862 if (Value *V = optimizeFPrintFString(CI, B)) {
3863 return V;
3864 }
3865
3866 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3867 // floating point arguments.
3868 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_fiprintf) &&
3869 !callHasFloatingPointArgument(CI)) {
3870 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_fiprintf,
3871 T: FT, AttributeList: Callee->getAttributes());
3872 CallInst *New = cast<CallInst>(Val: CI->clone());
3873 New->setCalledFunction(FIPrintFFn);
3874 B.Insert(I: New);
3875 return New;
3876 }
3877
3878 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3879 // 128-bit floating point arguments.
3880 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_fprintf) &&
3881 !callHasFP128Argument(CI)) {
3882 auto SmallFPrintFFn =
3883 getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_fprintf, T: FT,
3884 AttributeList: Callee->getAttributes());
3885 CallInst *New = cast<CallInst>(Val: CI->clone());
3886 New->setCalledFunction(SmallFPrintFFn);
3887 B.Insert(I: New);
3888 return New;
3889 }
3890
3891 return nullptr;
3892}
3893
3894Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3895 optimizeErrorReporting(CI, B, StreamArg: 3);
3896
3897 // Get the element size and count.
3898 ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3899 ConstantInt *CountC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
3900 if (SizeC && CountC) {
3901 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3902
3903 // If this is writing zero records, remove the call (it's a noop).
3904 if (Bytes == 0)
3905 return ConstantInt::get(Ty: CI->getType(), V: 0);
3906
3907 // If this is writing one byte, turn it into fputc.
3908 // This optimisation is only valid, if the return value is unused.
3909 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
3910 Value *Char = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0), Name: "char");
3911 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3912 Value *Cast = B.CreateIntCast(V: Char, DestTy: IntTy, /*isSigned*/ true, Name: "chari");
3913 Value *NewCI = emitFPutC(Char: Cast, File: CI->getArgOperand(i: 3), B, TLI);
3914 return NewCI ? ConstantInt::get(Ty: CI->getType(), V: 1) : nullptr;
3915 }
3916 }
3917
3918 return nullptr;
3919}
3920
3921Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
3922 optimizeErrorReporting(CI, B, StreamArg: 1);
3923
3924 // Don't rewrite fputs to fwrite when optimising for size because fwrite
3925 // requires more arguments and thus extra MOVs are required.
3926 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3927 QueryType: PGSOQueryType::IRPass))
3928 return nullptr;
3929
3930 // We can't optimize if return value is used.
3931 if (!CI->use_empty())
3932 return nullptr;
3933
3934 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
3935 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: 0));
3936 if (!Len)
3937 return nullptr;
3938
3939 // Known to have no uses (see above).
3940 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
3941 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
3942 return copyFlags(
3943 Old: *CI,
3944 New: emitFWrite(Ptr: CI->getArgOperand(i: 0),
3945 Size: ConstantInt::get(Ty: SizeTTy, V: Len - 1),
3946 File: CI->getArgOperand(i: 1), B, DL, TLI));
3947}
3948
3949Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
3950 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3951 if (!CI->use_empty())
3952 return nullptr;
3953
3954 // Check for a constant string.
3955 // puts("") -> putchar('\n')
3956 StringRef Str;
3957 if (getConstantStringInfo(V: CI->getArgOperand(i: 0), Str) && Str.empty()) {
3958 // putchar takes an argument of the same type as puts returns, i.e.,
3959 // int, which need not be 32 bits wide.
3960 Type *IntTy = CI->getType();
3961 return copyFlags(Old: *CI, New: emitPutChar(Char: ConstantInt::get(Ty: IntTy, V: '\n'), B, TLI));
3962 }
3963
3964 return nullptr;
3965}
3966
3967Value *LibCallSimplifier::optimizeExit(CallInst *CI) {
3968
3969 // Mark 'exit' as cold if its not exit(0) (success).
3970 const APInt *C;
3971 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
3972 match(V: CI->getArgOperand(i: 0), P: m_APInt(Res&: C)) && !C->isZero()) {
3973 CI->addFnAttr(Kind: Attribute::Cold);
3974 }
3975 return nullptr;
3976}
3977
3978Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
3979 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
3980 return copyFlags(Old: *CI, New: B.CreateMemMove(Dst: CI->getArgOperand(i: 1), DstAlign: Align(1),
3981 Src: CI->getArgOperand(i: 0), SrcAlign: Align(1),
3982 Size: CI->getArgOperand(i: 2)));
3983}
3984
3985bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
3986 SmallString<20> FloatFuncName = FuncName;
3987 FloatFuncName += 'f';
3988 return isLibFuncEmittable(M, TLI, Name: FloatFuncName);
3989}
3990
3991Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
3992 IRBuilderBase &Builder) {
3993 Module *M = CI->getModule();
3994 Function *Callee = CI->getCalledFunction();
3995 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
3996
3997 // Check for string/memory library functions.
3998 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3999 // Make sure we never change the calling convention.
4000 assert(
4001 (ignoreCallingConv(Func) ||
4002 TargetLibraryInfoImpl::isCallingConvCCompatible(CI)) &&
4003 "Optimizing string/memory libcall would change the calling convention");
4004 switch (Func) {
4005 case LibFunc_strcat:
4006 return optimizeStrCat(CI, B&: Builder);
4007 case LibFunc_strncat:
4008 return optimizeStrNCat(CI, B&: Builder);
4009 case LibFunc_strchr:
4010 return optimizeStrChr(CI, B&: Builder);
4011 case LibFunc_strrchr:
4012 return optimizeStrRChr(CI, B&: Builder);
4013 case LibFunc_strcmp:
4014 return optimizeStrCmp(CI, B&: Builder);
4015 case LibFunc_strncmp:
4016 return optimizeStrNCmp(CI, B&: Builder);
4017 case LibFunc_strcpy:
4018 return optimizeStrCpy(CI, B&: Builder);
4019 case LibFunc_stpcpy:
4020 return optimizeStpCpy(CI, B&: Builder);
4021 case LibFunc_strlcpy:
4022 return optimizeStrLCpy(CI, B&: Builder);
4023 case LibFunc_stpncpy:
4024 return optimizeStringNCpy(CI, /*RetEnd=*/true, B&: Builder);
4025 case LibFunc_strncpy:
4026 return optimizeStringNCpy(CI, /*RetEnd=*/false, B&: Builder);
4027 case LibFunc_strlen:
4028 return optimizeStrLen(CI, B&: Builder);
4029 case LibFunc_strnlen:
4030 return optimizeStrNLen(CI, B&: Builder);
4031 case LibFunc_strpbrk:
4032 return optimizeStrPBrk(CI, B&: Builder);
4033 case LibFunc_strndup:
4034 return optimizeStrNDup(CI, B&: Builder);
4035 case LibFunc_strtol:
4036 case LibFunc_strtod:
4037 case LibFunc_strtof:
4038 case LibFunc_strtoul:
4039 case LibFunc_strtoll:
4040 case LibFunc_strtold:
4041 case LibFunc_strtoull:
4042 return optimizeStrTo(CI, B&: Builder);
4043 case LibFunc_strspn:
4044 return optimizeStrSpn(CI, B&: Builder);
4045 case LibFunc_strcspn:
4046 return optimizeStrCSpn(CI, B&: Builder);
4047 case LibFunc_strstr:
4048 return optimizeStrStr(CI, B&: Builder);
4049 case LibFunc_memchr:
4050 return optimizeMemChr(CI, B&: Builder);
4051 case LibFunc_memrchr:
4052 return optimizeMemRChr(CI, B&: Builder);
4053 case LibFunc_bcmp:
4054 return optimizeBCmp(CI, B&: Builder);
4055 case LibFunc_memcmp:
4056 return optimizeMemCmp(CI, B&: Builder);
4057 case LibFunc_memcpy:
4058 return optimizeMemCpy(CI, B&: Builder);
4059 case LibFunc_memccpy:
4060 return optimizeMemCCpy(CI, B&: Builder);
4061 case LibFunc_mempcpy:
4062 return optimizeMemPCpy(CI, B&: Builder);
4063 case LibFunc_memmove:
4064 return optimizeMemMove(CI, B&: Builder);
4065 case LibFunc_memset:
4066 return optimizeMemSet(CI, B&: Builder);
4067 case LibFunc_realloc:
4068 return optimizeRealloc(CI, B&: Builder);
4069 case LibFunc_wcslen:
4070 return optimizeWcslen(CI, B&: Builder);
4071 case LibFunc_bcopy:
4072 return optimizeBCopy(CI, B&: Builder);
4073 case LibFunc_Znwm:
4074 case LibFunc_ZnwmRKSt9nothrow_t:
4075 case LibFunc_ZnwmSt11align_val_t:
4076 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
4077 case LibFunc_Znam:
4078 case LibFunc_ZnamRKSt9nothrow_t:
4079 case LibFunc_ZnamSt11align_val_t:
4080 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
4081 case LibFunc_Znwm12__hot_cold_t:
4082 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
4083 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
4084 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4085 case LibFunc_Znam12__hot_cold_t:
4086 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
4087 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
4088 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4089 case LibFunc_size_returning_new:
4090 case LibFunc_size_returning_new_hot_cold:
4091 case LibFunc_size_returning_new_aligned:
4092 case LibFunc_size_returning_new_aligned_hot_cold:
4093 return optimizeNew(CI, B&: Builder, Func);
4094 default:
4095 break;
4096 }
4097 }
4098 return nullptr;
4099}
4100
4101/// Constant folding nan/nanf/nanl.
4102static Value *optimizeNaN(CallInst *CI) {
4103 StringRef CharSeq;
4104 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: CharSeq))
4105 return nullptr;
4106
4107 APInt Fill;
4108 // Treat empty strings as if they were zero.
4109 if (CharSeq.empty())
4110 Fill = APInt(32, 0);
4111 else if (CharSeq.getAsInteger(Radix: 0, Result&: Fill))
4112 return nullptr;
4113
4114 return ConstantFP::getQNaN(Ty: CI->getType(), /*Negative=*/false, Payload: &Fill);
4115}
4116
4117Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
4118 LibFunc Func,
4119 IRBuilderBase &Builder) {
4120 const Module *M = CI->getModule();
4121
4122 // Don't optimize calls that require strict floating point semantics.
4123 if (CI->isStrictFP())
4124 return nullptr;
4125
4126 if (Value *V = optimizeSymmetric(CI, Func, B&: Builder))
4127 return V;
4128
4129 switch (Func) {
4130 case LibFunc_sinpif:
4131 case LibFunc_sinpi:
4132 return optimizeSinCosPi(CI, /*IsSin*/true, B&: Builder);
4133 case LibFunc_cospif:
4134 case LibFunc_cospi:
4135 return optimizeSinCosPi(CI, /*IsSin*/false, B&: Builder);
4136 case LibFunc_sinf:
4137 case LibFunc_sinl:
4138 if (CI->doesNotAccessMemory())
4139 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::sin);
4140 return nullptr;
4141 case LibFunc_cosf:
4142 case LibFunc_cosl:
4143 if (CI->doesNotAccessMemory())
4144 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::cos);
4145 return nullptr;
4146 case LibFunc_powf:
4147 case LibFunc_pow:
4148 case LibFunc_powl:
4149 return optimizePow(Pow: CI, B&: Builder);
4150 case LibFunc_exp2l:
4151 case LibFunc_exp2:
4152 case LibFunc_exp2f:
4153 return optimizeExp2(CI, B&: Builder);
4154 case LibFunc_fabsf:
4155 case LibFunc_fabs:
4156 case LibFunc_fabsl:
4157 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::fabs);
4158 case LibFunc_sqrtf:
4159 case LibFunc_sqrt:
4160 case LibFunc_sqrtl:
4161 return optimizeSqrt(CI, B&: Builder);
4162 case LibFunc_fmod:
4163 case LibFunc_fmodf:
4164 case LibFunc_fmodl:
4165 return optimizeFMod(CI, B&: Builder);
4166 case LibFunc_logf:
4167 case LibFunc_log:
4168 case LibFunc_logl:
4169 case LibFunc_log10f:
4170 case LibFunc_log10:
4171 case LibFunc_log10l:
4172 case LibFunc_log1pf:
4173 case LibFunc_log1p:
4174 case LibFunc_log1pl:
4175 case LibFunc_log2f:
4176 case LibFunc_log2:
4177 case LibFunc_log2l:
4178 case LibFunc_logbf:
4179 case LibFunc_logb:
4180 case LibFunc_logbl:
4181 return optimizeLog(Log: CI, B&: Builder);
4182 case LibFunc_tan:
4183 case LibFunc_tanf:
4184 case LibFunc_tanl:
4185 case LibFunc_sinh:
4186 case LibFunc_sinhf:
4187 case LibFunc_sinhl:
4188 case LibFunc_asinh:
4189 case LibFunc_asinhf:
4190 case LibFunc_asinhl:
4191 case LibFunc_cosh:
4192 case LibFunc_coshf:
4193 case LibFunc_coshl:
4194 case LibFunc_atanh:
4195 case LibFunc_atanhf:
4196 case LibFunc_atanhl:
4197 return optimizeTrigInversionPairs(CI, B&: Builder);
4198 case LibFunc_ceil:
4199 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::ceil);
4200 case LibFunc_floor:
4201 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::floor);
4202 case LibFunc_round:
4203 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::round);
4204 case LibFunc_roundeven:
4205 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::roundeven);
4206 case LibFunc_nearbyint:
4207 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::nearbyint);
4208 case LibFunc_rint:
4209 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::rint);
4210 case LibFunc_trunc:
4211 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::trunc);
4212 case LibFunc_sin:
4213 case LibFunc_cos:
4214 if (UnsafeFPShrink &&
4215 hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4216 if (Value *V = optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true))
4217 return V;
4218 if (CI->doesNotAccessMemory())
4219 return replaceUnaryCall(
4220 CI, B&: Builder, IID: Func == LibFunc_sin ? Intrinsic::sin : Intrinsic::cos);
4221 return nullptr;
4222 case LibFunc_acos:
4223 case LibFunc_acosh:
4224 case LibFunc_asin:
4225 case LibFunc_atan:
4226 case LibFunc_cbrt:
4227 case LibFunc_exp:
4228 case LibFunc_exp10:
4229 case LibFunc_expm1:
4230 case LibFunc_tanh:
4231 if (UnsafeFPShrink && hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4232 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true);
4233 return nullptr;
4234 case LibFunc_copysign:
4235 if (hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4236 return optimizeBinaryDoubleFP(CI, B&: Builder, TLI);
4237 return nullptr;
4238 case LibFunc_fdim:
4239 case LibFunc_fdimf:
4240 case LibFunc_fdiml:
4241 return optimizeFdim(CI, B&: Builder);
4242 case LibFunc_fminf:
4243 case LibFunc_fmin:
4244 case LibFunc_fminl:
4245 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::minnum);
4246 case LibFunc_fmaxf:
4247 case LibFunc_fmax:
4248 case LibFunc_fmaxl:
4249 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::maxnum);
4250 case LibFunc_fminimum_numf:
4251 case LibFunc_fminimum_num:
4252 case LibFunc_fminimum_numl:
4253 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::minimumnum);
4254 case LibFunc_fmaximum_numf:
4255 case LibFunc_fmaximum_num:
4256 case LibFunc_fmaximum_numl:
4257 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::maximumnum);
4258 case LibFunc_cabs:
4259 case LibFunc_cabsf:
4260 case LibFunc_cabsl:
4261 return optimizeCAbs(CI, B&: Builder);
4262 case LibFunc_remquo:
4263 case LibFunc_remquof:
4264 case LibFunc_remquol:
4265 return optimizeRemquo(CI, B&: Builder);
4266 case LibFunc_nan:
4267 case LibFunc_nanf:
4268 case LibFunc_nanl:
4269 return optimizeNaN(CI);
4270 default:
4271 return nullptr;
4272 }
4273}
4274
4275Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) {
4276 Module *M = CI->getModule();
4277 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
4278
4279 // TODO: Split out the code below that operates on FP calls so that
4280 // we can all non-FP calls with the StrictFP attribute to be
4281 // optimized.
4282 if (CI->isNoBuiltin()) {
4283 // Optionally update operator new calls.
4284 return maybeOptimizeNoBuiltinOperatorNew(CI, B&: Builder);
4285 }
4286
4287 Function *Callee = CI->getCalledFunction();
4288 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4289 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4290
4291 SmallVector<OperandBundleDef, 2> OpBundles;
4292 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4293
4294 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4295 Builder.setDefaultOperandBundles(OpBundles);
4296
4297 // Command-line parameter overrides instruction attribute.
4298 // This can't be moved to optimizeFloatingPointLibCall() because it may be
4299 // used by the intrinsic optimizations.
4300 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
4301 UnsafeFPShrink = EnableUnsafeFPShrink;
4302 else if (isa<FPMathOperator>(Val: CI) && CI->isFast())
4303 UnsafeFPShrink = true;
4304
4305 // First, check for intrinsics.
4306 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI)) {
4307 if (!IsCallingConvC)
4308 return nullptr;
4309 // The FP intrinsics have corresponding constrained versions so we don't
4310 // need to check for the StrictFP attribute here.
4311 switch (II->getIntrinsicID()) {
4312 case Intrinsic::pow:
4313 return optimizePow(Pow: CI, B&: Builder);
4314 case Intrinsic::exp2:
4315 return optimizeExp2(CI, B&: Builder);
4316 case Intrinsic::log:
4317 case Intrinsic::log2:
4318 case Intrinsic::log10:
4319 return optimizeLog(Log: CI, B&: Builder);
4320 case Intrinsic::sqrt:
4321 return optimizeSqrt(CI, B&: Builder);
4322 case Intrinsic::memset:
4323 return optimizeMemSet(CI, B&: Builder);
4324 case Intrinsic::memcpy:
4325 return optimizeMemCpy(CI, B&: Builder);
4326 case Intrinsic::memmove:
4327 return optimizeMemMove(CI, B&: Builder);
4328 case Intrinsic::sin:
4329 case Intrinsic::cos:
4330 if (UnsafeFPShrink)
4331 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, /*isPrecise=*/true);
4332 return nullptr;
4333 case Intrinsic::sincos:
4334 if (UnsafeFPShrink)
4335 return optimizeSinCosDoubleFP(CI, B&: Builder);
4336 return nullptr;
4337 default:
4338 return nullptr;
4339 }
4340 }
4341
4342 // Also try to simplify calls to fortified library functions.
4343 if (Value *SimplifiedFortifiedCI =
4344 FortifiedSimplifier.optimizeCall(CI, B&: Builder))
4345 return SimplifiedFortifiedCI;
4346
4347 // Then check for known library functions.
4348 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4349 // We never change the calling convention.
4350 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4351 return nullptr;
4352 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
4353 return V;
4354 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
4355 return V;
4356 switch (Func) {
4357 case LibFunc_ffs:
4358 case LibFunc_ffsl:
4359 case LibFunc_ffsll:
4360 return optimizeFFS(CI, B&: Builder);
4361 case LibFunc_fls:
4362 case LibFunc_flsl:
4363 case LibFunc_flsll:
4364 return optimizeFls(CI, B&: Builder);
4365 case LibFunc_abs:
4366 case LibFunc_labs:
4367 case LibFunc_llabs:
4368 return optimizeAbs(CI, B&: Builder);
4369 case LibFunc_isdigit:
4370 return optimizeIsDigit(CI, B&: Builder);
4371 case LibFunc_isascii:
4372 return optimizeIsAscii(CI, B&: Builder);
4373 case LibFunc_toascii:
4374 return optimizeToAscii(CI, B&: Builder);
4375 case LibFunc_atoi:
4376 case LibFunc_atol:
4377 case LibFunc_atoll:
4378 return optimizeAtoi(CI, B&: Builder);
4379 case LibFunc_strtol:
4380 case LibFunc_strtoll:
4381 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/true);
4382 case LibFunc_strtoul:
4383 case LibFunc_strtoull:
4384 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/false);
4385 case LibFunc_printf:
4386 return optimizePrintF(CI, B&: Builder);
4387 case LibFunc_sprintf:
4388 return optimizeSPrintF(CI, B&: Builder);
4389 case LibFunc_snprintf:
4390 return optimizeSnPrintF(CI, B&: Builder);
4391 case LibFunc_fprintf:
4392 return optimizeFPrintF(CI, B&: Builder);
4393 case LibFunc_fwrite:
4394 return optimizeFWrite(CI, B&: Builder);
4395 case LibFunc_fputs:
4396 return optimizeFPuts(CI, B&: Builder);
4397 case LibFunc_puts:
4398 return optimizePuts(CI, B&: Builder);
4399 case LibFunc_perror:
4400 return optimizeErrorReporting(CI, B&: Builder);
4401 case LibFunc_vfprintf:
4402 case LibFunc_fiprintf:
4403 return optimizeErrorReporting(CI, B&: Builder, StreamArg: 0);
4404 case LibFunc_exit:
4405 case LibFunc_Exit:
4406 return optimizeExit(CI);
4407 default:
4408 return nullptr;
4409 }
4410 }
4411 return nullptr;
4412}
4413
4414LibCallSimplifier::LibCallSimplifier(
4415 const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT,
4416 DomConditionCache *DC, AssumptionCache *AC, OptimizationRemarkEmitter &ORE,
4417 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
4418 function_ref<void(Instruction *, Value *)> Replacer,
4419 function_ref<void(Instruction *)> Eraser)
4420 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), DT(DT), DC(DC), AC(AC),
4421 ORE(ORE), BFI(BFI), PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
4422
4423void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
4424 // Indirect through the replacer used in this instance.
4425 Replacer(I, With);
4426}
4427
4428void LibCallSimplifier::eraseFromParent(Instruction *I) {
4429 Eraser(I);
4430}
4431
4432// TODO:
4433// Additional cases that we need to add to this file:
4434//
4435// cbrt:
4436// * cbrt(expN(X)) -> expN(x/3)
4437// * cbrt(sqrt(x)) -> pow(x,1/6)
4438// * cbrt(cbrt(x)) -> pow(x,1/9)
4439//
4440// exp, expf, expl:
4441// * exp(log(x)) -> x
4442//
4443// log, logf, logl:
4444// * log(exp(x)) -> x
4445// * log(exp(y)) -> y*log(e)
4446// * log(exp10(y)) -> y*log(10)
4447// * log(sqrt(x)) -> 0.5*log(x)
4448//
4449// pow, powf, powl:
4450// * pow(sqrt(x),y) -> pow(x,y*0.5)
4451// * pow(pow(x,y),z)-> pow(x,y*z)
4452//
4453// signbit:
4454// * signbit(cnst) -> cnst'
4455// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
4456//
4457// sqrt, sqrtf, sqrtl:
4458// * sqrt(expN(x)) -> expN(x*0.5)
4459// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
4460// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
4461//
4462
4463//===----------------------------------------------------------------------===//
4464// Fortified Library Call Optimizations
4465//===----------------------------------------------------------------------===//
4466
4467bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
4468 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
4469 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
4470 // If this function takes a flag argument, the implementation may use it to
4471 // perform extra checks. Don't fold into the non-checking variant.
4472 if (FlagOp) {
4473 ConstantInt *Flag = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *FlagOp));
4474 if (!Flag || !Flag->isZero())
4475 return false;
4476 }
4477
4478 if (SizeOp && CI->getArgOperand(i: ObjSizeOp) == CI->getArgOperand(i: *SizeOp))
4479 return true;
4480
4481 if (ConstantInt *ObjSizeCI =
4482 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: ObjSizeOp))) {
4483 if (ObjSizeCI->isMinusOne())
4484 return true;
4485 // If the object size wasn't -1 (unknown), bail out if we were asked to.
4486 if (OnlyLowerUnknownSize)
4487 return false;
4488 if (StrOp) {
4489 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: *StrOp));
4490 // If the length is 0 we don't know how long it is and so we can't
4491 // remove the check.
4492 if (Len)
4493 annotateDereferenceableBytes(CI, ArgNos: *StrOp, DereferenceableBytes: Len);
4494 else
4495 return false;
4496 return ObjSizeCI->getZExtValue() >= Len;
4497 }
4498
4499 if (SizeOp) {
4500 if (ConstantInt *SizeCI =
4501 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *SizeOp)))
4502 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4503 }
4504 }
4505 return false;
4506}
4507
4508Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4509 IRBuilderBase &B) {
4510 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4511 CallInst *NewCI =
4512 B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4513 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4514 mergeAttributesAndFlags(NewCI, Old: *CI);
4515 return CI->getArgOperand(i: 0);
4516 }
4517 return nullptr;
4518}
4519
4520Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4521 IRBuilderBase &B) {
4522 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4523 CallInst *NewCI =
4524 B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4525 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4526 mergeAttributesAndFlags(NewCI, Old: *CI);
4527 return CI->getArgOperand(i: 0);
4528 }
4529 return nullptr;
4530}
4531
4532Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4533 IRBuilderBase &B) {
4534 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4535 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
4536 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val,
4537 Size: CI->getArgOperand(i: 2), Align: Align(1));
4538 mergeAttributesAndFlags(NewCI, Old: *CI);
4539 return CI->getArgOperand(i: 0);
4540 }
4541 return nullptr;
4542}
4543
4544Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4545 IRBuilderBase &B) {
4546 const DataLayout &DL = CI->getDataLayout();
4547 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2))
4548 if (Value *Call = emitMemPCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4549 Len: CI->getArgOperand(i: 2), B, DL, TLI)) {
4550 return mergeAttributesAndFlags(NewCI: cast<CallInst>(Val: Call), Old: *CI);
4551 }
4552 return nullptr;
4553}
4554
4555Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4556 IRBuilderBase &B,
4557 LibFunc Func) {
4558 const DataLayout &DL = CI->getDataLayout();
4559 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1),
4560 *ObjSize = CI->getArgOperand(i: 2);
4561
4562 // __stpcpy_chk(x,x,...) -> x+strlen(x)
4563 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4564 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
4565 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
4566 }
4567
4568 // If a) we don't have any length information, or b) we know this will
4569 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4570 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
4571 // TODO: It might be nice to get a maximum length out of the possible
4572 // string lengths for varying.
4573 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: 1)) {
4574 if (Func == LibFunc_strcpy_chk)
4575 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
4576 else
4577 return copyFlags(Old: *CI, New: emitStpCpy(Dst, Src, B, TLI));
4578 }
4579
4580 if (OnlyLowerUnknownSize)
4581 return nullptr;
4582
4583 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4584 uint64_t Len = GetStringLength(V: Src);
4585 if (Len)
4586 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
4587 else
4588 return nullptr;
4589
4590 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4591 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4592 Value *LenV = ConstantInt::get(Ty: SizeTTy, V: Len);
4593 Value *Ret = emitMemCpyChk(Dst, Src, Len: LenV, ObjSize, B, DL, TLI);
4594 // If the function was an __stpcpy_chk, and we were able to fold it into
4595 // a __memcpy_chk, we still need to return the correct end pointer.
4596 if (Ret && Func == LibFunc_stpcpy_chk)
4597 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst,
4598 IdxList: ConstantInt::get(Ty: SizeTTy, V: Len - 1));
4599 return copyFlags(Old: *CI, New: cast<CallInst>(Val: Ret));
4600}
4601
4602Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4603 IRBuilderBase &B) {
4604 if (isFortifiedCallFoldable(CI, ObjSizeOp: 1, SizeOp: std::nullopt, StrOp: 0))
4605 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B,
4606 DL: CI->getDataLayout(), TLI));
4607 return nullptr;
4608}
4609
4610Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4611 IRBuilderBase &B,
4612 LibFunc Func) {
4613 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4614 if (Func == LibFunc_strncpy_chk)
4615 return copyFlags(Old: *CI,
4616 New: emitStrNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4617 Len: CI->getArgOperand(i: 2), B, TLI));
4618 else
4619 return copyFlags(Old: *CI,
4620 New: emitStpNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4621 Len: CI->getArgOperand(i: 2), B, TLI));
4622 }
4623
4624 return nullptr;
4625}
4626
4627Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4628 IRBuilderBase &B) {
4629 if (isFortifiedCallFoldable(CI, ObjSizeOp: 4, SizeOp: 3))
4630 return copyFlags(
4631 Old: *CI, New: emitMemCCpy(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
4632 Val: CI->getArgOperand(i: 2), Len: CI->getArgOperand(i: 3), B, TLI));
4633
4634 return nullptr;
4635}
4636
4637Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4638 IRBuilderBase &B) {
4639 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2)) {
4640 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 5));
4641 return copyFlags(Old: *CI,
4642 New: emitSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4643 Fmt: CI->getArgOperand(i: 4), Args: VariadicArgs, B, TLI));
4644 }
4645
4646 return nullptr;
4647}
4648
4649Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4650 IRBuilderBase &B) {
4651 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1)) {
4652 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 4));
4653 return copyFlags(Old: *CI,
4654 New: emitSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4655 VariadicArgs, B, TLI));
4656 }
4657
4658 return nullptr;
4659}
4660
4661Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4662 IRBuilderBase &B) {
4663 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2))
4664 return copyFlags(
4665 Old: *CI, New: emitStrCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1), B, TLI));
4666
4667 return nullptr;
4668}
4669
4670Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4671 IRBuilderBase &B) {
4672 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4673 return copyFlags(Old: *CI,
4674 New: emitStrLCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4675 Size: CI->getArgOperand(i: 2), B, TLI));
4676
4677 return nullptr;
4678}
4679
4680Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4681 IRBuilderBase &B) {
4682 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4683 return copyFlags(Old: *CI,
4684 New: emitStrNCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4685 Size: CI->getArgOperand(i: 2), B, TLI));
4686
4687 return nullptr;
4688}
4689
4690Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4691 IRBuilderBase &B) {
4692 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4693 return copyFlags(Old: *CI,
4694 New: emitStrLCpy(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4695 Size: CI->getArgOperand(i: 2), B, TLI));
4696
4697 return nullptr;
4698}
4699
4700Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4701 IRBuilderBase &B) {
4702 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2))
4703 return copyFlags(
4704 Old: *CI, New: emitVSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4705 Fmt: CI->getArgOperand(i: 4), VAList: CI->getArgOperand(i: 5), B, TLI));
4706
4707 return nullptr;
4708}
4709
4710Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4711 IRBuilderBase &B) {
4712 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1))
4713 return copyFlags(Old: *CI,
4714 New: emitVSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4715 VAList: CI->getArgOperand(i: 4), B, TLI));
4716
4717 return nullptr;
4718}
4719
4720Value *FortifiedLibCallSimplifier::optimizeCall(CallInst *CI,
4721 IRBuilderBase &Builder) {
4722 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4723 // Some clang users checked for _chk libcall availability using:
4724 // __has_builtin(__builtin___memcpy_chk)
4725 // When compiling with -fno-builtin, this is always true.
4726 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4727 // end up with fortified libcalls, which isn't acceptable in a freestanding
4728 // environment which only provides their non-fortified counterparts.
4729 //
4730 // Until we change clang and/or teach external users to check for availability
4731 // differently, disregard the "nobuiltin" attribute and TLI::has.
4732 //
4733 // PR23093.
4734
4735 Function *Callee = CI->getCalledFunction();
4736 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4737
4738 SmallVector<OperandBundleDef, 2> OpBundles;
4739 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4740
4741 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4742 Builder.setDefaultOperandBundles(OpBundles);
4743
4744 // First, check that this is a known library functions and that the prototype
4745 // is correct.
4746 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4747 if (Func == NotLibFunc)
4748 return nullptr;
4749
4750 // We never change the calling convention.
4751 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4752 return nullptr;
4753
4754 switch (Func) {
4755 case LibFunc_memcpy_chk:
4756 return optimizeMemCpyChk(CI, B&: Builder);
4757 case LibFunc_mempcpy_chk:
4758 return optimizeMemPCpyChk(CI, B&: Builder);
4759 case LibFunc_memmove_chk:
4760 return optimizeMemMoveChk(CI, B&: Builder);
4761 case LibFunc_memset_chk:
4762 return optimizeMemSetChk(CI, B&: Builder);
4763 case LibFunc_stpcpy_chk:
4764 case LibFunc_strcpy_chk:
4765 return optimizeStrpCpyChk(CI, B&: Builder, Func);
4766 case LibFunc_strlen_chk:
4767 return optimizeStrLenChk(CI, B&: Builder);
4768 case LibFunc_stpncpy_chk:
4769 case LibFunc_strncpy_chk:
4770 return optimizeStrpNCpyChk(CI, B&: Builder, Func);
4771 case LibFunc_memccpy_chk:
4772 return optimizeMemCCpyChk(CI, B&: Builder);
4773 case LibFunc_snprintf_chk:
4774 return optimizeSNPrintfChk(CI, B&: Builder);
4775 case LibFunc_sprintf_chk:
4776 return optimizeSPrintfChk(CI, B&: Builder);
4777 case LibFunc_strcat_chk:
4778 return optimizeStrCatChk(CI, B&: Builder);
4779 case LibFunc_strlcat_chk:
4780 return optimizeStrLCat(CI, B&: Builder);
4781 case LibFunc_strncat_chk:
4782 return optimizeStrNCatChk(CI, B&: Builder);
4783 case LibFunc_strlcpy_chk:
4784 return optimizeStrLCpyChk(CI, B&: Builder);
4785 case LibFunc_vsnprintf_chk:
4786 return optimizeVSNPrintfChk(CI, B&: Builder);
4787 case LibFunc_vsprintf_chk:
4788 return optimizeVSPrintfChk(CI, B&: Builder);
4789 default:
4790 break;
4791 }
4792 return nullptr;
4793}
4794
4795FortifiedLibCallSimplifier::FortifiedLibCallSimplifier(
4796 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4797 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
4798