1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/SmallString.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/Analysis/ConstantFolding.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/Utils/Local.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/AttributeMask.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/PatternMatch.h"
33#include "llvm/IR/ProfDataUtils.h"
34#include "llvm/Support/Casting.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/KnownBits.h"
37#include "llvm/Support/KnownFPClass.h"
38#include "llvm/Support/MathExtras.h"
39#include "llvm/TargetParser/Triple.h"
40#include "llvm/Transforms/Utils/BuildLibCalls.h"
41#include "llvm/Transforms/Utils/Local.h"
42#include "llvm/Transforms/Utils/SizeOpts.h"
43
44#include <cmath>
45
46using namespace llvm;
47using namespace PatternMatch;
48
49#define DEBUG_TYPE "simplify-lib-calls"
50
51static cl::opt<bool>
52 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
53 cl::init(Val: false),
54 cl::desc("Enable unsafe double to float "
55 "shrinking for math lib calls"));
56
57// Enable conversion of operator new calls with a MemProf hot or cold hint
58// to an operator new call that takes a hot/cold hint. Off by default since
59// not all allocators currently support this extension.
60static cl::opt<bool>
61 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(Val: false),
62 cl::desc("Enable hot/cold operator new library calls"));
63enum class OptimizeExistingHotColdNewKind {
64 None,
65 Cold,
66 Always,
67};
68static cl::opt<OptimizeExistingHotColdNewKind> OptimizeExistingHotColdNew(
69 "optimize-existing-hot-cold-new", cl::Hidden,
70 cl::desc(
71 "Enable optimization of existing hot/cold operator new library calls"),
72 cl::values(
73 clEnumValN(
74 OptimizeExistingHotColdNewKind::None, "none",
75 "Do not optimize existing hot/cold operator new library calls"),
76 clEnumValN(OptimizeExistingHotColdNewKind::Cold, "cold",
77 "Only optimize existing hot/cold operator new library calls "
78 "if determined to be cold"),
79 clEnumValN(
80 OptimizeExistingHotColdNewKind::Always, "always",
81 "Always optimize existing hot/cold operator new library calls"),
82 clEnumValN(
83 OptimizeExistingHotColdNewKind::Always, "",
84 "Always optimize existing hot/cold operator new library calls")),
85 cl::init(Val: OptimizeExistingHotColdNewKind::None), cl::ValueOptional);
86static cl::opt<bool> OptimizeNoBuiltinHotColdNew(
87 "optimize-nobuiltin-hot-cold-new-new", cl::Hidden, cl::init(Val: false),
88 cl::desc("Enable transformation of nobuiltin operator new library calls"));
89static cl::opt<bool> MinExistingHotColdNewHint(
90 "min-existing-hot-cold-new-hint", cl::Hidden, cl::init(Val: false),
91 cl::desc("Take the minimum of compiler hint and existing hint when "
92 "optimizing existing hot/cold operator new library calls"));
93
94namespace llvm {
95extern cl::opt<bool> ProfcheckDisableMetadataFixes;
96} // namespace llvm
97
98namespace {
99
100// Specialized parser to ensure the hint is an 8 bit value (we can't specify
101// uint8_t to opt<> as that is interpreted to mean that we are passing a char
102// option with a specific set of values.
103struct HotColdHintParser : public cl::parser<unsigned> {
104 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
105
106 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
107 if (Arg.getAsInteger(Radix: 0, Result&: Value))
108 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
109
110 if (Value > 255)
111 return O.error(Message: "'" + Arg + "' value must be in the range [0, 255]!");
112
113 return false;
114 }
115};
116
117} // end anonymous namespace
118
119// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
120// and 255 is the hottest. Default to 1 value away from the coldest and hottest
121// hints, so that the compiler hinted allocations are slightly less strong than
122// manually inserted hints at the two extremes.
123static cl::opt<unsigned, false, HotColdHintParser> ColdNewHintValue(
124 "cold-new-hint-value", cl::Hidden, cl::init(Val: 1),
125 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
126static cl::opt<unsigned, false, HotColdHintParser>
127 NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(Val: 128),
128 cl::desc("Value to pass to hot/cold operator new for "
129 "notcold (warm) allocation"));
130static cl::opt<unsigned, false, HotColdHintParser> HotNewHintValue(
131 "hot-new-hint-value", cl::Hidden, cl::init(Val: 254),
132 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
133static cl::opt<unsigned, false, HotColdHintParser> AmbiguousNewHintValue(
134 "ambiguous-new-hint-value", cl::Hidden, cl::init(Val: 222),
135 cl::desc(
136 "Value to pass to hot/cold operator new for ambiguous allocation"));
137
138//===----------------------------------------------------------------------===//
139// Helper Functions
140//===----------------------------------------------------------------------===//
141
142static bool ignoreCallingConv(LibFunc Func) {
143 return Func == LibFunc_abs || Func == LibFunc_labs ||
144 Func == LibFunc_llabs || Func == LibFunc_strlen;
145}
146
147/// Return true if it is only used in equality comparisons with With.
148static bool isOnlyUsedInEqualityComparison(Value *V, Value *With) {
149 for (User *U : V->users()) {
150 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
151 if (IC->isEquality() && IC->getOperand(i_nocapture: 1) == With)
152 continue;
153 // Unknown instruction.
154 return false;
155 }
156 return true;
157}
158
159static bool callHasFloatingPointArgument(const CallInst *CI) {
160 return any_of(Range: CI->operands(), P: [](const Use &OI) {
161 return OI->getType()->isFloatingPointTy();
162 });
163}
164
165static bool callHasFP128Argument(const CallInst *CI) {
166 return any_of(Range: CI->operands(), P: [](const Use &OI) {
167 return OI->getType()->isFP128Ty();
168 });
169}
170
171// Convert the entire string Str representing an integer in Base, up to
172// the terminating nul if present, to a constant according to the rules
173// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
174// return the result, otherwise null.
175// The function assumes the string is encoded in ASCII and carefully
176// avoids converting sequences (including "") that the corresponding
177// library call might fail and set errno for.
178static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
179 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
180 if (Base < 2 || Base > 36)
181 if (Base != 0)
182 // Fail for an invalid base (required by POSIX).
183 return nullptr;
184
185 // Current offset into the original string to reflect in EndPtr.
186 size_t Offset = 0;
187 // Strip leading whitespace.
188 for ( ; Offset != Str.size(); ++Offset)
189 if (!isSpace(C: (unsigned char)Str[Offset])) {
190 Str = Str.substr(Start: Offset);
191 break;
192 }
193
194 if (Str.empty())
195 // Fail for empty subject sequences (POSIX allows but doesn't require
196 // strtol[l]/strtoul[l] to fail with EINVAL).
197 return nullptr;
198
199 // Strip but remember the sign.
200 bool Negate = Str[0] == '-';
201 if (Str[0] == '-' || Str[0] == '+') {
202 Str = Str.drop_front();
203 if (Str.empty())
204 // Fail for a sign with nothing after it.
205 return nullptr;
206 ++Offset;
207 }
208
209 // Set Max to the absolute value of the minimum (for signed), or
210 // to the maximum (for unsigned) value representable in the type.
211 Type *RetTy = CI->getType();
212 unsigned NBits = RetTy->getPrimitiveSizeInBits();
213 uint64_t Max = AsSigned && Negate ? 1 : 0;
214 Max += AsSigned ? maxIntN(N: NBits) : maxUIntN(N: NBits);
215
216 // Autodetect Base if it's zero and consume the "0x" prefix.
217 if (Str.size() > 1) {
218 if (Str[0] == '0') {
219 if (toUpper(x: (unsigned char)Str[1]) == 'X') {
220 if (Str.size() == 2 || (Base && Base != 16))
221 // Fail if Base doesn't allow the "0x" prefix or for the prefix
222 // alone that implementations like BSD set errno to EINVAL for.
223 return nullptr;
224
225 Str = Str.drop_front(N: 2);
226 Offset += 2;
227 Base = 16;
228 }
229 else if (Base == 0)
230 Base = 8;
231 } else if (Base == 0)
232 Base = 10;
233 }
234 else if (Base == 0)
235 Base = 10;
236
237 // Convert the rest of the subject sequence, not including the sign,
238 // to its uint64_t representation (this assumes the source character
239 // set is ASCII).
240 uint64_t Result = 0;
241 for (unsigned i = 0; i != Str.size(); ++i) {
242 unsigned char DigVal = Str[i];
243 if (isDigit(C: DigVal))
244 DigVal = DigVal - '0';
245 else {
246 DigVal = toUpper(x: DigVal);
247 if (isAlpha(C: DigVal))
248 DigVal = DigVal - 'A' + 10;
249 else
250 return nullptr;
251 }
252
253 if (DigVal >= Base)
254 // Fail if the digit is not valid in the Base.
255 return nullptr;
256
257 // Add the digit and fail if the result is not representable in
258 // the (unsigned form of the) destination type.
259 bool VFlow;
260 Result = SaturatingMultiplyAdd(X: Result, Y: Base, A: (uint64_t)DigVal, ResultOverflowed: &VFlow);
261 if (VFlow || Result > Max)
262 return nullptr;
263 }
264
265 if (EndPtr) {
266 // Store the pointer to the end.
267 Value *Off = B.getInt64(C: Offset + Str.size());
268 Value *StrBeg = CI->getArgOperand(i: 0);
269 Value *StrEnd = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: StrBeg, IdxList: Off, Name: "endptr");
270 B.CreateStore(Val: StrEnd, Ptr: EndPtr);
271 }
272
273 if (Negate) {
274 // Unsigned negation doesn't overflow.
275 Result = -Result;
276 // For unsigned numbers, discard sign bits.
277 if (!AsSigned)
278 Result &= maxUIntN(N: NBits);
279 }
280
281 return ConstantInt::get(Ty: RetTy, V: Result, IsSigned: AsSigned);
282}
283
284static bool isOnlyUsedInComparisonWithZero(Value *V) {
285 for (User *U : V->users()) {
286 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
287 if (Constant *C = dyn_cast<Constant>(Val: IC->getOperand(i_nocapture: 1)))
288 if (C->isNullValue())
289 continue;
290 // Unknown instruction.
291 return false;
292 }
293 return true;
294}
295
296static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
297 const SimplifyQuery &SQ) {
298 if (!isOnlyUsedInComparisonWithZero(V: CI))
299 return false;
300
301 if (!isDereferenceablePointer(V: Str, Size: APInt(64, Len), Q: SQ))
302 return false;
303
304 if (CI->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemory))
305 return false;
306
307 return true;
308}
309
310static void annotateDereferenceableBytes(CallInst *CI,
311 ArrayRef<unsigned> ArgNos,
312 uint64_t DereferenceableBytes) {
313 const Function *F = CI->getCaller();
314 if (!F)
315 return;
316 for (unsigned ArgNo : ArgNos) {
317 uint64_t DerefBytes = DereferenceableBytes;
318 unsigned AS = CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
319 if (!llvm::NullPointerIsDefined(F, AS) ||
320 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
321 DerefBytes = std::max(a: CI->getParamDereferenceableOrNullBytes(i: ArgNo),
322 b: DereferenceableBytes);
323
324 if (CI->getParamDereferenceableBytes(i: ArgNo) < DerefBytes) {
325 CI->removeParamAttr(ArgNo, Kind: Attribute::Dereferenceable);
326 if (!llvm::NullPointerIsDefined(F, AS) ||
327 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
328 CI->removeParamAttr(ArgNo, Kind: Attribute::DereferenceableOrNull);
329 CI->addParamAttr(ArgNo, Attr: Attribute::getWithDereferenceableBytes(
330 Context&: CI->getContext(), Bytes: DerefBytes));
331 }
332 }
333}
334
335static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI,
336 ArrayRef<unsigned> ArgNos) {
337 Function *F = CI->getCaller();
338 if (!F)
339 return;
340
341 for (unsigned ArgNo : ArgNos) {
342 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NoUndef))
343 CI->addParamAttr(ArgNo, Kind: Attribute::NoUndef);
344
345 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull)) {
346 unsigned AS =
347 CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
348 if (llvm::NullPointerIsDefined(F, AS))
349 continue;
350 CI->addParamAttr(ArgNo, Kind: Attribute::NonNull);
351 }
352
353 annotateDereferenceableBytes(CI, ArgNos: ArgNo, DereferenceableBytes: 1);
354 }
355}
356
357static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> ArgNos,
358 Value *Size, const DataLayout &DL) {
359 if (ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size)) {
360 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
361 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: LenC->getZExtValue());
362 } else if (isKnownNonZero(V: Size, Q: DL)) {
363 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
364 uint64_t X, Y;
365 uint64_t DerefMin = 1;
366 if (match(V: Size, P: m_Select(C: m_Value(), L: m_ConstantInt(V&: X), R: m_ConstantInt(V&: Y)))) {
367 DerefMin = std::min(a: X, b: Y);
368 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: DerefMin);
369 }
370 }
371}
372
373// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
374// easier chaining. Calls to emit* and B.createCall should probably be wrapped
375// in this function when New is created to replace Old. Callers should take
376// care to check Old.isMustTailCall() if they aren't replacing Old directly
377// with New.
378static Value *copyFlags(const CallInst &Old, Value *New) {
379 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
380 assert(!Old.isNoTailCall() && "do not copy notail call flags");
381 if (auto *NewCI = dyn_cast_or_null<CallInst>(Val: New))
382 NewCI->setTailCallKind(Old.getTailCallKind());
383 return New;
384}
385
386static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
387 NewCI->setAttributes(AttributeList::get(
388 C&: NewCI->getContext(), Attrs: {NewCI->getAttributes(), Old.getAttributes()}));
389 NewCI->removeRetAttrs(AttrsToRemove: AttributeFuncs::typeIncompatible(
390 Ty: NewCI->getType(), AS: NewCI->getRetAttributes()));
391 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
392 NewCI->removeParamAttrs(
393 ArgNo: I, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewCI->getArgOperand(i: I)->getType(),
394 AS: NewCI->getParamAttributes(ArgNo: I)));
395
396 return copyFlags(Old, New: NewCI);
397}
398
399// Helper to avoid truncating the length if size_t is 32-bits.
400static StringRef substr(StringRef Str, uint64_t Len) {
401 return Len >= Str.size() ? Str : Str.substr(Start: 0, N: Len);
402}
403
404//===----------------------------------------------------------------------===//
405// String and Memory Library Call Optimizations
406//===----------------------------------------------------------------------===//
407
408Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
409 // Extract some information from the instruction
410 Value *Dst = CI->getArgOperand(i: 0);
411 Value *Src = CI->getArgOperand(i: 1);
412 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
413
414 // See if we can get the length of the input string.
415 uint64_t Len = GetStringLength(V: Src);
416 if (Len)
417 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
418 else
419 return nullptr;
420 --Len; // Unbias length.
421
422 // Handle the simple, do-nothing case: strcat(x, "") -> x
423 if (Len == 0)
424 return Dst;
425
426 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len, B));
427}
428
429Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
430 IRBuilderBase &B) {
431 // We need to find the end of the destination string. That's where the
432 // memory is to be moved to. We just generate a call to strlen.
433 Value *DstLen = emitStrLen(Ptr: Dst, B, DL, TLI);
434 if (!DstLen)
435 return nullptr;
436
437 // Now that we have the destination's length, we must index into the
438 // destination's pointer to get the actual memcpy destination (end of
439 // the string .. we're concatenating).
440 Value *CpyDst = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: DstLen, Name: "endptr");
441
442 // We have enough information to now generate the memcpy call to do the
443 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
444 B.CreateMemCpy(Dst: CpyDst, DstAlign: Align(1), Src, SrcAlign: Align(1),
445 Size: TLI->getAsSizeT(V: Len + 1, M: *B.GetInsertBlock()->getModule()));
446 return Dst;
447}
448
449Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
450 // Extract some information from the instruction.
451 Value *Dst = CI->getArgOperand(i: 0);
452 Value *Src = CI->getArgOperand(i: 1);
453 Value *Size = CI->getArgOperand(i: 2);
454 uint64_t Len;
455 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
456 if (isKnownNonZero(V: Size, Q: DL))
457 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
458
459 // We don't do anything if length is not constant.
460 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size);
461 if (LengthArg) {
462 Len = LengthArg->getZExtValue();
463 // strncat(x, c, 0) -> x
464 if (!Len)
465 return Dst;
466 } else {
467 return nullptr;
468 }
469
470 // See if we can get the length of the input string.
471 uint64_t SrcLen = GetStringLength(V: Src);
472 if (SrcLen) {
473 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
474 --SrcLen; // Unbias length.
475 } else {
476 return nullptr;
477 }
478
479 // strncat(x, "", c) -> x
480 if (SrcLen == 0)
481 return Dst;
482
483 // We don't optimize this case.
484 if (Len < SrcLen)
485 return nullptr;
486
487 // strncat(x, s, c) -> strcat(x, s)
488 // s is constant so the strcat can be optimized further.
489 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len: SrcLen, B));
490}
491
492// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
493// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
494// is that either S is dereferenceable or the value of N is nonzero.
495static Value* memChrToCharCompare(CallInst *CI, Value *NBytes,
496 IRBuilderBase &B, const DataLayout &DL)
497{
498 Value *Src = CI->getArgOperand(i: 0);
499 Value *CharVal = CI->getArgOperand(i: 1);
500
501 // Fold memchr(A, C, N) == A to N && *A == C.
502 Type *CharTy = B.getInt8Ty();
503 Value *Char0 = B.CreateLoad(Ty: CharTy, Ptr: Src);
504 CharVal = B.CreateTrunc(V: CharVal, DestTy: CharTy);
505 Value *Cmp = B.CreateICmpEQ(LHS: Char0, RHS: CharVal, Name: "char0cmp");
506
507 if (NBytes) {
508 Value *Zero = ConstantInt::get(Ty: NBytes->getType(), V: 0);
509 Value *And = B.CreateICmpNE(LHS: NBytes, RHS: Zero);
510 Cmp = B.CreateLogicalAnd(Cond1: And, Cond2: Cmp);
511 // The and above is based on the byte count and the query, neither of which
512 // we know without value profiling, so mark the profile as unknown.
513 if (auto *SI = dyn_cast<SelectInst>(Val: Cmp))
514 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE);
515 }
516
517 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
518 return B.CreateSelect(C: Cmp, True: Src, False: NullPtr);
519}
520
521Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
522 Value *SrcStr = CI->getArgOperand(i: 0);
523 Value *CharVal = CI->getArgOperand(i: 1);
524 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
525
526 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
527 return memChrToCharCompare(CI, NBytes: nullptr, B, DL);
528
529 // If the second operand is non-constant, see if we can compute the length
530 // of the input string and turn this into memchr.
531 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
532 if (!CharC) {
533 uint64_t Len = GetStringLength(V: SrcStr);
534 if (Len)
535 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len);
536 else
537 return nullptr;
538
539 Function *Callee = CI->getCalledFunction();
540 FunctionType *FT = Callee->getFunctionType();
541 unsigned IntBits = TLI->getIntSize();
542 if (!FT->getParamType(i: 1)->isIntegerTy(BitWidth: IntBits)) // memchr needs 'int'.
543 return nullptr;
544
545 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
546 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
547 return copyFlags(Old: *CI,
548 New: emitMemChr(Ptr: SrcStr, Val: CharVal, // include nul.
549 Len: ConstantInt::get(Ty: SizeTTy, V: Len), B,
550 DL, TLI));
551 }
552
553 if (CharC->isZero()) {
554 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
555 if (isOnlyUsedInEqualityComparison(V: CI, With: NullPtr))
556 // Pre-empt the transformation to strlen below and fold
557 // strchr(A, '\0') == null to false.
558 return B.CreateIntToPtr(V: B.getTrue(), DestTy: CI->getType());
559 }
560
561 // Otherwise, the character is a constant, see if the first argument is
562 // a string literal. If so, we can constant fold.
563 StringRef Str;
564 if (!getConstantStringInfo(V: SrcStr, Str)) {
565 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
566 if (Value *StrLen = emitStrLen(Ptr: SrcStr, B, DL, TLI))
567 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: StrLen, Name: "strchr");
568 return nullptr;
569 }
570
571 // Compute the offset, make sure to handle the case when we're searching for
572 // zero (a weird way to spell strlen).
573 size_t I = (0xFF & CharC->getSExtValue()) == 0
574 ? Str.size()
575 : Str.find(C: CharC->getSExtValue());
576 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
577 return Constant::getNullValue(Ty: CI->getType());
578
579 // strchr(s+n,c) -> gep(s+n+i,c)
580 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: I), Name: "strchr");
581}
582
583Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
584 Value *SrcStr = CI->getArgOperand(i: 0);
585 Value *CharVal = CI->getArgOperand(i: 1);
586 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
587 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
588
589 StringRef Str;
590 if (!getConstantStringInfo(V: SrcStr, Str)) {
591 // strrchr(s, 0) -> strchr(s, 0)
592 if (CharC && CharC->isZero())
593 return copyFlags(Old: *CI, New: emitStrChr(Ptr: SrcStr, C: '\0', B, TLI));
594 return nullptr;
595 }
596
597 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
598 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
599
600 // Try to expand strrchr to the memrchr nonstandard extension if it's
601 // available, or simply fail otherwise.
602 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
603 Value *Size = ConstantInt::get(Ty: SizeTTy, V: NBytes);
604 return copyFlags(Old: *CI, New: emitMemRChr(Ptr: SrcStr, Val: CharVal, Len: Size, B, DL, TLI));
605}
606
607Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
608 Value *Str1P = CI->getArgOperand(i: 0), *Str2P = CI->getArgOperand(i: 1);
609 if (Str1P == Str2P) // strcmp(x,x) -> 0
610 return ConstantInt::get(Ty: CI->getType(), V: 0);
611
612 StringRef Str1, Str2;
613 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
614 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
615
616 // strcmp(x, y) -> cnst (if both x and y are constant strings)
617 if (HasStr1 && HasStr2)
618 return ConstantInt::getSigned(Ty: CI->getType(),
619 V: std::clamp(val: Str1.compare(RHS: Str2), lo: -1, hi: 1));
620
621 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
622 return B.CreateNeg(V: B.CreateZExt(
623 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
624
625 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
626 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
627 DestTy: CI->getType());
628
629 // strcmp(P, "x") -> memcmp(P, "x", 2)
630 uint64_t Len1 = GetStringLength(V: Str1P);
631 if (Len1)
632 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
633 uint64_t Len2 = GetStringLength(V: Str2P);
634 if (Len2)
635 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
636
637 if (Len1 && Len2) {
638 return copyFlags(
639 Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
640 Len: TLI->getAsSizeT(V: std::min(a: Len1, b: Len2), M: *CI->getModule()),
641 B, DL, TLI));
642 }
643
644 // strcmp to memcmp
645 SimplifyQuery SQ(DL, TLI, DT, AC, CI);
646 if (!HasStr1 && HasStr2) {
647 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ))
648 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
649 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
650 B, DL, TLI));
651 } else if (HasStr1 && !HasStr2) {
652 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ))
653 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
654 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
655 B, DL, TLI));
656 }
657
658 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
659 return nullptr;
660}
661
662// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
663// arrays LHS and RHS and nonconstant Size.
664static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
665 Value *Size, bool StrNCmp,
666 IRBuilderBase &B, const DataLayout &DL);
667
668Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
669 Value *Str1P = CI->getArgOperand(i: 0);
670 Value *Str2P = CI->getArgOperand(i: 1);
671 Value *Size = CI->getArgOperand(i: 2);
672 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
673 return ConstantInt::get(Ty: CI->getType(), V: 0);
674
675 if (isKnownNonZero(V: Size, Q: DL))
676 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
677 // Get the length argument if it is constant.
678 uint64_t Length;
679 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size))
680 Length = LengthArg->getZExtValue();
681 else
682 return optimizeMemCmpVarSize(CI, LHS: Str1P, RHS: Str2P, Size, StrNCmp: true, B, DL);
683
684 if (Length == 0) // strncmp(x,y,0) -> 0
685 return ConstantInt::get(Ty: CI->getType(), V: 0);
686
687 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
688 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P, Len: Size, B, DL, TLI));
689
690 StringRef Str1, Str2;
691 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
692 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
693
694 // strncmp(x, y) -> cnst (if both x and y are constant strings)
695 if (HasStr1 && HasStr2) {
696 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
697 StringRef SubStr1 = substr(Str: Str1, Len: Length);
698 StringRef SubStr2 = substr(Str: Str2, Len: Length);
699 return ConstantInt::getSigned(Ty: CI->getType(),
700 V: std::clamp(val: SubStr1.compare(RHS: SubStr2), lo: -1, hi: 1));
701 }
702
703 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
704 return B.CreateNeg(V: B.CreateZExt(
705 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
706
707 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
708 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
709 DestTy: CI->getType());
710
711 uint64_t Len1 = GetStringLength(V: Str1P);
712 if (Len1)
713 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
714 uint64_t Len2 = GetStringLength(V: Str2P);
715 if (Len2)
716 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
717
718 // strncmp to memcmp
719 if (!HasStr1 && HasStr2) {
720 Len2 = std::min(a: Len2, b: Length);
721 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, SQ: DL))
722 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
723 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
724 B, DL, TLI));
725 } else if (HasStr1 && !HasStr2) {
726 Len1 = std::min(a: Len1, b: Length);
727 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, SQ: DL))
728 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
729 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
730 B, DL, TLI));
731 }
732
733 return nullptr;
734}
735
736Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
737 Value *Src = CI->getArgOperand(i: 0);
738 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
739 uint64_t SrcLen = GetStringLength(V: Src);
740 if (SrcLen && Size) {
741 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: SrcLen);
742 if (SrcLen <= Size->getZExtValue() + 1)
743 return copyFlags(Old: *CI, New: emitStrDup(Ptr: Src, B, TLI));
744 }
745
746 return nullptr;
747}
748
749Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
750 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
751 if (Dst == Src) // strcpy(x,x) -> x
752 return Src;
753
754 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
755 // See if we can get the length of the input string.
756 uint64_t Len = GetStringLength(V: Src);
757 if (Len)
758 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
759 else
760 return nullptr;
761
762 // We have enough information to now generate the memcpy call to do the
763 // copy for us. Make a memcpy to copy the nul byte with align = 1.
764 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
765 Size: TLI->getAsSizeT(V: Len, M: *CI->getModule()));
766 mergeAttributesAndFlags(NewCI, Old: *CI);
767 return Dst;
768}
769
770Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
771 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
772
773 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
774 if (CI->use_empty())
775 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
776
777 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
778 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
779 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
780 }
781
782 // See if we can get the length of the input string.
783 uint64_t Len = GetStringLength(V: Src);
784 if (Len)
785 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
786 else
787 return nullptr;
788
789 Value *LenV = TLI->getAsSizeT(V: Len, M: *CI->getModule());
790 Value *DstEnd = B.CreateInBoundsGEP(
791 Ty: B.getInt8Ty(), Ptr: Dst, IdxList: TLI->getAsSizeT(V: Len - 1, M: *CI->getModule()));
792
793 // We have enough information to now generate the memcpy call to do the
794 // copy for us. Make a memcpy to copy the nul byte with align = 1.
795 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: LenV);
796 mergeAttributesAndFlags(NewCI, Old: *CI);
797 return DstEnd;
798}
799
800// Optimize a call to size_t strlcpy(char*, const char*, size_t).
801
802Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
803 Value *Size = CI->getArgOperand(i: 2);
804 if (isKnownNonZero(V: Size, Q: DL))
805 // Like snprintf, the function stores into the destination only when
806 // the size argument is nonzero.
807 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
808 // The function reads the source argument regardless of Size (it returns
809 // its length).
810 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
811
812 uint64_t NBytes;
813 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
814 NBytes = SizeC->getZExtValue();
815 else
816 return nullptr;
817
818 Value *Dst = CI->getArgOperand(i: 0);
819 Value *Src = CI->getArgOperand(i: 1);
820 if (NBytes <= 1) {
821 if (NBytes == 1)
822 // For a call to strlcpy(D, S, 1) first store a nul in *D.
823 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
824
825 // Transform strlcpy(D, S, 0) to a call to strlen(S).
826 return copyFlags(Old: *CI, New: emitStrLen(Ptr: Src, B, DL, TLI));
827 }
828
829 // Try to determine the length of the source, substituting its size
830 // when it's not nul-terminated (as it's required to be) to avoid
831 // reading past its end.
832 StringRef Str;
833 if (!getConstantStringInfo(V: Src, Str, /*TrimAtNul=*/false))
834 return nullptr;
835
836 uint64_t SrcLen = Str.find(C: '\0');
837 // Set if the terminating nul should be copied by the call to memcpy
838 // below.
839 bool NulTerm = SrcLen < NBytes;
840
841 if (NulTerm)
842 // Overwrite NBytes with the number of bytes to copy, including
843 // the terminating nul.
844 NBytes = SrcLen + 1;
845 else {
846 // Set the length of the source for the function to return to its
847 // size, and cap NBytes at the same.
848 SrcLen = std::min(a: SrcLen, b: uint64_t(Str.size()));
849 NBytes = std::min(a: NBytes - 1, b: SrcLen);
850 }
851
852 if (SrcLen == 0) {
853 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
854 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
855 return ConstantInt::get(Ty: CI->getType(), V: 0);
856 }
857
858 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
859 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
860 // D[N' - 1] if necessary.
861 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
862 Size: TLI->getAsSizeT(V: NBytes, M: *CI->getModule()));
863 mergeAttributesAndFlags(NewCI, Old: *CI);
864
865 if (!NulTerm) {
866 Value *EndOff = ConstantInt::get(Ty: CI->getType(), V: NBytes);
867 Value *EndPtr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: EndOff);
868 B.CreateStore(Val: B.getInt8(C: 0), Ptr: EndPtr);
869 }
870
871 // Like snprintf, strlcpy returns the number of nonzero bytes that would
872 // have been copied if the bound had been sufficiently big (which in this
873 // case is strlen(Src)).
874 return ConstantInt::get(Ty: CI->getType(), V: SrcLen);
875}
876
877// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
878// otherwise.
879Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
880 IRBuilderBase &B) {
881 Value *Dst = CI->getArgOperand(i: 0);
882 Value *Src = CI->getArgOperand(i: 1);
883 Value *Size = CI->getArgOperand(i: 2);
884
885 if (isKnownNonZero(V: Size, Q: DL)) {
886 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
887 // only when N is nonzero.
888 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
889 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
890 }
891
892 // If the "bound" argument is known set N to it. Otherwise set it to
893 // UINT64_MAX and handle it later.
894 uint64_t N = UINT64_MAX;
895 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
896 N = SizeC->getZExtValue();
897
898 if (N == 0)
899 // Fold st{p,r}ncpy(D, S, 0) to D.
900 return Dst;
901
902 if (N == 1) {
903 Type *CharTy = B.getInt8Ty();
904 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "stxncpy.char0");
905 B.CreateStore(Val: CharVal, Ptr: Dst);
906 if (!RetEnd)
907 // Transform strncpy(D, S, 1) to return (*D = *S), D.
908 return Dst;
909
910 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
911 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
912 Value *Cmp = B.CreateICmpEQ(LHS: CharVal, RHS: ZeroChar, Name: "stpncpy.char0cmp");
913
914 Value *Off1 = B.getInt32(C: 1);
915 Value *EndPtr = B.CreateInBoundsGEP(Ty: CharTy, Ptr: Dst, IdxList: Off1, Name: "stpncpy.end");
916 return B.CreateSelect(C: Cmp, True: Dst, False: EndPtr, Name: "stpncpy.sel");
917 }
918
919 // If the length of the input string is known set SrcLen to it.
920 uint64_t SrcLen = GetStringLength(V: Src);
921 if (SrcLen)
922 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
923 else
924 return nullptr;
925
926 --SrcLen; // Unbias length.
927
928 if (SrcLen == 0) {
929 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
930 Align MemSetAlign =
931 CI->getAttributes().getParamAttrs(ArgNo: 0).getAlignment().valueOrOne();
932 CallInst *NewCI = B.CreateMemSet(Ptr: Dst, Val: B.getInt8(C: '\0'), Size, Align: MemSetAlign);
933 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(ArgNo: 0));
934 NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
935 C&: CI->getContext(), ArgNo: 0, B: ArgAttrs));
936 copyFlags(Old: *CI, New: NewCI);
937 return Dst;
938 }
939
940 if (N > SrcLen + 1) {
941 if (N > 128)
942 // Bail if N is large or unknown.
943 return nullptr;
944
945 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
946 StringRef Str;
947 if (!getConstantStringInfo(V: Src, Str))
948 return nullptr;
949 std::string SrcStr = Str.str();
950 // Create a bigger, nul-padded array with the same length, SrcLen,
951 // as the original string.
952 SrcStr.resize(n: N, c: '\0');
953 Src = B.CreateGlobalString(Str: SrcStr, Name: "str", /*AddressSpace=*/0,
954 /*M=*/nullptr, /*AddNull=*/false);
955 }
956
957 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
958 // S and N are constant.
959 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
960 Size: TLI->getAsSizeT(V: N, M: *CI->getModule()));
961 mergeAttributesAndFlags(NewCI, Old: *CI);
962 if (!RetEnd)
963 return Dst;
964
965 // stpncpy(D, S, N) returns the address of the first null in D if it writes
966 // one, otherwise D + N.
967 Value *Off = B.getInt64(C: std::min(a: SrcLen, b: N));
968 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: Off, Name: "endptr");
969}
970
971Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
972 unsigned CharSize,
973 Value *Bound) {
974 Value *Src = CI->getArgOperand(i: 0);
975 Type *CharTy = B.getIntNTy(N: CharSize);
976
977 if (isOnlyUsedInZeroEqualityComparison(CxtI: CI) &&
978 (!Bound || isKnownNonZero(V: Bound, Q: DL))) {
979 // Fold strlen:
980 // strlen(x) != 0 --> *x != 0
981 // strlen(x) == 0 --> *x == 0
982 // and likewise strnlen with constant N > 0:
983 // strnlen(x, N) != 0 --> *x != 0
984 // strnlen(x, N) == 0 --> *x == 0
985 return B.CreateZExt(V: B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "char0"),
986 DestTy: CI->getType());
987 }
988
989 if (Bound) {
990 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Val: Bound)) {
991 if (BoundCst->isZero())
992 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
993 return ConstantInt::get(Ty: CI->getType(), V: 0);
994
995 if (BoundCst->isOne()) {
996 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
997 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "strnlen.char0");
998 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
999 Value *Cmp = B.CreateICmpNE(LHS: CharVal, RHS: ZeroChar, Name: "strnlen.char0cmp");
1000 return B.CreateZExt(V: Cmp, DestTy: CI->getType());
1001 }
1002 }
1003 }
1004
1005 if (uint64_t Len = GetStringLength(V: Src, CharSize)) {
1006 Value *LenC = ConstantInt::get(Ty: CI->getType(), V: Len - 1);
1007 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
1008 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
1009 if (Bound)
1010 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: LenC, RHS: Bound);
1011 return LenC;
1012 }
1013
1014 if (Bound)
1015 // Punt for strnlen for now.
1016 return nullptr;
1017
1018 // If s is a constant pointer pointing to a string literal, we can fold
1019 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
1020 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
1021 // We only try to simplify strlen when the pointer s points to an array
1022 // of CharSize elements. Otherwise, we would need to scale the offset x before
1023 // doing the subtraction. This will make the optimization more complex, and
1024 // it's not very useful because calling strlen for a pointer of other types is
1025 // very uncommon.
1026 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: Src)) {
1027 unsigned BW = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1028 SmallMapVector<Value *, APInt, 4> VarOffsets;
1029 APInt ConstOffset(BW, 0);
1030 assert(CharSize % 8 == 0 && "Expected a multiple of 8 sized CharSize");
1031 // Check the gep is a single variable offset.
1032 if (!GEP->collectOffset(DL, BitWidth: BW, VariableOffsets&: VarOffsets, ConstantOffset&: ConstOffset) ||
1033 VarOffsets.size() != 1 || ConstOffset != 0 ||
1034 VarOffsets.begin()->second != CharSize / 8)
1035 return nullptr;
1036
1037 ConstantDataArraySlice Slice;
1038 if (getConstantDataArrayInfo(V: GEP->getOperand(i_nocapture: 0), Slice, ElementSize: CharSize)) {
1039 uint64_t NullTermIdx;
1040 if (Slice.Array == nullptr) {
1041 NullTermIdx = 0;
1042 } else {
1043 NullTermIdx = ~((uint64_t)0);
1044 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
1045 if (Slice.Array->getElementAsInteger(i: I + Slice.Offset) == 0) {
1046 NullTermIdx = I;
1047 break;
1048 }
1049 }
1050 // If the string does not have '\0', leave it to strlen to compute
1051 // its length.
1052 if (NullTermIdx == ~((uint64_t)0))
1053 return nullptr;
1054 }
1055
1056 Value *Offset = VarOffsets.begin()->first;
1057 KnownBits Known = computeKnownBits(V: Offset, DL, AC: nullptr, CxtI: CI, DT: nullptr);
1058
1059 // If Offset is not provably in the range [0, NullTermIdx], we can still
1060 // optimize if we can prove that the program has undefined behavior when
1061 // Offset is outside that range. That is the case when GEP->getOperand(0)
1062 // is a pointer to an object whose memory extent is NullTermIdx+1.
1063 if ((Known.isNonNegative() && Known.getMaxValue().ule(RHS: NullTermIdx)) ||
1064 (isa<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)) &&
1065 NullTermIdx == Slice.Length - 1)) {
1066 Offset = B.CreateSExtOrTrunc(V: Offset, DestTy: CI->getType());
1067 return B.CreateSub(LHS: ConstantInt::get(Ty: CI->getType(), V: NullTermIdx),
1068 RHS: Offset);
1069 }
1070 }
1071 }
1072
1073 // strlen(x?"foo":"bars") --> x ? 3 : 4
1074 if (SelectInst *SI = dyn_cast<SelectInst>(Val: Src)) {
1075 uint64_t LenTrue = GetStringLength(V: SI->getTrueValue(), CharSize);
1076 uint64_t LenFalse = GetStringLength(V: SI->getFalseValue(), CharSize);
1077 if (LenTrue && LenFalse) {
1078 ORE.emit(RemarkBuilder: [&]() {
1079 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1080 << "folded strlen(select) to select of constants";
1081 });
1082 return B.CreateSelect(C: SI->getCondition(),
1083 True: ConstantInt::get(Ty: CI->getType(), V: LenTrue - 1),
1084 False: ConstantInt::get(Ty: CI->getType(), V: LenFalse - 1), Name: "",
1085 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : SI);
1086 }
1087 }
1088
1089 return nullptr;
1090}
1091
1092Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1093 if (Value *V = optimizeStringLength(CI, B, CharSize: 8))
1094 return V;
1095 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1096 return nullptr;
1097}
1098
1099Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1100 Value *Bound = CI->getArgOperand(i: 1);
1101 if (Value *V = optimizeStringLength(CI, B, CharSize: 8, Bound))
1102 return V;
1103
1104 if (isKnownNonZero(V: Bound, Q: DL))
1105 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1106 return nullptr;
1107}
1108
1109Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1110 Module &M = *CI->getModule();
1111 unsigned WCharSize = TLI->getWCharSize(M) * 8;
1112 // We cannot perform this optimization without wchar_size metadata.
1113 if (WCharSize == 0)
1114 return nullptr;
1115
1116 return optimizeStringLength(CI, B, CharSize: WCharSize);
1117}
1118
1119Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1120 StringRef S1, S2;
1121 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1122 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1123
1124 // strpbrk(s, "") -> nullptr
1125 // strpbrk("", s) -> nullptr
1126 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1127 return Constant::getNullValue(Ty: CI->getType());
1128
1129 // Constant folding.
1130 if (HasS1 && HasS2) {
1131 size_t I = S1.find_first_of(Chars: S2);
1132 if (I == StringRef::npos) // No match.
1133 return Constant::getNullValue(Ty: CI->getType());
1134
1135 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1136 IdxList: B.getInt64(C: I), Name: "strpbrk");
1137 }
1138
1139 // strpbrk(s, "a") -> strchr(s, 'a')
1140 if (HasS2 && S2.size() == 1)
1141 return copyFlags(Old: *CI, New: emitStrChr(Ptr: CI->getArgOperand(i: 0), C: S2[0], B, TLI));
1142
1143 return nullptr;
1144}
1145
1146Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1147 Value *EndPtr = CI->getArgOperand(i: 1);
1148 if (isa<ConstantPointerNull>(Val: EndPtr)) {
1149 // With a null EndPtr, this function won't capture the main argument.
1150 // It would be readonly too, except that it still may write to errno.
1151 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
1152 CI: CaptureInfo::none()));
1153 }
1154
1155 return nullptr;
1156}
1157
1158Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1159 StringRef S1, S2;
1160 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1161 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1162
1163 // strspn(s, "") -> 0
1164 // strspn("", s) -> 0
1165 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1166 return Constant::getNullValue(Ty: CI->getType());
1167
1168 // Constant folding.
1169 if (HasS1 && HasS2) {
1170 size_t Pos = S1.find_first_not_of(Chars: S2);
1171 if (Pos == StringRef::npos)
1172 Pos = S1.size();
1173 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1174 }
1175
1176 return nullptr;
1177}
1178
1179Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1180 StringRef S1, S2;
1181 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1182 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1183
1184 // strcspn("", s) -> 0
1185 if (HasS1 && S1.empty())
1186 return Constant::getNullValue(Ty: CI->getType());
1187
1188 // Constant folding.
1189 if (HasS1 && HasS2) {
1190 size_t Pos = S1.find_first_of(Chars: S2);
1191 if (Pos == StringRef::npos)
1192 Pos = S1.size();
1193 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1194 }
1195
1196 // strcspn(s, "") -> strlen(s)
1197 if (HasS2 && S2.empty())
1198 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B, DL, TLI));
1199
1200 return nullptr;
1201}
1202
1203Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1204 // fold strstr(x, x) -> x.
1205 if (CI->getArgOperand(i: 0) == CI->getArgOperand(i: 1))
1206 return CI->getArgOperand(i: 0);
1207
1208 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1209 if (isOnlyUsedInEqualityComparison(V: CI, With: CI->getArgOperand(i: 0))) {
1210 Value *StrLen = emitStrLen(Ptr: CI->getArgOperand(i: 1), B, DL, TLI);
1211 if (!StrLen)
1212 return nullptr;
1213 Value *StrNCmp = emitStrNCmp(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
1214 Len: StrLen, B, DL, TLI);
1215 if (!StrNCmp)
1216 return nullptr;
1217 for (User *U : llvm::make_early_inc_range(Range: CI->users())) {
1218 ICmpInst *Old = cast<ICmpInst>(Val: U);
1219 Value *Cmp =
1220 B.CreateICmp(P: Old->getPredicate(), LHS: StrNCmp,
1221 RHS: ConstantInt::getNullValue(Ty: StrNCmp->getType()), Name: "cmp");
1222 replaceAllUsesWith(I: Old, With: Cmp);
1223 }
1224 return CI;
1225 }
1226
1227 // See if either input string is a constant string.
1228 StringRef SearchStr, ToFindStr;
1229 bool HasStr1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: SearchStr);
1230 bool HasStr2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: ToFindStr);
1231
1232 // fold strstr(x, "") -> x.
1233 if (HasStr2 && ToFindStr.empty())
1234 return CI->getArgOperand(i: 0);
1235
1236 // If both strings are known, constant fold it.
1237 if (HasStr1 && HasStr2) {
1238 size_t Offset = SearchStr.find(Str: ToFindStr);
1239
1240 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1241 return Constant::getNullValue(Ty: CI->getType());
1242
1243 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1244 return B.CreateConstInBoundsGEP1_64(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1245 Idx0: Offset, Name: "strstr");
1246 }
1247
1248 // fold strstr(x, "y") -> strchr(x, 'y').
1249 if (HasStr2 && ToFindStr.size() == 1) {
1250 return emitStrChr(Ptr: CI->getArgOperand(i: 0), C: ToFindStr[0], B, TLI);
1251 }
1252
1253 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
1254 return nullptr;
1255}
1256
1257Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1258 Value *SrcStr = CI->getArgOperand(i: 0);
1259 Value *Size = CI->getArgOperand(i: 2);
1260 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1261 Value *CharVal = CI->getArgOperand(i: 1);
1262 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1263 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1264
1265 if (LenC) {
1266 if (LenC->isZero())
1267 // Fold memrchr(x, y, 0) --> null.
1268 return NullPtr;
1269
1270 if (LenC->isOne()) {
1271 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1272 // constant or otherwise.
1273 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memrchr.char0");
1274 // Slice off the character's high end bits.
1275 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1276 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memrchr.char0cmp");
1277 return B.CreateSelect(C: Cmp, True: SrcStr, False: NullPtr, Name: "memrchr.sel");
1278 }
1279 }
1280
1281 StringRef Str;
1282 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1283 return nullptr;
1284
1285 if (Str.size() == 0)
1286 // If the array is empty fold memrchr(A, C, N) to null for any value
1287 // of C and N on the basis that the only valid value of N is zero
1288 // (otherwise the call is undefined).
1289 return NullPtr;
1290
1291 uint64_t EndOff = UINT64_MAX;
1292 if (LenC) {
1293 EndOff = LenC->getZExtValue();
1294 if (Str.size() < EndOff)
1295 // Punt out-of-bounds accesses to sanitizers and/or libc.
1296 return nullptr;
1297 }
1298
1299 if (ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal)) {
1300 // Fold memrchr(S, C, N) for a constant C.
1301 size_t Pos = Str.rfind(C: CharC->getZExtValue(), From: EndOff);
1302 if (Pos == StringRef::npos)
1303 // When the character is not in the source array fold the result
1304 // to null regardless of Size.
1305 return NullPtr;
1306
1307 if (LenC)
1308 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1309 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos));
1310
1311 if (Str.find(C: Str[Pos]) == Pos) {
1312 // When there is just a single occurrence of C in S, i.e., the one
1313 // in Str[Pos], fold
1314 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1315 // for nonconstant N.
1316 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1317 Name: "memrchr.cmp");
1318 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr,
1319 IdxList: B.getInt64(C: Pos), Name: "memrchr.ptr_plus");
1320 return B.CreateSelect(C: Cmp, True: NullPtr, False: SrcPlus, Name: "memrchr.sel");
1321 }
1322 }
1323
1324 // Truncate the string to search at most EndOff characters.
1325 Str = Str.substr(Start: 0, N: EndOff);
1326 if (Str.find_first_not_of(C: Str[0]) != StringRef::npos)
1327 return nullptr;
1328
1329 // If the source array consists of all equal characters, then for any
1330 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1331 // N != 0 && *S == C ? S + N - 1 : null
1332 Type *SizeTy = Size->getType();
1333 Type *Int8Ty = B.getInt8Ty();
1334 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1335 // Slice off the sought character's high end bits.
1336 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1337 Value *CEqS0 = B.CreateICmpEQ(LHS: ConstantInt::get(Ty: Int8Ty, V: Str[0]), RHS: CharVal);
1338 Value *And = B.CreateLogicalAnd(Cond1: NNeZ, Cond2: CEqS0);
1339 Value *SizeM1 = B.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
1340 Value *SrcPlus =
1341 B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: SrcStr, IdxList: SizeM1, Name: "memrchr.ptr_plus");
1342 return B.CreateSelect(C: And, True: SrcPlus, False: NullPtr, Name: "memrchr.sel");
1343}
1344
1345Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1346 Value *SrcStr = CI->getArgOperand(i: 0);
1347 Value *Size = CI->getArgOperand(i: 2);
1348
1349 if (isKnownNonZero(V: Size, Q: DL)) {
1350 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1351 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1352 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1353 }
1354
1355 Value *CharVal = CI->getArgOperand(i: 1);
1356 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
1357 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1358 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1359
1360 // memchr(x, y, 0) -> null
1361 if (LenC) {
1362 if (LenC->isZero())
1363 return NullPtr;
1364
1365 if (LenC->isOne()) {
1366 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1367 // constant or otherwise.
1368 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memchr.char0");
1369 // Slice off the character's high end bits.
1370 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1371 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memchr.char0cmp");
1372 // The condition depends on the value of the string being equal to the
1373 // query, neither of which we know without value profiling, so mark the
1374 // profile unknown.
1375 return B.CreateSelectWithUnknownProfile(C: Cmp, True: SrcStr, False: NullPtr, DEBUG_TYPE,
1376 Name: "memchr.sel");
1377 }
1378 }
1379
1380 StringRef Str;
1381 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1382 return nullptr;
1383
1384 if (CharC) {
1385 size_t Pos = Str.find(C: CharC->getZExtValue());
1386 if (Pos == StringRef::npos)
1387 // When the character is not in the source array fold the result
1388 // to null regardless of Size.
1389 return NullPtr;
1390
1391 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1392 // When the constant Size is less than or equal to the character
1393 // position also fold the result to null.
1394 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1395 Name: "memchr.cmp");
1396 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos),
1397 Name: "memchr.ptr");
1398 // The condition is dependent upon the value of n, which we cannot infer
1399 // without value profiling, so mark the profile unknown.
1400 return B.CreateSelectWithUnknownProfile(C: Cmp, True: NullPtr, False: SrcPlus, DEBUG_TYPE);
1401 }
1402
1403 if (Str.size() == 0)
1404 // If the array is empty fold memchr(A, C, N) to null for any value
1405 // of C and N on the basis that the only valid value of N is zero
1406 // (otherwise the call is undefined).
1407 return NullPtr;
1408
1409 if (LenC)
1410 Str = substr(Str, Len: LenC->getZExtValue());
1411
1412 size_t Pos = Str.find_first_not_of(C: Str[0]);
1413 if (Pos == StringRef::npos
1414 || Str.find_first_not_of(C: Str[Pos], From: Pos) == StringRef::npos) {
1415 // If the source array consists of at most two consecutive sequences
1416 // of the same characters, then for any C and N (whether in bounds or
1417 // not), fold memchr(S, C, N) to
1418 // N != 0 && *S == C ? S : null
1419 // or for the two sequences to:
1420 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1421 // ^Sel2 ^Sel1 are denoted above.
1422 // The latter makes it also possible to fold strchr() calls with strings
1423 // of the same characters.
1424 Type *SizeTy = Size->getType();
1425 Type *Int8Ty = B.getInt8Ty();
1426
1427 // Slice off the sought character's high end bits.
1428 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1429
1430 Value *Sel1 = NullPtr;
1431 if (Pos != StringRef::npos) {
1432 // Handle two consecutive sequences of the same characters.
1433 Value *PosVal = ConstantInt::get(Ty: SizeTy, V: Pos);
1434 Value *StrPos = ConstantInt::get(Ty: Int8Ty, V: Str[Pos]);
1435 Value *CEqSPos = B.CreateICmpEQ(LHS: CharVal, RHS: StrPos);
1436 Value *NGtPos = B.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: Size, RHS: PosVal);
1437 Value *And = B.CreateAnd(LHS: CEqSPos, RHS: NGtPos);
1438 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: PosVal);
1439 // The condition depends on the value of the query and size, neither of
1440 // which we know without value profiling, so mark the profile unknown.
1441 Sel1 = B.CreateSelectWithUnknownProfile(C: And, True: SrcPlus, False: NullPtr, DEBUG_TYPE,
1442 Name: "memchr.sel1");
1443 }
1444
1445 Value *Str0 = ConstantInt::get(Ty: Int8Ty, V: Str[0]);
1446 Value *CEqS0 = B.CreateICmpEQ(LHS: Str0, RHS: CharVal);
1447 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1448 Value *And = B.CreateAnd(LHS: NNeZ, RHS: CEqS0);
1449 // The condition depends on the value of the query and size, neither of
1450 // which we know without value profiling, so mark the profile unknown.
1451 return B.CreateSelectWithUnknownProfile(C: And, True: SrcStr, False: Sel1, DEBUG_TYPE,
1452 Name: "memchr.sel2");
1453 }
1454
1455 if (!LenC) {
1456 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1457 // S is dereferenceable so it's safe to load from it and fold
1458 // memchr(S, C, N) == S to N && *S == C for any C and N.
1459 // TODO: This is safe even for nonconstant S.
1460 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1461
1462 // From now on we need a constant length and constant array.
1463 return nullptr;
1464 }
1465
1466 bool OptForSize = llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
1467 QueryType: PGSOQueryType::IRPass);
1468
1469 // If the char is variable but the input str and length are not we can turn
1470 // this memchr call into a simple bit field test. Of course this only works
1471 // when the return value is only checked against null.
1472 //
1473 // It would be really nice to reuse switch lowering here but we can't change
1474 // the CFG at this point.
1475 //
1476 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1477 // != 0
1478 // after bounds check.
1479 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CxtI: CI))
1480 return nullptr;
1481
1482 unsigned char Max =
1483 *std::max_element(first: reinterpret_cast<const unsigned char *>(Str.begin()),
1484 last: reinterpret_cast<const unsigned char *>(Str.end()));
1485
1486 // Make sure the bit field we're about to create fits in a register on the
1487 // target.
1488 // FIXME: On a 64 bit architecture this prevents us from using the
1489 // interesting range of alpha ascii chars. We could do better by emitting
1490 // two bitfields or shifting the range by 64 if no lower chars are used.
1491 if (!DL.fitsInLegalInteger(Width: Max + 1)) {
1492 // Build chain of ORs
1493 // Transform:
1494 // memchr("abcd", C, 4) != nullptr
1495 // to:
1496 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1497 std::string SortedStr = Str.str();
1498 llvm::sort(C&: SortedStr);
1499 // Compute the number of of non-contiguous ranges.
1500 unsigned NonContRanges = 1;
1501 for (size_t i = 1; i < SortedStr.size(); ++i) {
1502 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1503 NonContRanges++;
1504 }
1505 }
1506
1507 // Restrict this optimization to profitable cases with one or two range
1508 // checks.
1509 if (NonContRanges > 2)
1510 return nullptr;
1511
1512 // Slice off the character's high end bits.
1513 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1514
1515 SmallVector<Value *> CharCompares;
1516 for (unsigned char C : SortedStr)
1517 CharCompares.push_back(Elt: B.CreateICmpEQ(LHS: CharVal, RHS: B.getInt8(C)));
1518
1519 return B.CreateIntToPtr(V: B.CreateOr(Ops: CharCompares), DestTy: CI->getType());
1520 }
1521
1522 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1523 // creating unnecessary illegal types.
1524 unsigned char Width = NextPowerOf2(A: std::max(a: (unsigned char)7, b: Max));
1525
1526 // Now build the bit field.
1527 APInt Bitfield(Width, 0);
1528 for (char C : Str)
1529 Bitfield.setBit((unsigned char)C);
1530 Value *BitfieldC = B.getInt(AI: Bitfield);
1531
1532 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1533 Value *C = B.CreateZExtOrTrunc(V: CharVal, DestTy: BitfieldC->getType());
1534 C = B.CreateAnd(LHS: C, RHS: B.getIntN(N: Width, C: 0xFF));
1535
1536 // First check that the bit field access is within bounds.
1537 Value *Bounds = B.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: C, RHS: B.getIntN(N: Width, C: Width),
1538 Name: "memchr.bounds");
1539
1540 // Create code that checks if the given bit is set in the field.
1541 Value *Shl = B.CreateShl(LHS: B.getIntN(N: Width, C: 1ULL), RHS: C);
1542 Value *Bits = B.CreateIsNotNull(Arg: B.CreateAnd(LHS: Shl, RHS: BitfieldC), Name: "memchr.bits");
1543
1544 // Finally merge both checks and cast to pointer type. The inttoptr
1545 // implicitly zexts the i1 to intptr type.
1546 Value *Memchr = B.CreateLogicalAnd(Cond1: Bounds, Cond2: Bits, Name: "memchr");
1547 // We construct an and between the value of the memory and the bytes to search
1548 // for. We cannot infer how often this would be true without value profiling
1549 // for the query, so mark the profile unknown.
1550 if (auto *SI = dyn_cast<SelectInst>(Val: Memchr))
1551 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE);
1552 return B.CreateIntToPtr(V: Memchr, DestTy: CI->getType());
1553}
1554
1555// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1556// arrays LHS and RHS and nonconstant Size.
1557static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
1558 Value *Size, bool StrNCmp,
1559 IRBuilderBase &B, const DataLayout &DL) {
1560 if (LHS == RHS) // memcmp(s,s,x) -> 0
1561 return Constant::getNullValue(Ty: CI->getType());
1562
1563 StringRef LStr, RStr;
1564 if (!getConstantStringInfo(V: LHS, Str&: LStr, /*TrimAtNul=*/false) ||
1565 !getConstantStringInfo(V: RHS, Str&: RStr, /*TrimAtNul=*/false))
1566 return nullptr;
1567
1568 // If the contents of both constant arrays are known, fold a call to
1569 // memcmp(A, B, N) to
1570 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1571 // where Pos is the first mismatch between A and B, determined below.
1572
1573 uint64_t Pos = 0;
1574 Value *Zero = ConstantInt::get(Ty: CI->getType(), V: 0);
1575 for (uint64_t MinSize = std::min(a: LStr.size(), b: RStr.size()); ; ++Pos) {
1576 if (Pos == MinSize ||
1577 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1578 // One array is a leading part of the other of equal or greater
1579 // size, or for strncmp, the arrays are equal strings.
1580 // Fold the result to zero. Size is assumed to be in bounds, since
1581 // otherwise the call would be undefined.
1582 return Zero;
1583 }
1584
1585 if (LStr[Pos] != RStr[Pos])
1586 break;
1587 }
1588
1589 // Normalize the result.
1590 typedef unsigned char UChar;
1591 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1592 Value *MaxSize = ConstantInt::get(Ty: Size->getType(), V: Pos);
1593 Value *Cmp = B.CreateICmp(P: ICmpInst::ICMP_ULE, LHS: Size, RHS: MaxSize);
1594 Value *Res = ConstantInt::getSigned(Ty: CI->getType(), V: IRes);
1595 return B.CreateSelect(C: Cmp, True: Zero, False: Res);
1596}
1597
1598// Optimize a memcmp call CI with constant size Len.
1599static Value *optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS,
1600 uint64_t Len, IRBuilderBase &B,
1601 const DataLayout &DL) {
1602 if (Len == 0) // memcmp(s1,s2,0) -> 0
1603 return Constant::getNullValue(Ty: CI->getType());
1604
1605 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1606 if (Len == 1) {
1607 Value *LHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: LHS, Name: "lhsc"),
1608 DestTy: CI->getType(), Name: "lhsv");
1609 Value *RHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: RHS, Name: "rhsc"),
1610 DestTy: CI->getType(), Name: "rhsv");
1611 return B.CreateSub(LHS: LHSV, RHS: RHSV, Name: "chardiff");
1612 }
1613
1614 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1615 // TODO: The case where both inputs are constants does not need to be limited
1616 // to legal integers or equality comparison. See block below this.
1617 if (DL.isLegalInteger(Width: Len * 8) && isOnlyUsedInZeroEqualityComparison(CxtI: CI)) {
1618 IntegerType *IntType = IntegerType::get(C&: CI->getContext(), NumBits: Len * 8);
1619 Align PrefAlignment = DL.getPrefTypeAlign(Ty: IntType);
1620
1621 // First, see if we can fold either argument to a constant.
1622 Value *LHSV = nullptr;
1623 if (auto *LHSC = dyn_cast<Constant>(Val: LHS))
1624 LHSV = ConstantFoldLoadFromConstPtr(C: LHSC, Ty: IntType, DL);
1625
1626 Value *RHSV = nullptr;
1627 if (auto *RHSC = dyn_cast<Constant>(Val: RHS))
1628 RHSV = ConstantFoldLoadFromConstPtr(C: RHSC, Ty: IntType, DL);
1629
1630 // Don't generate unaligned loads. If either source is constant data,
1631 // alignment doesn't matter for that source because there is no load.
1632 if ((LHSV || getKnownAlignment(V: LHS, DL, CxtI: CI) >= PrefAlignment) &&
1633 (RHSV || getKnownAlignment(V: RHS, DL, CxtI: CI) >= PrefAlignment)) {
1634 if (!LHSV)
1635 LHSV = B.CreateLoad(Ty: IntType, Ptr: LHS, Name: "lhsv");
1636 if (!RHSV)
1637 RHSV = B.CreateLoad(Ty: IntType, Ptr: RHS, Name: "rhsv");
1638 return B.CreateZExt(V: B.CreateICmpNE(LHS: LHSV, RHS: RHSV), DestTy: CI->getType(), Name: "memcmp");
1639 }
1640 }
1641
1642 return nullptr;
1643}
1644
1645// Most simplifications for memcmp also apply to bcmp.
1646Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1647 IRBuilderBase &B) {
1648 Value *LHS = CI->getArgOperand(i: 0), *RHS = CI->getArgOperand(i: 1);
1649 Value *Size = CI->getArgOperand(i: 2);
1650
1651 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1652
1653 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, StrNCmp: false, B, DL))
1654 return Res;
1655
1656 // Handle constant Size.
1657 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1658 if (!LenC)
1659 return nullptr;
1660
1661 return optimizeMemCmpConstantSize(CI, LHS, RHS, Len: LenC->getZExtValue(), B, DL);
1662}
1663
1664Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1665 Module *M = CI->getModule();
1666 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1667 return V;
1668
1669 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1670 // bcmp can be more efficient than memcmp because it only has to know that
1671 // there is a difference, not how different one is to the other.
1672 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_bcmp) &&
1673 isOnlyUsedInZeroEqualityComparison(CxtI: CI)) {
1674 Value *LHS = CI->getArgOperand(i: 0);
1675 Value *RHS = CI->getArgOperand(i: 1);
1676 Value *Size = CI->getArgOperand(i: 2);
1677 return copyFlags(Old: *CI, New: emitBCmp(Ptr1: LHS, Ptr2: RHS, Len: Size, B, DL, TLI));
1678 }
1679
1680 return nullptr;
1681}
1682
1683Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1684 return optimizeMemCmpBCmpCommon(CI, B);
1685}
1686
1687Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1688 Value *Size = CI->getArgOperand(i: 2);
1689 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1690 if (isa<IntrinsicInst>(Val: CI))
1691 return nullptr;
1692
1693 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1694 CallInst *NewCI = B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1695 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1696 mergeAttributesAndFlags(NewCI, Old: *CI);
1697 return CI->getArgOperand(i: 0);
1698}
1699
1700Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1701 Value *Dst = CI->getArgOperand(i: 0);
1702 Value *Src = CI->getArgOperand(i: 1);
1703 ConstantInt *StopChar = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
1704 ConstantInt *N = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
1705 StringRef SrcStr;
1706 if (CI->use_empty() && Dst == Src)
1707 return Dst;
1708 // memccpy(d, s, c, 0) -> nullptr
1709 if (N) {
1710 if (N->isNullValue())
1711 return Constant::getNullValue(Ty: CI->getType());
1712 if (!getConstantStringInfo(V: Src, Str&: SrcStr, /*TrimAtNul=*/false) ||
1713 // TODO: Handle zeroinitializer.
1714 !StopChar)
1715 return nullptr;
1716 } else {
1717 return nullptr;
1718 }
1719
1720 // Wrap arg 'c' of type int to char
1721 size_t Pos = SrcStr.find(C: StopChar->getSExtValue() & 0xFF);
1722 if (Pos == StringRef::npos) {
1723 if (N->getZExtValue() <= SrcStr.size()) {
1724 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
1725 Size: CI->getArgOperand(i: 3)));
1726 return Constant::getNullValue(Ty: CI->getType());
1727 }
1728 return nullptr;
1729 }
1730
1731 Value *NewN =
1732 ConstantInt::get(Ty: N->getType(), V: std::min(a: uint64_t(Pos + 1), b: N->getZExtValue()));
1733 // memccpy -> llvm.memcpy
1734 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: NewN));
1735 return Pos + 1 <= N->getZExtValue()
1736 ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: NewN)
1737 : Constant::getNullValue(Ty: CI->getType());
1738}
1739
1740Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1741 Value *Dst = CI->getArgOperand(i: 0);
1742 Value *N = CI->getArgOperand(i: 2);
1743 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1744 CallInst *NewCI =
1745 B.CreateMemCpy(Dst, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size: N);
1746 // Propagate attributes, but memcpy has no return value, so make sure that
1747 // any return attributes are compliant.
1748 // TODO: Attach return value attributes to the 1st operand to preserve them?
1749 mergeAttributesAndFlags(NewCI, Old: *CI);
1750 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: N);
1751}
1752
1753Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1754 Value *Size = CI->getArgOperand(i: 2);
1755 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1756 if (isa<IntrinsicInst>(Val: CI))
1757 return nullptr;
1758
1759 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1760 CallInst *NewCI = B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1761 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1762 mergeAttributesAndFlags(NewCI, Old: *CI);
1763 return CI->getArgOperand(i: 0);
1764}
1765
1766Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1767 Value *Size = CI->getArgOperand(i: 2);
1768 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1769 if (isa<IntrinsicInst>(Val: CI))
1770 return nullptr;
1771
1772 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1773 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
1774 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val, Size, Align: Align(1));
1775 mergeAttributesAndFlags(NewCI, Old: *CI);
1776 return CI->getArgOperand(i: 0);
1777}
1778
1779Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1780 if (isa<ConstantPointerNull>(Val: CI->getArgOperand(i: 0))) {
1781 Value *Malloc = emitMalloc(Num: CI->getArgOperand(i: 1), B, DL, TLI);
1782 if (auto *MallocCI = dyn_cast_or_null<CallInst>(Val: Malloc))
1783 if (MDNode *MD = CI->getMetadata(KindID: LLVMContext::MD_alloc_token))
1784 MallocCI->setMetadata(KindID: LLVMContext::MD_alloc_token, Node: MD);
1785 return copyFlags(Old: *CI, New: Malloc);
1786 }
1787
1788 return nullptr;
1789}
1790
1791// Optionally allow optimization of nobuiltin calls to operator new and its
1792// variants.
1793Value *LibCallSimplifier::maybeOptimizeNoBuiltinOperatorNew(CallInst *CI,
1794 IRBuilderBase &B) {
1795 if (!OptimizeHotColdNew)
1796 return nullptr;
1797 Function *Callee = CI->getCalledFunction();
1798 if (!Callee)
1799 return nullptr;
1800 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
1801 if (Func == NotLibFunc)
1802 return nullptr;
1803 switch (Func) {
1804 case LibFunc_Znwm:
1805 case LibFunc_ZnwmRKSt9nothrow_t:
1806 case LibFunc_ZnwmSt11align_val_t:
1807 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1808 case LibFunc_Znam:
1809 case LibFunc_ZnamRKSt9nothrow_t:
1810 case LibFunc_ZnamSt11align_val_t:
1811 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1812 case LibFunc_size_returning_new:
1813 case LibFunc_size_returning_new_aligned:
1814 // By default normal operator new calls (not already passing a hot_cold_t
1815 // parameter) are not mutated if the call is not marked builtin. Optionally
1816 // enable that in cases where it is known to be safe.
1817 if (!OptimizeNoBuiltinHotColdNew)
1818 return nullptr;
1819 break;
1820 case LibFunc_Znwm12__hot_cold_t:
1821 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1822 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1823 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1824 case LibFunc_Znam12__hot_cold_t:
1825 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1826 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1827 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1828 case LibFunc_size_returning_new_hot_cold:
1829 case LibFunc_size_returning_new_aligned_hot_cold:
1830 // If the nobuiltin call already passes a hot_cold_t parameter, allow update
1831 // of that parameter when enabled.
1832 if (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::None)
1833 return nullptr;
1834 break;
1835 default:
1836 return nullptr;
1837 }
1838 return optimizeNew(CI, B, Func);
1839}
1840
1841// When enabled, replace operator new() calls marked with a hot or cold memprof
1842// attribute with an operator new() call that takes a __hot_cold_t parameter.
1843// Currently this is supported by the open source version of tcmalloc, see:
1844// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1845Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1846 LibFunc &Func) {
1847 if (!OptimizeHotColdNew)
1848 return nullptr;
1849
1850 uint8_t HotCold;
1851 bool IsCold = false;
1852 if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold") {
1853 HotCold = ColdNewHintValue;
1854 IsCold = true;
1855 } else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1856 "notcold")
1857 HotCold = NotColdNewHintValue;
1858 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "hot")
1859 HotCold = HotNewHintValue;
1860 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1861 "ambiguous")
1862 HotCold = AmbiguousNewHintValue;
1863 else
1864 return nullptr;
1865
1866 bool ShouldOptimizeExistingHotColdNew =
1867 OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Always ||
1868 (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Cold &&
1869 IsCold);
1870
1871 Value *HotColdVal = B.getInt8(C: HotCold);
1872 auto getHotColdHintForExisting = [&](uint8_t HotCold) -> Value * {
1873 // If not taking the minimum, simply use the compiler hint value.
1874 if (!MinExistingHotColdNewHint)
1875 return HotColdVal;
1876 Value *ExistingHint = CI->getArgOperand(i: CI->arg_size() - 1);
1877 if (ExistingHint->getType() != B.getInt8Ty())
1878 ExistingHint = B.CreateTruncOrBitCast(V: ExistingHint, DestTy: B.getInt8Ty());
1879 // Emit a umin intrinsic to take the minimum of the existing hint and the
1880 // compiler hint. When the existing hint is a compile-time constant, the
1881 // IRBuilder folder will automatically constant-fold this into a constant.
1882 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: ExistingHint, RHS: HotColdVal);
1883 };
1884
1885 // For calls that already pass a hot/cold hint, only update the hint if
1886 // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
1887 // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
1888 // Note that in cases where we decide it is "notcold", it might be slightly
1889 // better to replace the hinted call with a non hinted call, to avoid the
1890 // extra parameter and the if condition check of the hint value in the
1891 // allocator. This can be considered in the future.
1892 Value *NewCall = nullptr;
1893 switch (Func) {
1894 case LibFunc_Znwm12__hot_cold_t:
1895 if (ShouldOptimizeExistingHotColdNew)
1896 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1897 NewFunc: LibFunc_Znwm12__hot_cold_t,
1898 HotCold: getHotColdHintForExisting(HotCold));
1899 break;
1900 case LibFunc_Znwm:
1901 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1902 NewFunc: LibFunc_Znwm12__hot_cold_t, HotCold: HotColdVal);
1903 break;
1904 case LibFunc_Znam12__hot_cold_t:
1905 if (ShouldOptimizeExistingHotColdNew)
1906 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1907 NewFunc: LibFunc_Znam12__hot_cold_t,
1908 HotCold: getHotColdHintForExisting(HotCold));
1909 break;
1910 case LibFunc_Znam:
1911 NewCall = emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1912 NewFunc: LibFunc_Znam12__hot_cold_t, HotCold: HotColdVal);
1913 break;
1914 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1915 if (ShouldOptimizeExistingHotColdNew)
1916 NewCall =
1917 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1918 TLI, NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
1919 HotCold: getHotColdHintForExisting(HotCold));
1920 break;
1921 case LibFunc_ZnwmRKSt9nothrow_t:
1922 NewCall = emitHotColdNewNoThrow(
1923 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1924 NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1925 break;
1926 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1927 if (ShouldOptimizeExistingHotColdNew)
1928 NewCall =
1929 emitHotColdNewNoThrow(Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B,
1930 TLI, NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
1931 HotCold: getHotColdHintForExisting(HotCold));
1932 break;
1933 case LibFunc_ZnamRKSt9nothrow_t:
1934 NewCall = emitHotColdNewNoThrow(
1935 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1936 NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold: HotColdVal);
1937 break;
1938 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1939 if (ShouldOptimizeExistingHotColdNew)
1940 NewCall =
1941 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1942 TLI, NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
1943 HotCold: getHotColdHintForExisting(HotCold));
1944 break;
1945 case LibFunc_ZnwmSt11align_val_t:
1946 NewCall = emitHotColdNewAligned(
1947 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1948 NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1949 break;
1950 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1951 if (ShouldOptimizeExistingHotColdNew)
1952 NewCall =
1953 emitHotColdNewAligned(Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B,
1954 TLI, NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t,
1955 HotCold: getHotColdHintForExisting(HotCold));
1956 break;
1957 case LibFunc_ZnamSt11align_val_t:
1958 NewCall = emitHotColdNewAligned(
1959 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1960 NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold: HotColdVal);
1961 break;
1962 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1963 if (ShouldOptimizeExistingHotColdNew)
1964 NewCall = emitHotColdNewAlignedNoThrow(
1965 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1966 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1967 HotCold: getHotColdHintForExisting(HotCold));
1968 break;
1969 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1970 NewCall = emitHotColdNewAlignedNoThrow(
1971 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1972 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1973 HotCold: HotColdVal);
1974 break;
1975 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1976 if (ShouldOptimizeExistingHotColdNew)
1977 NewCall = emitHotColdNewAlignedNoThrow(
1978 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1979 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1980 HotCold: getHotColdHintForExisting(HotCold));
1981 break;
1982 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1983 NewCall = emitHotColdNewAlignedNoThrow(
1984 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1985 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1986 HotCold: HotColdVal);
1987 break;
1988 case LibFunc_size_returning_new:
1989 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
1990 NewFunc: LibFunc_size_returning_new_hot_cold,
1991 HotCold: HotColdVal);
1992 break;
1993 case LibFunc_size_returning_new_hot_cold:
1994 if (ShouldOptimizeExistingHotColdNew)
1995 NewCall = emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
1996 NewFunc: LibFunc_size_returning_new_hot_cold,
1997 HotCold: getHotColdHintForExisting(HotCold));
1998 break;
1999 case LibFunc_size_returning_new_aligned:
2000 NewCall = emitHotColdSizeReturningNewAligned(
2001 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
2002 NewFunc: LibFunc_size_returning_new_aligned_hot_cold, HotCold: HotColdVal);
2003 break;
2004 case LibFunc_size_returning_new_aligned_hot_cold:
2005 if (ShouldOptimizeExistingHotColdNew)
2006 NewCall = emitHotColdSizeReturningNewAligned(
2007 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
2008 NewFunc: LibFunc_size_returning_new_aligned_hot_cold,
2009 HotCold: getHotColdHintForExisting(HotCold));
2010 break;
2011 default:
2012 return nullptr;
2013 }
2014
2015 if (auto *NewCI = dyn_cast_or_null<Instruction>(Val: NewCall))
2016 NewCI->copyMetadata(SrcInst: *CI);
2017
2018 return NewCall;
2019}
2020
2021//===----------------------------------------------------------------------===//
2022// Math Library Optimizations
2023//===----------------------------------------------------------------------===//
2024
2025// Replace a libcall \p CI with a call to intrinsic \p IID
2026static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
2027 Intrinsic::ID IID) {
2028 Value *NewCall = B.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0), FMFSource: CI);
2029 NewCall->takeName(V: CI);
2030 return copyFlags(Old: *CI, New: NewCall);
2031}
2032
2033static Value *replaceBinaryCall(CallInst *CI, IRBuilderBase &B,
2034 Intrinsic::ID IID) {
2035 Value *NewCall = B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2036 RHS: CI->getArgOperand(i: 1), FMFSource: CI);
2037 NewCall->takeName(V: CI);
2038 return copyFlags(Old: *CI, New: NewCall);
2039}
2040
2041/// Return a variant of Val with float type.
2042/// Currently this works in two cases: If Val is an FPExtension of a float
2043/// value to something bigger, simply return the operand.
2044/// If Val is a ConstantFP but can be converted to a float ConstantFP without
2045/// loss of precision do so.
2046static Value *valueHasFloatPrecision(Value *Val) {
2047 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
2048 Value *Op = Cast->getOperand(i_nocapture: 0);
2049 if (Op->getType()->isFloatTy())
2050 return Op;
2051 }
2052 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
2053 APFloat F = Const->getValueAPF();
2054 bool losesInfo;
2055 (void)F.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
2056 losesInfo: &losesInfo);
2057 if (!losesInfo)
2058 return ConstantFP::get(Context&: Const->getContext(), V: F);
2059 }
2060 return nullptr;
2061}
2062
2063/// Shrink double -> float functions.
2064static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
2065 bool isBinary, const TargetLibraryInfo *TLI,
2066 bool isPrecise = false) {
2067 Function *CalleeFn = CI->getCalledFunction();
2068 if (!CI->getType()->isDoubleTy() || !CalleeFn)
2069 return nullptr;
2070
2071 // If not all the uses of the function are converted to float, then bail out.
2072 // This matters if the precision of the result is more important than the
2073 // precision of the arguments.
2074 if (isPrecise)
2075 for (User *U : CI->users()) {
2076 FPTruncInst *Cast = dyn_cast<FPTruncInst>(Val: U);
2077 if (!Cast || !Cast->getType()->isFloatTy())
2078 return nullptr;
2079 }
2080
2081 // If this is something like 'g((double) float)', convert to 'gf(float)'.
2082 Value *V[2];
2083 V[0] = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2084 V[1] = isBinary ? valueHasFloatPrecision(Val: CI->getArgOperand(i: 1)) : nullptr;
2085 if (!V[0] || (isBinary && !V[1]))
2086 return nullptr;
2087
2088 // If call isn't an intrinsic, check that it isn't within a function with the
2089 // same name as the float version of this call, otherwise the result is an
2090 // infinite loop. For example, from MinGW-w64:
2091 //
2092 // float expf(float val) { return (float) exp((double) val); }
2093 StringRef CalleeName = CalleeFn->getName();
2094 bool IsIntrinsic = CalleeFn->isIntrinsic();
2095 if (!IsIntrinsic) {
2096 StringRef CallerName = CI->getFunction()->getName();
2097 if (CallerName.ends_with(Suffix: 'f') &&
2098 CallerName.size() == (CalleeName.size() + 1) &&
2099 CallerName.starts_with(Prefix: CalleeName))
2100 return nullptr;
2101 }
2102
2103 // Propagate the math semantics from the current function to the new function.
2104 IRBuilderBase::FastMathFlagGuard Guard(B);
2105 B.setFastMathFlags(CI->getFastMathFlags());
2106
2107 // g((double) float) -> (double) gf(float)
2108 Value *R;
2109 if (IsIntrinsic) {
2110 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
2111 R = isBinary ? B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V)
2112 : B.CreateIntrinsic(ID: IID, OverloadTypes: B.getFloatTy(), Args: V[0]);
2113 } else {
2114 AttributeList CallsiteAttrs = CI->getAttributes();
2115 R = isBinary
2116 ? emitBinaryFloatFnCall(Op1: V[0], Op2: V[1], TLI, Name: CalleeName, B,
2117 Attrs: CallsiteAttrs)
2118 : emitUnaryFloatFnCall(Op: V[0], TLI, Name: CalleeName, B, Attrs: CallsiteAttrs);
2119 }
2120 return B.CreateFPExt(V: R, DestTy: B.getDoubleTy());
2121}
2122
2123/// Shrink double -> float for unary functions.
2124static Value *optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2125 const TargetLibraryInfo *TLI,
2126 bool isPrecise = false) {
2127 return optimizeDoubleFP(CI, B, isBinary: false, TLI, isPrecise);
2128}
2129
2130/// Shrink double -> float for binary functions.
2131static Value *optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B,
2132 const TargetLibraryInfo *TLI,
2133 bool isPrecise = false) {
2134 return optimizeDoubleFP(CI, B, isBinary: true, TLI, isPrecise);
2135}
2136
2137/// Shrink double -> float for llvm.sincos.
2138static Value *optimizeSinCosDoubleFP(CallInst *CI, IRBuilderBase &B) {
2139 auto *RetTy = dyn_cast<StructType>(Val: CI->getType());
2140 if (!RetTy || RetTy->getNumElements() != 2 ||
2141 !RetTy->getElementType(N: 0)->getScalarType()->isDoubleTy())
2142 return nullptr;
2143
2144 Value *X = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
2145 if (!X)
2146 if (auto *Ext = dyn_cast<FPExtInst>(Val: CI->getArgOperand(i: 0)))
2147 if (Ext->getOperand(i_nocapture: 0)->getType()->getScalarType()->isFloatTy())
2148 X = Ext->getOperand(i_nocapture: 0);
2149 if (!X)
2150 return nullptr;
2151
2152 for (User *U : CI->users()) {
2153 auto *EV = dyn_cast<ExtractValueInst>(Val: U);
2154 if (!EV)
2155 return nullptr;
2156 for (User *EVU : EV->users()) {
2157 auto *Cast = dyn_cast<FPTruncInst>(Val: EVU);
2158 if (!Cast || !Cast->getType()->getScalarType()->isFloatTy())
2159 return nullptr;
2160 }
2161 }
2162
2163 IRBuilderBase::FastMathFlagGuard Guard(B);
2164 B.setFastMathFlags(CI->getFastMathFlags());
2165
2166 Value *NewCall = B.CreateIntrinsic(ID: Intrinsic::sincos, OverloadTypes: X->getType(), Args: X);
2167 cast<Instruction>(Val: NewCall)->setMetadata(
2168 KindID: LLVMContext::MD_fpmath, Node: CI->getMetadata(KindID: LLVMContext::MD_fpmath));
2169 Value *Res = PoisonValue::get(T: RetTy);
2170 for (unsigned I = 0; I != 2; ++I) {
2171 Value *Ext = B.CreateFPExt(V: B.CreateExtractValue(Agg: NewCall, Idxs: I),
2172 DestTy: RetTy->getElementType(N: I));
2173 Res = B.CreateInsertValue(Agg: Res, Val: Ext, Idxs: I);
2174 }
2175 return Res;
2176}
2177
2178// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
2179Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
2180 Value *Real, *Imag;
2181
2182 if (CI->arg_size() == 1) {
2183
2184 if (!CI->isFast())
2185 return nullptr;
2186
2187 Value *Op = CI->getArgOperand(i: 0);
2188 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
2189
2190 Real = B.CreateExtractValue(Agg: Op, Idxs: 0, Name: "real");
2191 Imag = B.CreateExtractValue(Agg: Op, Idxs: 1, Name: "imag");
2192
2193 } else {
2194 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
2195
2196 Real = CI->getArgOperand(i: 0);
2197 Imag = CI->getArgOperand(i: 1);
2198
2199 // if real or imaginary part is zero, simplify to abs(cimag(z))
2200 // or abs(creal(z))
2201 Value *AbsOp = nullptr;
2202 if (ConstantFP *ConstReal = dyn_cast<ConstantFP>(Val: Real)) {
2203 if (ConstReal->isZero())
2204 AbsOp = Imag;
2205
2206 } else if (ConstantFP *ConstImag = dyn_cast<ConstantFP>(Val: Imag)) {
2207 if (ConstImag->isZero())
2208 AbsOp = Real;
2209 }
2210
2211 if (AbsOp)
2212 return copyFlags(Old: *CI, New: B.CreateFAbs(V: AbsOp, FMFSource: CI, Name: "cabs"));
2213
2214 if (!CI->isFast())
2215 return nullptr;
2216 }
2217
2218 // Propagate fast-math flags from the existing call to new instructions.
2219 Value *RealReal = B.CreateFMulFMF(L: Real, R: Real, FMFSource: CI);
2220 Value *ImagImag = B.CreateFMulFMF(L: Imag, R: Imag, FMFSource: CI);
2221 return copyFlags(
2222 Old: *CI, New: B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt,
2223 Op: B.CreateFAddFMF(L: RealReal, R: ImagImag, FMFSource: CI), FMFSource: CI,
2224 Name: "cabs"));
2225}
2226
2227// Return a properly extended integer (DstWidth bits wide) if the operation is
2228// an itofp.
2229static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
2230 if (isa<SIToFPInst>(Val: I2F) || isa<UIToFPInst>(Val: I2F)) {
2231 Value *Op = cast<Instruction>(Val: I2F)->getOperand(i: 0);
2232 // Make sure that the exponent fits inside an "int" of size DstWidth,
2233 // thus avoiding any range issues that FP has not.
2234 unsigned BitWidth = Op->getType()->getScalarSizeInBits();
2235 if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(Val: I2F))) {
2236 Type *IntTy = Op->getType()->getWithNewBitWidth(NewBitWidth: DstWidth);
2237 return isa<SIToFPInst>(Val: I2F) ? B.CreateSExt(V: Op, DestTy: IntTy)
2238 : B.CreateZExt(V: Op, DestTy: IntTy);
2239 }
2240 }
2241
2242 return nullptr;
2243}
2244
2245/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
2246/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
2247/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
2248Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
2249 Module *M = Pow->getModule();
2250 Value *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2251 Type *Ty = Pow->getType();
2252 bool Ignored;
2253
2254 // Evaluate special cases related to a nested function as the base.
2255
2256 // pow(exp(x), y) -> exp(x * y)
2257 // pow(exp2(x), y) -> exp2(x * y)
2258 // If exp{,2}() is used only once, it is better to fold two transcendental
2259 // math functions into one. If used again, exp{,2}() would still have to be
2260 // called with the original argument, then keep both original transcendental
2261 // functions. However, this transformation is only safe with fully relaxed
2262 // math semantics, since, besides rounding differences, it changes overflow
2263 // and underflow behavior quite dramatically. For example:
2264 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
2265 // Whereas:
2266 // exp(1000 * 0.001) = exp(1)
2267 // TODO: Loosen the requirement for fully relaxed math semantics.
2268 // TODO: Handle exp10() when more targets have it available.
2269 CallInst *BaseFn = dyn_cast<CallInst>(Val: Base);
2270 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
2271 Function *CalleeFn = BaseFn->getCalledFunction();
2272 LibFunc LibFn =
2273 CalleeFn ? TLI->getLibFunc(funcName: CalleeFn->getName()) : NotLibFunc;
2274 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFn)) {
2275 StringRef ExpName;
2276 Intrinsic::ID ID;
2277 Value *ExpFn;
2278 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2279
2280 switch (LibFn) {
2281 default:
2282 return nullptr;
2283 case LibFunc_expf:
2284 case LibFunc_exp:
2285 case LibFunc_expl:
2286 ExpName = TLI->getName(F: LibFunc_exp);
2287 ID = Intrinsic::exp;
2288 LibFnFloat = LibFunc_expf;
2289 LibFnDouble = LibFunc_exp;
2290 LibFnLongDouble = LibFunc_expl;
2291 break;
2292 case LibFunc_exp2f:
2293 case LibFunc_exp2:
2294 case LibFunc_exp2l:
2295 ExpName = TLI->getName(F: LibFunc_exp2);
2296 ID = Intrinsic::exp2;
2297 LibFnFloat = LibFunc_exp2f;
2298 LibFnDouble = LibFunc_exp2;
2299 LibFnLongDouble = LibFunc_exp2l;
2300 break;
2301 }
2302
2303 // Create new exp{,2}() with the product as its argument.
2304 Value *FMul = B.CreateFMul(L: BaseFn->getArgOperand(i: 0), R: Expo, Name: "mul");
2305 ExpFn = BaseFn->doesNotAccessMemory()
2306 ? B.CreateUnaryIntrinsic(ID, Op: FMul, FMFSource: nullptr, Name: ExpName)
2307 : emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFnDouble, FloatFn: LibFnFloat,
2308 LongDoubleFn: LibFnLongDouble, B,
2309 Attrs: BaseFn->getAttributes());
2310
2311 // Since the new exp{,2}() is different from the original one, dead code
2312 // elimination cannot be trusted to remove it, since it may have side
2313 // effects (e.g., errno). When the only consumer for the original
2314 // exp{,2}() is pow(), then it has to be explicitly erased.
2315 substituteInParent(I: BaseFn, With: ExpFn);
2316 return ExpFn;
2317 }
2318 }
2319
2320 // Evaluate special cases related to a constant base.
2321
2322 const APFloat *BaseF;
2323 if (!match(V: Base, P: m_APFloat(Res&: BaseF)))
2324 return nullptr;
2325
2326 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2327
2328 const bool UseIntrinsic = Pow->doesNotAccessMemory();
2329
2330 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2331 if ((UseIntrinsic || !Ty->isVectorTy()) && BaseF->isExactlyValue(V: 2.0) &&
2332 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo)) &&
2333 (UseIntrinsic ||
2334 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2335
2336 // TODO: Shouldn't really need to depend on getIntToFPVal for intrinsic. Can
2337 // just directly use the original integer type.
2338 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize())) {
2339 Constant *One = ConstantFP::get(Ty, V: 1.0);
2340
2341 if (UseIntrinsic) {
2342 return copyFlags(Old: *Pow, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2343 OverloadTypes: {Ty, ExpoI->getType()},
2344 Args: {One, ExpoI}, FMFSource: Pow, Name: "exp2"));
2345 }
2346
2347 return copyFlags(Old: *Pow, New: emitBinaryFloatFnCall(
2348 Op1: One, Op2: ExpoI, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2349 LongDoubleFn: LibFunc_ldexpl, B, Attrs: NoAttrs));
2350 }
2351 }
2352
2353 // pow(2.0 ** n, x) -> exp2(n * x)
2354 if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f, LongDoubleFn: LibFunc_exp2l)) {
2355 APFloat BaseR = APFloat(1.0);
2356 BaseR.convert(ToSemantics: BaseF->getSemantics(), RM: APFloat::rmTowardZero, losesInfo: &Ignored);
2357 BaseR = BaseR / *BaseF;
2358 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2359 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2360 APSInt NI(64, false);
2361 if ((IsInteger || IsReciprocal) &&
2362 NF->convertToInteger(Result&: NI, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2363 APFloat::opOK &&
2364 NI > 1 && NI.isPowerOf2()) {
2365 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2366 Value *FMul = B.CreateFMul(L: Expo, R: ConstantFP::get(Ty, V: N), Name: "mul");
2367 if (Pow->doesNotAccessMemory())
2368 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2369 FMFSource: nullptr, Name: "exp2"));
2370 else
2371 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2372 FloatFn: LibFunc_exp2f,
2373 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2374 }
2375 }
2376
2377 // pow(10.0, x) -> exp10(x)
2378 if (BaseF->isExactlyValue(V: 10.0) &&
2379 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp10, FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l)) {
2380
2381 if (Pow->doesNotAccessMemory()) {
2382 return B.CreateIntrinsic(ID: Intrinsic::exp10, OverloadTypes: {Ty}, Args: {Expo}, FMFSource: Pow, Name: "exp10", OpBundles: {},
2383 SetFn: [Pow](CallInst *CI) { CI->copyIRFlags(V: Pow); });
2384 }
2385
2386 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: Expo, TLI, DoubleFn: LibFunc_exp10,
2387 FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l,
2388 B, Attrs: NoAttrs));
2389 }
2390
2391 // pow(x, y) -> exp2(log2(x) * y)
2392 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2393 !BaseF->isNegative()) {
2394 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2395 // Luckily optimizePow has already handled the x == 1 case.
2396 assert(!match(Base, m_FPOne()) &&
2397 "pow(1.0, y) should have been simplified earlier!");
2398
2399 Value *Log = nullptr;
2400 if (Ty->isFloatTy())
2401 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToFloat()));
2402 else if (Ty->isDoubleTy())
2403 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToDouble()));
2404
2405 if (Log) {
2406 Value *FMul = B.CreateFMul(L: Log, R: Expo, Name: "mul");
2407 if (Pow->doesNotAccessMemory())
2408 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, Op: FMul,
2409 FMFSource: nullptr, Name: "exp2"));
2410 else if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f,
2411 LongDoubleFn: LibFunc_exp2l))
2412 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2413 FloatFn: LibFunc_exp2f,
2414 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2415 }
2416 }
2417
2418 return nullptr;
2419}
2420
2421static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2422 Module *M, IRBuilderBase &B,
2423 const TargetLibraryInfo *TLI) {
2424 // If errno is never set, then use the intrinsic for sqrt().
2425 if (NoErrno)
2426 return B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: V, FMFSource: nullptr, Name: "sqrt");
2427
2428 // Otherwise, use the libcall for sqrt().
2429 if (hasFloatFn(M, TLI, Ty: V->getType(), DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2430 LongDoubleFn: LibFunc_sqrtl))
2431 // TODO: We also should check that the target can in fact lower the sqrt()
2432 // libcall. We currently have no way to ask this question, so we ask if
2433 // the target has a sqrt() libcall, which is not exactly the same.
2434 return emitUnaryFloatFnCall(Op: V, TLI, DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2435 LongDoubleFn: LibFunc_sqrtl, B, Attrs);
2436
2437 return nullptr;
2438}
2439
2440/// Use square root in place of pow(x, +/-0.5).
2441Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2442 Value *Sqrt, *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2443 Module *Mod = Pow->getModule();
2444 Type *Ty = Pow->getType();
2445
2446 const APFloat *ExpoF;
2447 if (!match(V: Expo, P: m_APFloat(Res&: ExpoF)) ||
2448 (!ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)))
2449 return nullptr;
2450
2451 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2452 // so that requires fast-math-flags (afn or reassoc).
2453 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2454 return nullptr;
2455
2456 // If we have a pow() library call (accesses memory) and we can't guarantee
2457 // that the base is not an infinity, give up:
2458 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2459 // errno), but sqrt(-Inf) is required by various standards to set errno.
2460 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2461 !isKnownNeverInfinity(
2462 V: Base, SQ: SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
2463 return nullptr;
2464
2465 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M: Mod, B,
2466 TLI);
2467 if (!Sqrt)
2468 return nullptr;
2469
2470 // Handle signed zero base by expanding to fabs(sqrt(x)).
2471 if (!Pow->hasNoSignedZeros())
2472 Sqrt = B.CreateFAbs(V: Sqrt, FMFSource: nullptr, Name: "abs");
2473
2474 Sqrt = copyFlags(Old: *Pow, New: Sqrt);
2475
2476 // Handle non finite base by expanding to
2477 // (x == -infinity ? +infinity : sqrt(x)).
2478 if (!Pow->hasNoInfs()) {
2479 Value *PosInf = ConstantFP::getInfinity(Ty),
2480 *NegInf = ConstantFP::getInfinity(Ty, Negative: true);
2481 Value *FCmp = B.CreateFCmpOEQ(LHS: Base, RHS: NegInf, Name: "isinf");
2482 Sqrt = B.CreateSelect(C: FCmp, True: PosInf, False: Sqrt);
2483 }
2484
2485 // If the exponent is negative, then get the reciprocal.
2486 if (ExpoF->isNegative())
2487 Sqrt = B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Sqrt, Name: "reciprocal");
2488
2489 return Sqrt;
2490}
2491
2492static Value *createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M,
2493 IRBuilderBase &B) {
2494 Value *Args[] = {Base, Expo};
2495 Type *Types[] = {Base->getType(), Expo->getType()};
2496 return B.CreateIntrinsic(ID: Intrinsic::powi, OverloadTypes: Types, Args);
2497}
2498
2499Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2500 Value *Base = Pow->getArgOperand(i: 0);
2501 Value *Expo = Pow->getArgOperand(i: 1);
2502 Function *Callee = Pow->getCalledFunction();
2503 StringRef Name = Callee->getName();
2504 Type *Ty = Pow->getType();
2505 Module *M = Pow->getModule();
2506 bool AllowApprox = Pow->hasApproxFunc();
2507 bool Ignored;
2508
2509 // Propagate the math semantics from the call to any created instructions.
2510 IRBuilderBase::FastMathFlagGuard Guard(B);
2511 B.setFastMathFlags(Pow->getFastMathFlags());
2512 // Evaluate special cases related to the base.
2513
2514 // pow(1.0, x) -> 1.0
2515 if (match(V: Base, P: m_FPOne()))
2516 return Base;
2517
2518 if (Value *Exp = replacePowWithExp(Pow, B))
2519 return Exp;
2520
2521 // Evaluate special cases related to the exponent.
2522
2523 // pow(x, -1.0) -> 1.0 / x
2524 if (match(V: Expo, P: m_SpecificFP(V: -1.0)))
2525 return B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Base, Name: "reciprocal");
2526
2527 // pow(x, +/-0.0) -> 1.0
2528 if (match(V: Expo, P: m_AnyZeroFP()))
2529 return ConstantFP::get(Ty, V: 1.0);
2530
2531 // pow(x, 1.0) -> x
2532 if (match(V: Expo, P: m_FPOne()))
2533 return Base;
2534
2535 // pow(x, 2.0) -> x * x
2536 if (match(V: Expo, P: m_SpecificFP(V: 2.0)) && Pow->doesNotAccessMemory())
2537 return B.CreateFMul(L: Base, R: Base, Name: "square");
2538
2539 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2540 return Sqrt;
2541
2542 // If we can approximate pow:
2543 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2544 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2545 const APFloat *ExpoF;
2546 if (AllowApprox && match(V: Expo, P: m_APFloat(Res&: ExpoF)) &&
2547 !ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)) {
2548 APFloat ExpoA(abs(X: *ExpoF));
2549 APFloat ExpoI(*ExpoF);
2550 Value *Sqrt = nullptr;
2551 if (!ExpoA.isInteger()) {
2552 APFloat Expo2 = ExpoA;
2553 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2554 // is no floating point exception and the result is an integer, then
2555 // ExpoA == integer + 0.5
2556 if (Expo2.add(RHS: ExpoA, RM: APFloat::rmNearestTiesToEven) != APFloat::opOK)
2557 return nullptr;
2558
2559 if (!Expo2.isInteger())
2560 return nullptr;
2561
2562 if (ExpoI.roundToIntegral(RM: APFloat::rmTowardNegative) !=
2563 APFloat::opInexact)
2564 return nullptr;
2565 if (!ExpoI.isInteger())
2566 return nullptr;
2567 ExpoF = &ExpoI;
2568
2569 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M,
2570 B, TLI);
2571 if (!Sqrt)
2572 return nullptr;
2573 }
2574
2575 // 0.5 fraction is now optionally handled.
2576 // Do pow -> powi for remaining integer exponent
2577 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2578 if (ExpoF->isInteger() &&
2579 ExpoF->convertToInteger(Result&: IntExpo, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2580 APFloat::opOK) {
2581 Value *PowI = copyFlags(
2582 Old: *Pow,
2583 New: createPowWithIntegerExponent(
2584 Base, Expo: ConstantInt::get(Ty: B.getIntNTy(N: TLI->getIntSize()), V: IntExpo),
2585 M, B));
2586
2587 if (PowI && Sqrt)
2588 return B.CreateFMul(L: PowI, R: Sqrt);
2589
2590 return PowI;
2591 }
2592 }
2593
2594 // powf(x, itofp(y)) -> powi(x, y)
2595 // The powi exponent must be a scalar integer, so a vector y is not usable.
2596 if (AllowApprox && !Expo->getType()->isVectorTy() &&
2597 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo))) {
2598 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize()))
2599 return copyFlags(Old: *Pow, New: createPowWithIntegerExponent(Base, Expo: ExpoI, M, B));
2600 }
2601
2602 // Shrink pow() to powf() if the arguments are single precision,
2603 // unless the result is expected to be double precision.
2604 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_pow) &&
2605 hasFloatVersion(M, FuncName: Name)) {
2606 if (Value *Shrunk = optimizeBinaryDoubleFP(CI: Pow, B, TLI, isPrecise: true))
2607 return Shrunk;
2608 }
2609
2610 return nullptr;
2611}
2612
2613Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2614 Module *M = CI->getModule();
2615 Function *Callee = CI->getCalledFunction();
2616 StringRef Name = Callee->getName();
2617 Value *Ret = nullptr;
2618 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_exp2) &&
2619 hasFloatVersion(M, FuncName: Name))
2620 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2621
2622 // If we have an llvm.exp2 intrinsic, emit the llvm.ldexp intrinsic. If we
2623 // have the libcall, emit the libcall.
2624 //
2625 // TODO: In principle we should be able to just always use the intrinsic for
2626 // any doesNotAccessMemory callsite.
2627
2628 const bool UseIntrinsic = Callee->isIntrinsic();
2629 // Bail out for vectors because the code below only expects scalars.
2630 Type *Ty = CI->getType();
2631 if (!UseIntrinsic && Ty->isVectorTy())
2632 return Ret;
2633
2634 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2635 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2636 Value *Op = CI->getArgOperand(i: 0);
2637 if ((isa<SIToFPInst>(Val: Op) || isa<UIToFPInst>(Val: Op)) &&
2638 (UseIntrinsic ||
2639 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2640 if (Value *Exp = getIntToFPVal(I2F: Op, B, DstWidth: TLI->getIntSize())) {
2641 Constant *One = ConstantFP::get(Ty, V: 1.0);
2642
2643 if (UseIntrinsic) {
2644 return copyFlags(Old: *CI, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2645 OverloadTypes: {Ty, Exp->getType()},
2646 Args: {One, Exp}, FMFSource: CI));
2647 }
2648
2649 IRBuilderBase::FastMathFlagGuard Guard(B);
2650 B.setFastMathFlags(CI->getFastMathFlags());
2651 return copyFlags(Old: *CI, New: emitBinaryFloatFnCall(
2652 Op1: One, Op2: Exp, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2653 LongDoubleFn: LibFunc_ldexpl, B, Attrs: AttributeList()));
2654 }
2655 }
2656
2657 return Ret;
2658}
2659
2660Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B,
2661 Intrinsic::ID IID) {
2662 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2663 // the intrinsics for improved optimization (for example, vectorization).
2664 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2665 // From the C standard draft WG14/N1256:
2666 // "Ideally, fmax would be sensitive to the sign of zero, for example
2667 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2668 // might be impractical."
2669 FastMathFlags FMF = CI->getFastMathFlags();
2670 FMF.setNoSignedZeros();
2671 return copyFlags(Old: *CI, New: B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2672 RHS: CI->getArgOperand(i: 1), FMFSource: FMF));
2673}
2674
2675Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2676 Function *LogFn = Log->getCalledFunction();
2677 StringRef LogNm = LogFn->getName();
2678 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2679 Module *Mod = Log->getModule();
2680 Type *Ty = Log->getType();
2681
2682 if (UnsafeFPShrink && hasFloatVersion(M: Mod, FuncName: LogNm))
2683 if (Value *Ret = optimizeUnaryDoubleFP(CI: Log, B, TLI, isPrecise: true))
2684 return Ret;
2685
2686 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2687
2688 // This is only applicable to log(), log2(), log10().
2689 LogLb = TLI->getLibFunc(funcName: LogNm);
2690 if (LogLb != NotLibFunc) {
2691 switch (LogLb) {
2692 case LibFunc_logf:
2693 LogID = Intrinsic::log;
2694 ExpLb = LibFunc_expf;
2695 Exp2Lb = LibFunc_exp2f;
2696 Exp10Lb = LibFunc_exp10f;
2697 PowLb = LibFunc_powf;
2698 break;
2699 case LibFunc_log:
2700 LogID = Intrinsic::log;
2701 ExpLb = LibFunc_exp;
2702 Exp2Lb = LibFunc_exp2;
2703 Exp10Lb = LibFunc_exp10;
2704 PowLb = LibFunc_pow;
2705 break;
2706 case LibFunc_logl:
2707 LogID = Intrinsic::log;
2708 ExpLb = LibFunc_expl;
2709 Exp2Lb = LibFunc_exp2l;
2710 Exp10Lb = LibFunc_exp10l;
2711 PowLb = LibFunc_powl;
2712 break;
2713 case LibFunc_log2f:
2714 LogID = Intrinsic::log2;
2715 ExpLb = LibFunc_expf;
2716 Exp2Lb = LibFunc_exp2f;
2717 Exp10Lb = LibFunc_exp10f;
2718 PowLb = LibFunc_powf;
2719 break;
2720 case LibFunc_log2:
2721 LogID = Intrinsic::log2;
2722 ExpLb = LibFunc_exp;
2723 Exp2Lb = LibFunc_exp2;
2724 Exp10Lb = LibFunc_exp10;
2725 PowLb = LibFunc_pow;
2726 break;
2727 case LibFunc_log2l:
2728 LogID = Intrinsic::log2;
2729 ExpLb = LibFunc_expl;
2730 Exp2Lb = LibFunc_exp2l;
2731 Exp10Lb = LibFunc_exp10l;
2732 PowLb = LibFunc_powl;
2733 break;
2734 case LibFunc_log10f:
2735 LogID = Intrinsic::log10;
2736 ExpLb = LibFunc_expf;
2737 Exp2Lb = LibFunc_exp2f;
2738 Exp10Lb = LibFunc_exp10f;
2739 PowLb = LibFunc_powf;
2740 break;
2741 case LibFunc_log10:
2742 LogID = Intrinsic::log10;
2743 ExpLb = LibFunc_exp;
2744 Exp2Lb = LibFunc_exp2;
2745 Exp10Lb = LibFunc_exp10;
2746 PowLb = LibFunc_pow;
2747 break;
2748 case LibFunc_log10l:
2749 LogID = Intrinsic::log10;
2750 ExpLb = LibFunc_expl;
2751 Exp2Lb = LibFunc_exp2l;
2752 Exp10Lb = LibFunc_exp10l;
2753 PowLb = LibFunc_powl;
2754 break;
2755 default:
2756 return nullptr;
2757 }
2758
2759 // Convert libcall to intrinsic if the value is known > 0.
2760 bool IsKnownNoErrno = Log->hasNoNaNs() && Log->hasNoInfs();
2761 if (!IsKnownNoErrno) {
2762 SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
2763 KnownFPClass Known = computeKnownFPClass(
2764 V: Log->getOperand(i_nocapture: 0),
2765 InterestedClasses: KnownFPClass::OrderedLessThanZeroMask | fcSubnormal, SQ);
2766 Function *F = Log->getParent()->getParent();
2767 const fltSemantics &FltSem = Ty->getScalarType()->getFltSemantics();
2768 IsKnownNoErrno =
2769 Known.cannotBeOrderedLessThanZero() &&
2770 Known.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
2771 }
2772 if (IsKnownNoErrno) {
2773 Value *NewLog = B.CreateUnaryIntrinsic(ID: LogID, Op: Log->getArgOperand(i: 0), FMFSource: Log);
2774 if (auto *I = dyn_cast<Instruction>(Val: NewLog)) {
2775 I->copyMetadata(SrcInst: *Log);
2776 return copyFlags(Old: *Log, New: I);
2777 }
2778 return NewLog;
2779 }
2780 } else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2781 LogID == Intrinsic::log10) {
2782 if (Ty->getScalarType()->isFloatTy()) {
2783 ExpLb = LibFunc_expf;
2784 Exp2Lb = LibFunc_exp2f;
2785 Exp10Lb = LibFunc_exp10f;
2786 PowLb = LibFunc_powf;
2787 } else if (Ty->getScalarType()->isDoubleTy()) {
2788 ExpLb = LibFunc_exp;
2789 Exp2Lb = LibFunc_exp2;
2790 Exp10Lb = LibFunc_exp10;
2791 PowLb = LibFunc_pow;
2792 } else
2793 return nullptr;
2794 } else
2795 return nullptr;
2796
2797 // The earlier call must also be 'fast' in order to do these transforms.
2798 CallInst *Arg = dyn_cast<CallInst>(Val: Log->getArgOperand(i: 0));
2799 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2800 return nullptr;
2801
2802 IRBuilderBase::FastMathFlagGuard Guard(B);
2803 B.setFastMathFlags(FastMathFlags::getFast());
2804
2805 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2806 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2807
2808 // log(pow(x,y)) -> y*log(x)
2809 AttributeList NoAttrs;
2810 if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2811 Value *LogX =
2812 Log->doesNotAccessMemory()
2813 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Arg->getOperand(i_nocapture: 0), FMFSource: nullptr, Name: "log")
2814 : emitUnaryFloatFnCall(Op: Arg->getOperand(i_nocapture: 0), TLI, Name: LogNm, B, Attrs: NoAttrs);
2815 Value *Y = Arg->getArgOperand(i: 1);
2816 // Cast exponent to FP if integer.
2817 if (ArgID == Intrinsic::powi)
2818 Y = B.CreateSIToFP(V: Y, DestTy: Ty, Name: "cast");
2819 Value *MulY = B.CreateFMul(L: Y, R: LogX, Name: "mul");
2820 // Since pow() may have side effects, e.g. errno,
2821 // dead code elimination may not be trusted to remove it.
2822 substituteInParent(I: Arg, With: MulY);
2823 return MulY;
2824 }
2825
2826 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2827 // TODO: There is no exp10() intrinsic yet.
2828 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2829 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
2830 Constant *Eul;
2831 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2832 // FIXME: Add more precise value of e for long double.
2833 Eul = ConstantFP::get(Ty: Log->getType(), V: numbers::e);
2834 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2835 Eul = ConstantFP::get(Ty: Log->getType(), V: 2.0);
2836 else
2837 Eul = ConstantFP::get(Ty: Log->getType(), V: 10.0);
2838 Value *LogE = Log->doesNotAccessMemory()
2839 ? B.CreateUnaryIntrinsic(ID: LogID, Op: Eul, FMFSource: nullptr, Name: "log")
2840 : emitUnaryFloatFnCall(Op: Eul, TLI, Name: LogNm, B, Attrs: NoAttrs);
2841 Value *MulY = B.CreateFMul(L: Arg->getArgOperand(i: 0), R: LogE, Name: "mul");
2842 // Since exp() may have side effects, e.g. errno,
2843 // dead code elimination may not be trusted to remove it.
2844 substituteInParent(I: Arg, With: MulY);
2845 return MulY;
2846 }
2847
2848 return nullptr;
2849}
2850
2851// sqrt(exp(X)) -> exp(X * 0.5)
2852Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2853 if (!CI->hasAllowReassoc())
2854 return nullptr;
2855
2856 Function *SqrtFn = CI->getCalledFunction();
2857 CallInst *Arg = dyn_cast<CallInst>(Val: CI->getArgOperand(i: 0));
2858 if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2859 return nullptr;
2860 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2861 LibFunc ArgLb = TLI->getLibFunc(CB: *Arg);
2862
2863 LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2864
2865 SqrtLb = TLI->getLibFunc(funcName: SqrtFn->getName());
2866 if (SqrtLb != NotLibFunc)
2867 switch (SqrtLb) {
2868 case LibFunc_sqrtf:
2869 ExpLb = LibFunc_expf;
2870 Exp2Lb = LibFunc_exp2f;
2871 Exp10Lb = LibFunc_exp10f;
2872 break;
2873 case LibFunc_sqrt:
2874 ExpLb = LibFunc_exp;
2875 Exp2Lb = LibFunc_exp2;
2876 Exp10Lb = LibFunc_exp10;
2877 break;
2878 case LibFunc_sqrtl:
2879 ExpLb = LibFunc_expl;
2880 Exp2Lb = LibFunc_exp2l;
2881 Exp10Lb = LibFunc_exp10l;
2882 break;
2883 default:
2884 return nullptr;
2885 }
2886 else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2887 if (CI->getType()->getScalarType()->isFloatTy()) {
2888 ExpLb = LibFunc_expf;
2889 Exp2Lb = LibFunc_exp2f;
2890 Exp10Lb = LibFunc_exp10f;
2891 } else if (CI->getType()->getScalarType()->isDoubleTy()) {
2892 ExpLb = LibFunc_exp;
2893 Exp2Lb = LibFunc_exp2;
2894 Exp10Lb = LibFunc_exp10;
2895 } else
2896 return nullptr;
2897 } else
2898 return nullptr;
2899
2900 if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2901 ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2902 return nullptr;
2903
2904 IRBuilderBase::InsertPointGuard Guard(B);
2905 B.SetInsertPoint(Arg);
2906 auto *ExpOperand = Arg->getOperand(i_nocapture: 0);
2907 auto *FMul =
2908 B.CreateFMulFMF(L: ExpOperand, R: ConstantFP::get(Ty: ExpOperand->getType(), V: 0.5),
2909 FMFSource: CI, Name: "merged.sqrt");
2910
2911 Arg->setOperand(i_nocapture: 0, Val_nocapture: FMul);
2912 return Arg;
2913}
2914
2915Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2916 Module *M = CI->getModule();
2917 Function *Callee = CI->getCalledFunction();
2918 Value *Ret = nullptr;
2919 // TODO: Once we have a way (other than checking for the existince of the
2920 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2921 // condition below.
2922 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_sqrtf) &&
2923 (Callee->getName() == "sqrt" ||
2924 Callee->getIntrinsicID() == Intrinsic::sqrt))
2925 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2926
2927 if (Value *Opt = mergeSqrtToExp(CI, B))
2928 return Opt;
2929
2930 if (!CI->isFast())
2931 return Ret;
2932
2933 Instruction *I = dyn_cast<Instruction>(Val: CI->getArgOperand(i: 0));
2934 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2935 return Ret;
2936
2937 // We're looking for a repeated factor in a multiplication tree,
2938 // so we can do this fold: sqrt(x * x) -> fabs(x);
2939 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2940 Value *Op0 = I->getOperand(i: 0);
2941 Value *Op1 = I->getOperand(i: 1);
2942 Value *RepeatOp = nullptr;
2943 Value *OtherOp = nullptr;
2944 if (Op0 == Op1) {
2945 // Simple match: the operands of the multiply are identical.
2946 RepeatOp = Op0;
2947 } else {
2948 // Look for a more complicated pattern: one of the operands is itself
2949 // a multiply, so search for a common factor in that multiply.
2950 // Note: We don't bother looking any deeper than this first level or for
2951 // variations of this pattern because instcombine's visitFMUL and/or the
2952 // reassociation pass should give us this form.
2953 Value *MulOp;
2954 if (match(V: Op0, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2955 cast<Instruction>(Val: Op0)->isFast()) {
2956 // Pattern: sqrt((x * x) * z)
2957 RepeatOp = MulOp;
2958 OtherOp = Op1;
2959 } else if (match(V: Op1, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2960 cast<Instruction>(Val: Op1)->isFast()) {
2961 // Pattern: sqrt(z * (x * x))
2962 RepeatOp = MulOp;
2963 OtherOp = Op0;
2964 }
2965 }
2966 if (!RepeatOp)
2967 return Ret;
2968
2969 // Fast math flags for any created instructions should match the sqrt
2970 // and multiply.
2971
2972 // If we found a repeated factor, hoist it out of the square root and
2973 // replace it with the fabs of that factor.
2974 Value *FabsCall = B.CreateFAbs(V: RepeatOp, FMFSource: I, Name: "fabs");
2975 if (OtherOp) {
2976 // If we found a non-repeated factor, we still need to get its square
2977 // root. We then multiply that by the value that was simplified out
2978 // of the square root calculation.
2979 Value *SqrtCall =
2980 B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, Op: OtherOp, FMFSource: I, Name: "sqrt");
2981 return copyFlags(Old: *CI, New: B.CreateFMulFMF(L: FabsCall, R: SqrtCall, FMFSource: I));
2982 }
2983 return copyFlags(Old: *CI, New: FabsCall);
2984}
2985
2986Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
2987
2988 // fmod(x,y) sets errno if y == 0 or x == +/-inf. frem does not set errno,
2989 // so the fold is valid only when we can prove fmod wouldn't either.
2990 bool IsNoErrno = CI->hasNoNaNs();
2991 if (!IsNoErrno) {
2992 SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
2993 KnownFPClass Known0 = computeKnownFPClass(V: CI->getOperand(i_nocapture: 0), InterestedClasses: fcInf, SQ);
2994 if (Known0.isKnownNeverInfinity()) {
2995 KnownFPClass Known1 =
2996 computeKnownFPClass(V: CI->getOperand(i_nocapture: 1), InterestedClasses: fcZero | fcSubnormal, SQ);
2997 Function *F = CI->getParent()->getParent();
2998 const fltSemantics &FltSem =
2999 CI->getType()->getScalarType()->getFltSemantics();
3000 IsNoErrno = Known1.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
3001 }
3002 }
3003
3004 if (IsNoErrno)
3005 return B.CreateFRemFMF(L: CI->getOperand(i_nocapture: 0), R: CI->getOperand(i_nocapture: 1), FMFSource: CI);
3006 return nullptr;
3007}
3008
3009Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
3010 IRBuilderBase &B) {
3011 Module *M = CI->getModule();
3012 Function *Callee = CI->getCalledFunction();
3013 Value *Ret = nullptr;
3014 StringRef Name = Callee->getName();
3015 if (UnsafeFPShrink &&
3016 (Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
3017 Name == "asinh") &&
3018 hasFloatVersion(M, FuncName: Name))
3019 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
3020
3021 Value *Op1 = CI->getArgOperand(i: 0);
3022 auto *OpC = dyn_cast<CallInst>(Val: Op1);
3023 if (!OpC)
3024 return Ret;
3025
3026 // Both calls must be 'fast' in order to remove them.
3027 if (!CI->isFast() || !OpC->isFast())
3028 return Ret;
3029
3030 // tan(atan(x)) -> x
3031 // atanh(tanh(x)) -> x
3032 // sinh(asinh(x)) -> x
3033 // asinh(sinh(x)) -> x
3034 // cosh(acosh(x)) -> x
3035 Function *F = OpC->getCalledFunction();
3036 LibFunc Func = F ? TLI->getLibFunc(funcName: F->getName()) : NotLibFunc;
3037 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3038 LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
3039 .Case(S: "tan", Value: LibFunc_atan)
3040 .Case(S: "atanh", Value: LibFunc_tanh)
3041 .Case(S: "sinh", Value: LibFunc_asinh)
3042 .Case(S: "cosh", Value: LibFunc_acosh)
3043 .Case(S: "tanf", Value: LibFunc_atanf)
3044 .Case(S: "atanhf", Value: LibFunc_tanhf)
3045 .Case(S: "sinhf", Value: LibFunc_asinhf)
3046 .Case(S: "coshf", Value: LibFunc_acoshf)
3047 .Case(S: "tanl", Value: LibFunc_atanl)
3048 .Case(S: "atanhl", Value: LibFunc_tanhl)
3049 .Case(S: "sinhl", Value: LibFunc_asinhl)
3050 .Case(S: "coshl", Value: LibFunc_acoshl)
3051 .Case(S: "asinh", Value: LibFunc_sinh)
3052 .Case(S: "asinhf", Value: LibFunc_sinhf)
3053 .Case(S: "asinhl", Value: LibFunc_sinhl)
3054 .Default(Value: NotLibFunc); // Used as error value
3055 if (Func == inverseFunc)
3056 Ret = OpC->getArgOperand(i: 0);
3057 }
3058 return Ret;
3059}
3060
3061static bool isTrigLibCall(CallInst *CI) {
3062 // We can only hope to do anything useful if we can ignore things like errno
3063 // and floating-point exceptions.
3064 // We already checked the prototype.
3065 return CI->doesNotThrow() && CI->doesNotAccessMemory();
3066}
3067
3068static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
3069 bool UseFloat, Value *&Sin, Value *&Cos,
3070 Value *&SinCos, const TargetLibraryInfo *TLI) {
3071 Module *M = OrigCallee->getParent();
3072 Type *ArgTy = Arg->getType();
3073 Type *ResTy;
3074 StringRef Name;
3075
3076 Triple T(OrigCallee->getParent()->getTargetTriple());
3077 if (UseFloat) {
3078 Name = "__sincospif_stret";
3079
3080 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
3081 // x86_64 can't use {float, float} since that would be returned in both
3082 // xmm0 and xmm1, which isn't what a real struct would do.
3083 ResTy = T.getArch() == Triple::x86_64
3084 ? static_cast<Type *>(FixedVectorType::get(ElementType: ArgTy, NumElts: 2))
3085 : static_cast<Type *>(StructType::get(elt1: ArgTy, elts: ArgTy));
3086 } else {
3087 Name = "__sincospi_stret";
3088 ResTy = StructType::get(elt1: ArgTy, elts: ArgTy);
3089 }
3090
3091 if (!isLibFuncEmittable(M, TLI, Name))
3092 return false;
3093 LibFunc TheLibFunc = TLI->getLibFunc(funcName: Name);
3094 FunctionCallee Callee = getOrInsertLibFunc(
3095 M, TLI: *TLI, TheLibFunc, AttributeList: OrigCallee->getAttributes(), RetTy: ResTy, Args: ArgTy);
3096
3097 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
3098 // If the argument is an instruction, it must dominate all uses so put our
3099 // sincos call there.
3100 B.SetInsertPoint(TheBB: ArgInst->getParent(), IP: ++ArgInst->getIterator());
3101 } else {
3102 // Otherwise (e.g. for a constant) the beginning of the function is as
3103 // good a place as any.
3104 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
3105 B.SetInsertPoint(TheBB: &EntryBB, IP: EntryBB.begin());
3106 }
3107
3108 SinCos = B.CreateCall(Callee, Args: Arg, Name: "sincospi");
3109
3110 if (SinCos->getType()->isStructTy()) {
3111 Sin = B.CreateExtractValue(Agg: SinCos, Idxs: 0, Name: "sinpi");
3112 Cos = B.CreateExtractValue(Agg: SinCos, Idxs: 1, Name: "cospi");
3113 } else {
3114 Sin = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{0}, Name: "sinpi");
3115 Cos = B.CreateExtractElement(Vec: SinCos, Idx: uint64_t{1}, Name: "cospi");
3116 }
3117
3118 return true;
3119}
3120
3121static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
3122 IRBuilderBase &B) {
3123 Value *X;
3124 Value *Src = CI->getArgOperand(i: 0);
3125
3126 if (match(V: Src, P: m_OneUse(SubPattern: m_FNeg(X: m_Value(V&: X))))) {
3127 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3128 auto *CallInst = copyFlags(Old: *CI, New: Call);
3129 if (IsEven) {
3130 // Even function: f(-x) = f(x)
3131 return CallInst;
3132 }
3133 // Odd function: f(-x) = -f(x)
3134 return B.CreateFNegFMF(V: CallInst, FMFSource: CI);
3135 }
3136
3137 // Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
3138 if (IsEven && (match(V: Src, P: m_FAbs(Op0: m_Value(V&: X))) ||
3139 match(V: Src, P: m_CopySign(Op0: m_Value(V&: X), Op1: m_Value())))) {
3140 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X}, /*FMFSource=*/CI);
3141 return copyFlags(Old: *CI, New: Call);
3142 }
3143
3144 return nullptr;
3145}
3146
3147Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
3148 IRBuilderBase &B) {
3149 switch (Func) {
3150 case LibFunc_cos:
3151 case LibFunc_cosf:
3152 case LibFunc_cosl:
3153
3154 case LibFunc_cosh:
3155 case LibFunc_coshf:
3156 case LibFunc_coshl:
3157 return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
3158
3159 case LibFunc_sin:
3160 case LibFunc_sinf:
3161 case LibFunc_sinl:
3162
3163 case LibFunc_sinh:
3164 case LibFunc_sinhf:
3165 case LibFunc_sinhl:
3166
3167 case LibFunc_tan:
3168 case LibFunc_tanf:
3169 case LibFunc_tanl:
3170
3171 case LibFunc_tanh:
3172 case LibFunc_tanhf:
3173 case LibFunc_tanhl:
3174
3175 case LibFunc_erf:
3176 case LibFunc_erff:
3177 case LibFunc_erfl:
3178 return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
3179
3180 default:
3181 return nullptr;
3182 }
3183}
3184
3185Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
3186 // Make sure the prototype is as expected, otherwise the rest of the
3187 // function is probably invalid and likely to abort.
3188 if (!isTrigLibCall(CI))
3189 return nullptr;
3190
3191 Value *Arg = CI->getArgOperand(i: 0);
3192 if (isa<ConstantData>(Val: Arg))
3193 return nullptr;
3194
3195 SmallVector<CallInst *, 1> SinCalls;
3196 SmallVector<CallInst *, 1> CosCalls;
3197 SmallVector<CallInst *, 1> SinCosCalls;
3198
3199 bool IsFloat = Arg->getType()->isFloatTy();
3200
3201 // Look for all compatible sinpi, cospi and sincospi calls with the same
3202 // argument. If there are enough (in some sense) we can make the
3203 // substitution.
3204 Function *F = CI->getFunction();
3205 for (User *U : Arg->users())
3206 classifyArgUse(Val: U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
3207
3208 // It's only worthwhile if both sinpi and cospi are actually used.
3209 if (SinCalls.empty() || CosCalls.empty())
3210 return nullptr;
3211
3212 Value *Sin, *Cos, *SinCos;
3213 if (!insertSinCosCall(B, OrigCallee: CI->getCalledFunction(), Arg, UseFloat: IsFloat, Sin, Cos,
3214 SinCos, TLI))
3215 return nullptr;
3216
3217 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
3218 Value *Res) {
3219 for (CallInst *C : Calls)
3220 replaceAllUsesWith(I: C, With: Res);
3221 };
3222
3223 replaceTrigInsts(SinCalls, Sin);
3224 replaceTrigInsts(CosCalls, Cos);
3225 replaceTrigInsts(SinCosCalls, SinCos);
3226
3227 return IsSin ? Sin : Cos;
3228}
3229
3230void LibCallSimplifier::classifyArgUse(
3231 Value *Val, Function *F, bool IsFloat,
3232 SmallVectorImpl<CallInst *> &SinCalls,
3233 SmallVectorImpl<CallInst *> &CosCalls,
3234 SmallVectorImpl<CallInst *> &SinCosCalls) {
3235 auto *CI = dyn_cast<CallInst>(Val);
3236 if (!CI || CI->use_empty())
3237 return;
3238
3239 // Don't consider calls in other functions.
3240 if (CI->getFunction() != F)
3241 return;
3242
3243 Module *M = CI->getModule();
3244 Function *Callee = CI->getCalledFunction();
3245 LibFunc Func = Callee ? TLI->getLibFunc(FDecl: *Callee) : NotLibFunc;
3246 if (!isLibFuncEmittable(M, TLI, TheLibFunc: Func) || !isTrigLibCall(CI))
3247 return;
3248
3249 if (IsFloat) {
3250 if (Func == LibFunc_sinpif)
3251 SinCalls.push_back(Elt: CI);
3252 else if (Func == LibFunc_cospif)
3253 CosCalls.push_back(Elt: CI);
3254 else if (Func == LibFunc_sincospif_stret)
3255 SinCosCalls.push_back(Elt: CI);
3256 } else {
3257 if (Func == LibFunc_sinpi)
3258 SinCalls.push_back(Elt: CI);
3259 else if (Func == LibFunc_cospi)
3260 CosCalls.push_back(Elt: CI);
3261 else if (Func == LibFunc_sincospi_stret)
3262 SinCosCalls.push_back(Elt: CI);
3263 }
3264}
3265
3266/// Constant folds remquo
3267Value *LibCallSimplifier::optimizeRemquo(CallInst *CI, IRBuilderBase &B) {
3268 const APFloat *X, *Y;
3269 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3270 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3271 return nullptr;
3272
3273 APFloat::opStatus Status;
3274 APFloat Quot = *X;
3275 Status = Quot.divide(RHS: *Y, RM: APFloat::rmNearestTiesToEven);
3276 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3277 return nullptr;
3278 APFloat Rem = *X;
3279 if (Rem.remainder(RHS: *Y) != APFloat::opOK)
3280 return nullptr;
3281
3282 // TODO: We can only keep at least the three of the last bits of x/y
3283 unsigned IntBW = TLI->getIntSize();
3284 APSInt QuotInt(IntBW, /*isUnsigned=*/false);
3285 bool IsExact;
3286 Status =
3287 Quot.convertToInteger(Result&: QuotInt, RM: APFloat::rmNearestTiesToEven, IsExact: &IsExact);
3288 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3289 return nullptr;
3290
3291 B.CreateAlignedStore(
3292 Val: ConstantInt::getSigned(Ty: B.getIntNTy(N: IntBW), V: QuotInt.getExtValue()),
3293 Ptr: CI->getArgOperand(i: 2), Align: CI->getParamAlign(ArgNo: 2));
3294 return ConstantFP::get(Ty: CI->getType(), V: Rem);
3295}
3296
3297/// Constant folds fdim
3298Value *LibCallSimplifier::optimizeFdim(CallInst *CI, IRBuilderBase &B) {
3299 // Cannot perform the fold unless the call has attribute memory(none)
3300 if (!CI->doesNotAccessMemory())
3301 return nullptr;
3302
3303 // TODO : Handle undef values
3304 // Propagate poison if any
3305 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 0)))
3306 return CI->getArgOperand(i: 0);
3307 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 1)))
3308 return CI->getArgOperand(i: 1);
3309
3310 const APFloat *X, *Y;
3311 // Check if both values are constants
3312 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3313 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3314 return nullptr;
3315
3316 // C99 fdim(x, y) = (x > y) ? x - y : +0.
3317 if (X->compare(RHS: *Y) != APFloat::cmpGreaterThan && !X->isNaN() && !Y->isNaN())
3318 return ConstantFP::getZero(Ty: CI->getType());
3319 APFloat Difference = *X;
3320 Difference.subtract(RHS: *Y, RM: RoundingMode::NearestTiesToEven);
3321 return ConstantFP::get(Ty: CI->getType(), V: Difference);
3322}
3323
3324//===----------------------------------------------------------------------===//
3325// Integer Library Call Optimizations
3326//===----------------------------------------------------------------------===//
3327
3328Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
3329 // All variants of ffs return int which need not be 32 bits wide.
3330 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
3331 Type *RetType = CI->getType();
3332 Value *Op = CI->getArgOperand(i: 0);
3333 Type *ArgType = Op->getType();
3334 Value *V = B.CreateIntrinsic(ID: Intrinsic::cttz, OverloadTypes: {ArgType}, Args: {Op, B.getTrue()},
3335 FMFSource: nullptr, Name: "cttz");
3336 V = B.CreateAdd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 1));
3337 V = B.CreateIntCast(V, DestTy: RetType, isSigned: false);
3338
3339 Value *Cond = B.CreateICmpNE(LHS: Op, RHS: Constant::getNullValue(Ty: ArgType));
3340 return B.CreateSelect(C: Cond, True: V, False: ConstantInt::get(Ty: RetType, V: 0));
3341}
3342
3343Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
3344 // All variants of fls return int which need not be 32 bits wide.
3345 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
3346 Value *Op = CI->getArgOperand(i: 0);
3347 Type *ArgType = Op->getType();
3348 Value *V = B.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {ArgType}, Args: {Op, B.getFalse()},
3349 FMFSource: nullptr, Name: "ctlz");
3350 V = B.CreateSub(LHS: ConstantInt::get(Ty: V->getType(), V: ArgType->getIntegerBitWidth()),
3351 RHS: V);
3352 return B.CreateIntCast(V, DestTy: CI->getType(), isSigned: false);
3353}
3354
3355Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
3356 // abs(x) -> x <s 0 ? -x : x
3357 // The negation has 'nsw' because abs of INT_MIN is undefined.
3358 Value *X = CI->getArgOperand(i: 0);
3359 Value *IsNeg = B.CreateIsNeg(Arg: X);
3360 Value *NegX = B.CreateNSWNeg(V: X, Name: "neg");
3361 return B.CreateSelect(C: IsNeg, True: NegX, False: X);
3362}
3363
3364Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
3365 // isdigit(c) -> (c-'0') <u 10
3366 Value *Op = CI->getArgOperand(i: 0);
3367 Type *ArgType = Op->getType();
3368 Op = B.CreateSub(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: '0'), Name: "isdigittmp");
3369 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 10), Name: "isdigit");
3370 return B.CreateZExt(V: Op, DestTy: CI->getType());
3371}
3372
3373Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
3374 // isascii(c) -> c <u 128
3375 Value *Op = CI->getArgOperand(i: 0);
3376 Type *ArgType = Op->getType();
3377 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 128), Name: "isascii");
3378 return B.CreateZExt(V: Op, DestTy: CI->getType());
3379}
3380
3381Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
3382 // toascii(c) -> c & 0x7f
3383 return B.CreateAnd(LHS: CI->getArgOperand(i: 0),
3384 RHS: ConstantInt::get(Ty: CI->getType(), V: 0x7F));
3385}
3386
3387// Fold calls to atoi, atol, and atoll.
3388Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
3389 StringRef Str;
3390 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3391 return nullptr;
3392
3393 return convertStrToInt(CI, Str, EndPtr: nullptr, Base: 10, /*AsSigned=*/true, B);
3394}
3395
3396// Fold calls to strtol, strtoll, strtoul, and strtoull.
3397Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
3398 bool AsSigned) {
3399 Value *EndPtr = CI->getArgOperand(i: 1);
3400 if (isa<ConstantPointerNull>(Val: EndPtr)) {
3401 // With a null EndPtr, this function won't capture the main argument.
3402 // It would be readonly too, except that it still may write to errno.
3403 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
3404 CI: CaptureInfo::none()));
3405 EndPtr = nullptr;
3406 } else if (!isKnownNonZero(V: EndPtr, Q: DL))
3407 return nullptr;
3408
3409 StringRef Str;
3410 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3411 return nullptr;
3412
3413 if (ConstantInt *CInt = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2))) {
3414 return convertStrToInt(CI, Str, EndPtr, Base: CInt->getSExtValue(), AsSigned, B);
3415 }
3416
3417 return nullptr;
3418}
3419
3420//===----------------------------------------------------------------------===//
3421// Formatting and IO Library Call Optimizations
3422//===----------------------------------------------------------------------===//
3423
3424static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
3425
3426Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3427 int StreamArg) {
3428 Function *Callee = CI->getCalledFunction();
3429 // Error reporting calls should be cold, mark them as such.
3430 // This applies even to non-builtin calls: it is only a hint and applies to
3431 // functions that the frontend might not understand as builtins.
3432
3433 // This heuristic was suggested in:
3434 // Improving Static Branch Prediction in a Compiler
3435 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3436 // Proceedings of PACT'98, Oct. 1998, IEEE
3437 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
3438 isReportingError(Callee, CI, StreamArg)) {
3439 CI->addFnAttr(Kind: Attribute::Cold);
3440 }
3441
3442 return nullptr;
3443}
3444
3445static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3446 if (!Callee || !Callee->isDeclaration())
3447 return false;
3448
3449 if (StreamArg < 0)
3450 return true;
3451
3452 // These functions might be considered cold, but only if their stream
3453 // argument is stderr.
3454
3455 if (StreamArg >= (int)CI->arg_size())
3456 return false;
3457 LoadInst *LI = dyn_cast<LoadInst>(Val: CI->getArgOperand(i: StreamArg));
3458 if (!LI)
3459 return false;
3460 GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: LI->getPointerOperand());
3461 if (!GV || !GV->isDeclaration())
3462 return false;
3463 return GV->getName() == "stderr";
3464}
3465
3466Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3467 // Check for a fixed format string.
3468 StringRef FormatStr;
3469 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: FormatStr))
3470 return nullptr;
3471
3472 // Empty format string -> noop.
3473 if (FormatStr.empty()) // Tolerate printf's declared void.
3474 return CI->use_empty() ? (Value *)CI : ConstantInt::get(Ty: CI->getType(), V: 0);
3475
3476 // Do not do any of the following transformations if the printf return value
3477 // is used, in general the printf return value is not compatible with either
3478 // putchar() or puts().
3479 if (!CI->use_empty())
3480 return nullptr;
3481
3482 Type *IntTy = CI->getType();
3483 // printf("x") -> putchar('x'), even for "%" and "%%".
3484 if (FormatStr.size() == 1 || FormatStr == "%%") {
3485 // Convert the character to unsigned char before passing it to putchar
3486 // to avoid host-specific sign extension in the IR. Putchar converts
3487 // it to unsigned char regardless.
3488 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)FormatStr[0]);
3489 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3490 }
3491
3492 // Try to remove call or emit putchar/puts.
3493 if (FormatStr == "%s" && CI->arg_size() > 1) {
3494 StringRef OperandStr;
3495 if (!getConstantStringInfo(V: CI->getOperand(i_nocapture: 1), Str&: OperandStr))
3496 return nullptr;
3497 // printf("%s", "") --> NOP
3498 if (OperandStr.empty())
3499 return (Value *)CI;
3500 // printf("%s", "a") --> putchar('a')
3501 if (OperandStr.size() == 1) {
3502 // Convert the character to unsigned char before passing it to putchar
3503 // to avoid host-specific sign extension in the IR. Putchar converts
3504 // it to unsigned char regardless.
3505 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)OperandStr[0]);
3506 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3507 }
3508 // printf("%s", str"\n") --> puts(str)
3509 if (OperandStr.back() == '\n') {
3510 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3511 return nullptr;
3512 OperandStr = OperandStr.drop_back();
3513 Value *GV = B.CreateGlobalString(Str: OperandStr, Name: "str");
3514 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3515 }
3516 return nullptr;
3517 }
3518
3519 // printf("foo\n") --> puts("foo")
3520 if (FormatStr.back() == '\n' &&
3521 !FormatStr.contains(C: '%')) { // No format characters.
3522 if (!isLibFuncEmittable(M: CI->getModule(), TLI, TheLibFunc: LibFunc_puts))
3523 return nullptr;
3524 // Create a string literal with no \n on it. We expect the constant merge
3525 // pass to be run after this pass, to merge duplicate strings.
3526 FormatStr = FormatStr.drop_back();
3527 Value *GV = B.CreateGlobalString(Str: FormatStr, Name: "str");
3528 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3529 }
3530
3531 // Optimize specific format strings.
3532 // printf("%c", chr) --> putchar(chr)
3533 if (FormatStr == "%c" && CI->arg_size() > 1 &&
3534 CI->getArgOperand(i: 1)->getType()->isIntegerTy()) {
3535 // Convert the argument to the type expected by putchar, i.e., int, which
3536 // need not be 32 bits wide but which is the same as printf's return type.
3537 Value *IntChar = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: IntTy, isSigned: false);
3538 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3539 }
3540
3541 // printf("%s\n", str) --> puts(str)
3542 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3543 CI->getArgOperand(i: 1)->getType()->isPointerTy())
3544 return copyFlags(Old: *CI, New: emitPutS(Str: CI->getArgOperand(i: 1), B, TLI));
3545 return nullptr;
3546}
3547
3548Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3549
3550 Module *M = CI->getModule();
3551 Function *Callee = CI->getCalledFunction();
3552 FunctionType *FT = Callee->getFunctionType();
3553 if (Value *V = optimizePrintFString(CI, B)) {
3554 return V;
3555 }
3556
3557 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3558
3559 // printf(format, ...) -> iprintf(format, ...) if no floating point
3560 // arguments.
3561 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_iprintf) &&
3562 !callHasFloatingPointArgument(CI)) {
3563 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_iprintf, T: FT,
3564 AttributeList: Callee->getAttributes());
3565 CallInst *New = cast<CallInst>(Val: CI->clone());
3566 New->setCalledFunction(IPrintFFn);
3567 B.Insert(I: New);
3568 return New;
3569 }
3570
3571 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3572 // arguments.
3573 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_printf) &&
3574 !callHasFP128Argument(CI)) {
3575 auto SmallPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_printf, T: FT,
3576 AttributeList: Callee->getAttributes());
3577 CallInst *New = cast<CallInst>(Val: CI->clone());
3578 New->setCalledFunction(SmallPrintFFn);
3579 B.Insert(I: New);
3580 return New;
3581 }
3582
3583 return nullptr;
3584}
3585
3586Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3587 IRBuilderBase &B) {
3588 // Check for a fixed format string.
3589 StringRef FormatStr;
3590 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3591 return nullptr;
3592
3593 // If we just have a format string (nothing else crazy) transform it.
3594 Value *Dest = CI->getArgOperand(i: 0);
3595 if (CI->arg_size() == 2) {
3596 // Make sure there's no % in the constant array. We could try to handle
3597 // %% -> % in the future if we cared.
3598 if (FormatStr.contains(C: '%'))
3599 return nullptr; // we found a format specifier, bail out.
3600
3601 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3602 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1),
3603 // Copy the null byte.
3604 Size: TLI->getAsSizeT(V: FormatStr.size() + 1, M: *CI->getModule()));
3605 return ConstantInt::get(Ty: CI->getType(), V: FormatStr.size());
3606 }
3607
3608 // The remaining optimizations require the format string to be "%s" or "%c"
3609 // and have an extra operand.
3610 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3611 return nullptr;
3612
3613 // Decode the second character of the format string.
3614 if (FormatStr[1] == 'c') {
3615 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3616 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3617 return nullptr;
3618 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 2), DestTy: B.getInt8Ty(), Name: "char");
3619 Value *Ptr = Dest;
3620 B.CreateStore(Val: V, Ptr);
3621 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3622 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3623
3624 return ConstantInt::get(Ty: CI->getType(), V: 1);
3625 }
3626
3627 if (FormatStr[1] == 's') {
3628 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3629 // strlen(str)+1)
3630 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3631 return nullptr;
3632
3633 if (CI->use_empty())
3634 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3635 return copyFlags(Old: *CI, New: emitStrCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI));
3636
3637 uint64_t SrcLen = GetStringLength(V: CI->getArgOperand(i: 2));
3638 if (SrcLen) {
3639 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1),
3640 Size: TLI->getAsSizeT(V: SrcLen, M: *CI->getModule()));
3641 // Returns total number of characters written without null-character.
3642 return ConstantInt::get(Ty: CI->getType(), V: SrcLen - 1);
3643 } else if (Value *V = emitStpCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI)) {
3644 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3645 Value *PtrDiff = B.CreatePtrDiff(LHS: V, RHS: Dest);
3646 return B.CreateIntCast(V: PtrDiff, DestTy: CI->getType(), isSigned: false);
3647 }
3648
3649 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3650 QueryType: PGSOQueryType::IRPass))
3651 return nullptr;
3652
3653 Value *Len = emitStrLen(Ptr: CI->getArgOperand(i: 2), B, DL, TLI);
3654 if (!Len)
3655 return nullptr;
3656 Value *IncLen =
3657 B.CreateAdd(LHS: Len, RHS: ConstantInt::get(Ty: Len->getType(), V: 1), Name: "leninc");
3658 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1), Size: IncLen);
3659
3660 // The sprintf result is the unincremented number of bytes in the string.
3661 return B.CreateIntCast(V: Len, DestTy: CI->getType(), isSigned: false);
3662 }
3663 return nullptr;
3664}
3665
3666Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3667 Module *M = CI->getModule();
3668 Function *Callee = CI->getCalledFunction();
3669 FunctionType *FT = Callee->getFunctionType();
3670 if (Value *V = optimizeSPrintFString(CI, B)) {
3671 return V;
3672 }
3673
3674 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
3675
3676 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3677 // point arguments.
3678 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_siprintf) &&
3679 !callHasFloatingPointArgument(CI)) {
3680 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_siprintf,
3681 T: FT, AttributeList: Callee->getAttributes());
3682 CallInst *New = cast<CallInst>(Val: CI->clone());
3683 New->setCalledFunction(SIPrintFFn);
3684 B.Insert(I: New);
3685 return New;
3686 }
3687
3688 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3689 // floating point arguments.
3690 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_sprintf) &&
3691 !callHasFP128Argument(CI)) {
3692 auto SmallSPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_sprintf, T: FT,
3693 AttributeList: Callee->getAttributes());
3694 CallInst *New = cast<CallInst>(Val: CI->clone());
3695 New->setCalledFunction(SmallSPrintFFn);
3696 B.Insert(I: New);
3697 return New;
3698 }
3699
3700 return nullptr;
3701}
3702
3703// Transform an snprintf call CI with the bound N to format the string Str
3704// either to a call to memcpy, or to single character a store, or to nothing,
3705// and fold the result to a constant. A nonnull StrArg refers to the string
3706// argument being formatted. Otherwise the call is one with N < 2 and
3707// the "%c" directive to format a single character.
3708Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3709 StringRef Str, uint64_t N,
3710 IRBuilderBase &B) {
3711 assert(StrArg || (N < 2 && Str.size() == 1));
3712
3713 unsigned IntBits = TLI->getIntSize();
3714 uint64_t IntMax = maxIntN(N: IntBits);
3715 if (Str.size() > IntMax)
3716 // Bail if the string is longer than INT_MAX. POSIX requires
3717 // implementations to set errno to EOVERFLOW in this case, in
3718 // addition to when N is larger than that (checked by the caller).
3719 return nullptr;
3720
3721 Value *StrLen = ConstantInt::get(Ty: CI->getType(), V: Str.size());
3722 if (N == 0)
3723 return StrLen;
3724
3725 // Set to the number of bytes to copy fron StrArg which is also
3726 // the offset of the terinating nul.
3727 uint64_t NCopy;
3728 if (N > Str.size())
3729 // Copy the full string, including the terminating nul (which must
3730 // be present regardless of the bound).
3731 NCopy = Str.size() + 1;
3732 else
3733 NCopy = N - 1;
3734
3735 Value *DstArg = CI->getArgOperand(i: 0);
3736 if (NCopy && StrArg)
3737 // Transform the call to lvm.memcpy(dst, fmt, N).
3738 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst: DstArg, DstAlign: Align(1), Src: StrArg, SrcAlign: Align(1),
3739 Size: TLI->getAsSizeT(V: NCopy, M: *CI->getModule())));
3740
3741 if (N > Str.size())
3742 // Return early when the whole format string, including the final nul,
3743 // has been copied.
3744 return StrLen;
3745
3746 // Otherwise, when truncating the string append a terminating nul.
3747 Type *Int8Ty = B.getInt8Ty();
3748 Value *NulOff = B.getIntN(N: IntBits, C: NCopy);
3749 Value *DstEnd = B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: DstArg, IdxList: NulOff, Name: "endptr");
3750 B.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: 0), Ptr: DstEnd);
3751 return StrLen;
3752}
3753
3754Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3755 IRBuilderBase &B) {
3756 // Check for size
3757 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3758 if (!Size)
3759 return nullptr;
3760
3761 uint64_t N = Size->getZExtValue();
3762 uint64_t IntMax = maxIntN(N: TLI->getIntSize());
3763 if (N > IntMax)
3764 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3765 // to set errno to EOVERFLOW in this case.
3766 return nullptr;
3767
3768 Value *DstArg = CI->getArgOperand(i: 0);
3769 Value *FmtArg = CI->getArgOperand(i: 2);
3770
3771 // Check for a fixed format string.
3772 StringRef FormatStr;
3773 if (!getConstantStringInfo(V: FmtArg, Str&: FormatStr))
3774 return nullptr;
3775
3776 // If we just have a format string (nothing else crazy) transform it.
3777 if (CI->arg_size() == 3) {
3778 if (FormatStr.contains(C: '%'))
3779 // Bail if the format string contains a directive and there are
3780 // no arguments. We could handle "%%" in the future.
3781 return nullptr;
3782
3783 return emitSnPrintfMemCpy(CI, StrArg: FmtArg, Str: FormatStr, N, B);
3784 }
3785
3786 // The remaining optimizations require the format string to be "%s" or "%c"
3787 // and have an extra operand.
3788 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3789 return nullptr;
3790
3791 // Decode the second character of the format string.
3792 if (FormatStr[1] == 'c') {
3793 if (N <= 1) {
3794 // Use an arbitary string of length 1 to transform the call into
3795 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3796 // to one.
3797 StringRef CharStr("*");
3798 return emitSnPrintfMemCpy(CI, StrArg: nullptr, Str: CharStr, N, B);
3799 }
3800
3801 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3802 if (!CI->getArgOperand(i: 3)->getType()->isIntegerTy())
3803 return nullptr;
3804 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 3), DestTy: B.getInt8Ty(), Name: "char");
3805 Value *Ptr = DstArg;
3806 B.CreateStore(Val: V, Ptr);
3807 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3808 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3809 return ConstantInt::get(Ty: CI->getType(), V: 1);
3810 }
3811
3812 if (FormatStr[1] != 's')
3813 return nullptr;
3814
3815 Value *StrArg = CI->getArgOperand(i: 3);
3816 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3817 StringRef Str;
3818 if (!getConstantStringInfo(V: StrArg, Str))
3819 return nullptr;
3820
3821 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3822}
3823
3824Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3825 if (Value *V = optimizeSnPrintFString(CI, B)) {
3826 return V;
3827 }
3828
3829 if (isKnownNonZero(V: CI->getOperand(i_nocapture: 1), Q: DL))
3830 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3831 return nullptr;
3832}
3833
3834Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3835 IRBuilderBase &B) {
3836 optimizeErrorReporting(CI, B, StreamArg: 0);
3837
3838 // All the optimizations depend on the format string.
3839 StringRef FormatStr;
3840 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3841 return nullptr;
3842
3843 // Do not do any of the following transformations if the fprintf return
3844 // value is used, in general the fprintf return value is not compatible
3845 // with fwrite(), fputc() or fputs().
3846 if (!CI->use_empty())
3847 return nullptr;
3848
3849 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3850 if (CI->arg_size() == 2) {
3851 // Could handle %% -> % if we cared.
3852 if (FormatStr.contains(C: '%'))
3853 return nullptr; // We found a format specifier.
3854
3855 return copyFlags(
3856 Old: *CI, New: emitFWrite(Ptr: CI->getArgOperand(i: 1),
3857 Size: TLI->getAsSizeT(V: FormatStr.size(), M: *CI->getModule()),
3858 File: CI->getArgOperand(i: 0), B, DL, TLI));
3859 }
3860
3861 // The remaining optimizations require the format string to be "%s" or "%c"
3862 // and have an extra operand.
3863 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3864 return nullptr;
3865
3866 // Decode the second character of the format string.
3867 if (FormatStr[1] == 'c') {
3868 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3869 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3870 return nullptr;
3871 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3872 Value *V = B.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntTy, /*isSigned*/ true,
3873 Name: "chari");
3874 return copyFlags(Old: *CI, New: emitFPutC(Char: V, File: CI->getArgOperand(i: 0), B, TLI));
3875 }
3876
3877 if (FormatStr[1] == 's') {
3878 // fprintf(F, "%s", str) --> fputs(str, F)
3879 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3880 return nullptr;
3881 return copyFlags(
3882 Old: *CI, New: emitFPutS(Str: CI->getArgOperand(i: 2), File: CI->getArgOperand(i: 0), B, TLI));
3883 }
3884 return nullptr;
3885}
3886
3887Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3888 Module *M = CI->getModule();
3889 Function *Callee = CI->getCalledFunction();
3890 FunctionType *FT = Callee->getFunctionType();
3891 if (Value *V = optimizeFPrintFString(CI, B)) {
3892 return V;
3893 }
3894
3895 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3896 // floating point arguments.
3897 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_fiprintf) &&
3898 !callHasFloatingPointArgument(CI)) {
3899 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_fiprintf,
3900 T: FT, AttributeList: Callee->getAttributes());
3901 CallInst *New = cast<CallInst>(Val: CI->clone());
3902 New->setCalledFunction(FIPrintFFn);
3903 B.Insert(I: New);
3904 return New;
3905 }
3906
3907 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3908 // 128-bit floating point arguments.
3909 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_fprintf) &&
3910 !callHasFP128Argument(CI)) {
3911 auto SmallFPrintFFn =
3912 getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_fprintf, T: FT,
3913 AttributeList: Callee->getAttributes());
3914 CallInst *New = cast<CallInst>(Val: CI->clone());
3915 New->setCalledFunction(SmallFPrintFFn);
3916 B.Insert(I: New);
3917 return New;
3918 }
3919
3920 return nullptr;
3921}
3922
3923Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3924 optimizeErrorReporting(CI, B, StreamArg: 3);
3925
3926 // Get the element size and count.
3927 ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3928 ConstantInt *CountC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
3929 if (SizeC && CountC) {
3930 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3931
3932 // If this is writing zero records, remove the call (it's a noop).
3933 if (Bytes == 0)
3934 return ConstantInt::get(Ty: CI->getType(), V: 0);
3935
3936 // If this is writing one byte, turn it into fputc.
3937 // This optimisation is only valid, if the return value is unused.
3938 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
3939 Value *Char = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0), Name: "char");
3940 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3941 Value *Cast = B.CreateIntCast(V: Char, DestTy: IntTy, /*isSigned*/ true, Name: "chari");
3942 Value *NewCI = emitFPutC(Char: Cast, File: CI->getArgOperand(i: 3), B, TLI);
3943 return NewCI ? ConstantInt::get(Ty: CI->getType(), V: 1) : nullptr;
3944 }
3945 }
3946
3947 return nullptr;
3948}
3949
3950Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
3951 optimizeErrorReporting(CI, B, StreamArg: 1);
3952
3953 // Don't rewrite fputs to fwrite when optimising for size because fwrite
3954 // requires more arguments and thus extra MOVs are required.
3955 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3956 QueryType: PGSOQueryType::IRPass))
3957 return nullptr;
3958
3959 // We can't optimize if return value is used.
3960 if (!CI->use_empty())
3961 return nullptr;
3962
3963 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
3964 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: 0));
3965 if (!Len)
3966 return nullptr;
3967
3968 // Known to have no uses (see above).
3969 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
3970 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
3971 return copyFlags(
3972 Old: *CI,
3973 New: emitFWrite(Ptr: CI->getArgOperand(i: 0),
3974 Size: ConstantInt::get(Ty: SizeTTy, V: Len - 1),
3975 File: CI->getArgOperand(i: 1), B, DL, TLI));
3976}
3977
3978Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
3979 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3980 if (!CI->use_empty())
3981 return nullptr;
3982
3983 // Check for a constant string.
3984 // puts("") -> putchar('\n')
3985 StringRef Str;
3986 if (getConstantStringInfo(V: CI->getArgOperand(i: 0), Str) && Str.empty()) {
3987 // putchar takes an argument of the same type as puts returns, i.e.,
3988 // int, which need not be 32 bits wide.
3989 Type *IntTy = CI->getType();
3990 return copyFlags(Old: *CI, New: emitPutChar(Char: ConstantInt::get(Ty: IntTy, V: '\n'), B, TLI));
3991 }
3992
3993 return nullptr;
3994}
3995
3996Value *LibCallSimplifier::optimizeExit(CallInst *CI) {
3997
3998 // Mark 'exit' as cold if its not exit(0) (success).
3999 const APInt *C;
4000 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
4001 match(V: CI->getArgOperand(i: 0), P: m_APInt(Res&: C)) && !C->isZero()) {
4002 CI->addFnAttr(Kind: Attribute::Cold);
4003 }
4004 return nullptr;
4005}
4006
4007Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
4008 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
4009 return copyFlags(Old: *CI, New: B.CreateMemMove(Dst: CI->getArgOperand(i: 1), DstAlign: Align(1),
4010 Src: CI->getArgOperand(i: 0), SrcAlign: Align(1),
4011 Size: CI->getArgOperand(i: 2)));
4012}
4013
4014bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
4015 SmallString<20> FloatFuncName = FuncName;
4016 FloatFuncName += 'f';
4017 return isLibFuncEmittable(M, TLI, Name: FloatFuncName);
4018}
4019
4020Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
4021 IRBuilderBase &Builder) {
4022 Module *M = CI->getModule();
4023 Function *Callee = CI->getCalledFunction();
4024 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4025
4026 // Check for string/memory library functions.
4027 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4028 // Make sure we never change the calling convention.
4029 assert(
4030 (ignoreCallingConv(Func) ||
4031 TargetLibraryInfoImpl::isCallingConvCCompatible(CI)) &&
4032 "Optimizing string/memory libcall would change the calling convention");
4033 switch (Func) {
4034 case LibFunc_strcat:
4035 return optimizeStrCat(CI, B&: Builder);
4036 case LibFunc_strncat:
4037 return optimizeStrNCat(CI, B&: Builder);
4038 case LibFunc_strchr:
4039 return optimizeStrChr(CI, B&: Builder);
4040 case LibFunc_strrchr:
4041 return optimizeStrRChr(CI, B&: Builder);
4042 case LibFunc_strcmp:
4043 return optimizeStrCmp(CI, B&: Builder);
4044 case LibFunc_strncmp:
4045 return optimizeStrNCmp(CI, B&: Builder);
4046 case LibFunc_strcpy:
4047 return optimizeStrCpy(CI, B&: Builder);
4048 case LibFunc_stpcpy:
4049 return optimizeStpCpy(CI, B&: Builder);
4050 case LibFunc_strlcpy:
4051 return optimizeStrLCpy(CI, B&: Builder);
4052 case LibFunc_stpncpy:
4053 return optimizeStringNCpy(CI, /*RetEnd=*/true, B&: Builder);
4054 case LibFunc_strncpy:
4055 return optimizeStringNCpy(CI, /*RetEnd=*/false, B&: Builder);
4056 case LibFunc_strlen:
4057 return optimizeStrLen(CI, B&: Builder);
4058 case LibFunc_strnlen:
4059 return optimizeStrNLen(CI, B&: Builder);
4060 case LibFunc_strpbrk:
4061 return optimizeStrPBrk(CI, B&: Builder);
4062 case LibFunc_strndup:
4063 return optimizeStrNDup(CI, B&: Builder);
4064 case LibFunc_strtol:
4065 case LibFunc_strtod:
4066 case LibFunc_strtof:
4067 case LibFunc_strtoul:
4068 case LibFunc_strtoll:
4069 case LibFunc_strtold:
4070 case LibFunc_strtoull:
4071 return optimizeStrTo(CI, B&: Builder);
4072 case LibFunc_strspn:
4073 return optimizeStrSpn(CI, B&: Builder);
4074 case LibFunc_strcspn:
4075 return optimizeStrCSpn(CI, B&: Builder);
4076 case LibFunc_strstr:
4077 return optimizeStrStr(CI, B&: Builder);
4078 case LibFunc_memchr:
4079 return optimizeMemChr(CI, B&: Builder);
4080 case LibFunc_memrchr:
4081 return optimizeMemRChr(CI, B&: Builder);
4082 case LibFunc_bcmp:
4083 return optimizeBCmp(CI, B&: Builder);
4084 case LibFunc_memcmp:
4085 return optimizeMemCmp(CI, B&: Builder);
4086 case LibFunc_memcpy:
4087 return optimizeMemCpy(CI, B&: Builder);
4088 case LibFunc_memccpy:
4089 return optimizeMemCCpy(CI, B&: Builder);
4090 case LibFunc_mempcpy:
4091 return optimizeMemPCpy(CI, B&: Builder);
4092 case LibFunc_memmove:
4093 return optimizeMemMove(CI, B&: Builder);
4094 case LibFunc_memset:
4095 return optimizeMemSet(CI, B&: Builder);
4096 case LibFunc_realloc:
4097 return optimizeRealloc(CI, B&: Builder);
4098 case LibFunc_wcslen:
4099 return optimizeWcslen(CI, B&: Builder);
4100 case LibFunc_bcopy:
4101 return optimizeBCopy(CI, B&: Builder);
4102 case LibFunc_Znwm:
4103 case LibFunc_ZnwmRKSt9nothrow_t:
4104 case LibFunc_ZnwmSt11align_val_t:
4105 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
4106 case LibFunc_Znam:
4107 case LibFunc_ZnamRKSt9nothrow_t:
4108 case LibFunc_ZnamSt11align_val_t:
4109 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
4110 case LibFunc_Znwm12__hot_cold_t:
4111 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
4112 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
4113 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4114 case LibFunc_Znam12__hot_cold_t:
4115 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
4116 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
4117 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4118 case LibFunc_size_returning_new:
4119 case LibFunc_size_returning_new_hot_cold:
4120 case LibFunc_size_returning_new_aligned:
4121 case LibFunc_size_returning_new_aligned_hot_cold:
4122 return optimizeNew(CI, B&: Builder, Func);
4123 default:
4124 break;
4125 }
4126 }
4127 return nullptr;
4128}
4129
4130/// Constant folding nan/nanf/nanl.
4131static Value *optimizeNaN(CallInst *CI) {
4132 StringRef CharSeq;
4133 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: CharSeq))
4134 return nullptr;
4135
4136 APInt Fill;
4137 // Treat empty strings as if they were zero.
4138 if (CharSeq.empty())
4139 Fill = APInt(32, 0);
4140 else if (CharSeq.getAsInteger(Radix: 0, Result&: Fill))
4141 return nullptr;
4142
4143 return ConstantFP::getQNaN(Ty: CI->getType(), /*Negative=*/false, Payload: &Fill);
4144}
4145
4146Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
4147 LibFunc Func,
4148 IRBuilderBase &Builder) {
4149 const Module *M = CI->getModule();
4150
4151 // Don't optimize calls that require strict floating point semantics.
4152 if (CI->isStrictFP())
4153 return nullptr;
4154
4155 if (Value *V = optimizeSymmetric(CI, Func, B&: Builder))
4156 return V;
4157
4158 switch (Func) {
4159 case LibFunc_sinpif:
4160 case LibFunc_sinpi:
4161 return optimizeSinCosPi(CI, /*IsSin*/true, B&: Builder);
4162 case LibFunc_cospif:
4163 case LibFunc_cospi:
4164 return optimizeSinCosPi(CI, /*IsSin*/false, B&: Builder);
4165 case LibFunc_sinf:
4166 case LibFunc_sinl:
4167 if (CI->doesNotAccessMemory())
4168 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::sin);
4169 return nullptr;
4170 case LibFunc_cosf:
4171 case LibFunc_cosl:
4172 if (CI->doesNotAccessMemory())
4173 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::cos);
4174 return nullptr;
4175 case LibFunc_powf:
4176 case LibFunc_pow:
4177 case LibFunc_powl:
4178 return optimizePow(Pow: CI, B&: Builder);
4179 case LibFunc_exp2l:
4180 case LibFunc_exp2:
4181 case LibFunc_exp2f:
4182 return optimizeExp2(CI, B&: Builder);
4183 case LibFunc_scalbn:
4184 case LibFunc_scalbnf:
4185 case LibFunc_scalbnl:
4186 // LLVM floating-point types have radix 2, so scalbn is equivalent to
4187 // ldexp. Do not replace a libcall that may set errno.
4188 if (CI->doesNotAccessMemory()) {
4189 Value *NewCall =
4190 Builder.CreateLdexp(Src: CI->getArgOperand(i: 0), Exp: CI->getArgOperand(i: 1), FMFSource: CI);
4191 NewCall->takeName(V: CI);
4192 return copyFlags(Old: *CI, New: NewCall);
4193 }
4194 return nullptr;
4195 case LibFunc_fabsf:
4196 case LibFunc_fabs:
4197 case LibFunc_fabsl:
4198 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::fabs);
4199 case LibFunc_sqrtf:
4200 case LibFunc_sqrt:
4201 case LibFunc_sqrtl:
4202 return optimizeSqrt(CI, B&: Builder);
4203 case LibFunc_fmod:
4204 case LibFunc_fmodf:
4205 case LibFunc_fmodl:
4206 return optimizeFMod(CI, B&: Builder);
4207 case LibFunc_logf:
4208 case LibFunc_log:
4209 case LibFunc_logl:
4210 case LibFunc_log10f:
4211 case LibFunc_log10:
4212 case LibFunc_log10l:
4213 case LibFunc_log1pf:
4214 case LibFunc_log1p:
4215 case LibFunc_log1pl:
4216 case LibFunc_log2f:
4217 case LibFunc_log2:
4218 case LibFunc_log2l:
4219 case LibFunc_logbf:
4220 case LibFunc_logb:
4221 case LibFunc_logbl:
4222 return optimizeLog(Log: CI, B&: Builder);
4223 case LibFunc_tan:
4224 case LibFunc_tanf:
4225 case LibFunc_tanl:
4226 case LibFunc_sinh:
4227 case LibFunc_sinhf:
4228 case LibFunc_sinhl:
4229 case LibFunc_asinh:
4230 case LibFunc_asinhf:
4231 case LibFunc_asinhl:
4232 case LibFunc_cosh:
4233 case LibFunc_coshf:
4234 case LibFunc_coshl:
4235 case LibFunc_atanh:
4236 case LibFunc_atanhf:
4237 case LibFunc_atanhl:
4238 return optimizeTrigInversionPairs(CI, B&: Builder);
4239 case LibFunc_ceil:
4240 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::ceil);
4241 case LibFunc_floor:
4242 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::floor);
4243 case LibFunc_round:
4244 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::round);
4245 case LibFunc_roundeven:
4246 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::roundeven);
4247 case LibFunc_nearbyint:
4248 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::nearbyint);
4249 case LibFunc_rint:
4250 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::rint);
4251 case LibFunc_trunc:
4252 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::trunc);
4253 case LibFunc_sin:
4254 case LibFunc_cos:
4255 if (UnsafeFPShrink &&
4256 hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4257 if (Value *V = optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true))
4258 return V;
4259 if (CI->doesNotAccessMemory())
4260 return replaceUnaryCall(
4261 CI, B&: Builder, IID: Func == LibFunc_sin ? Intrinsic::sin : Intrinsic::cos);
4262 return nullptr;
4263 case LibFunc_acos:
4264 case LibFunc_acosh:
4265 case LibFunc_asin:
4266 case LibFunc_atan:
4267 case LibFunc_cbrt:
4268 case LibFunc_exp:
4269 case LibFunc_exp10:
4270 case LibFunc_expm1:
4271 case LibFunc_tanh:
4272 if (UnsafeFPShrink && hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4273 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true);
4274 return nullptr;
4275 case LibFunc_copysign:
4276 if (hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4277 return optimizeBinaryDoubleFP(CI, B&: Builder, TLI);
4278 return nullptr;
4279 case LibFunc_fdim:
4280 case LibFunc_fdimf:
4281 case LibFunc_fdiml:
4282 return optimizeFdim(CI, B&: Builder);
4283 case LibFunc_fminf:
4284 case LibFunc_fmin:
4285 case LibFunc_fminl:
4286 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::minnum);
4287 case LibFunc_fmaxf:
4288 case LibFunc_fmax:
4289 case LibFunc_fmaxl:
4290 return optimizeFMinFMax(CI, B&: Builder, IID: Intrinsic::maxnum);
4291 case LibFunc_fminimum_numf:
4292 case LibFunc_fminimum_num:
4293 case LibFunc_fminimum_numl:
4294 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::minimumnum);
4295 case LibFunc_fmaximum_numf:
4296 case LibFunc_fmaximum_num:
4297 case LibFunc_fmaximum_numl:
4298 return replaceBinaryCall(CI, B&: Builder, IID: Intrinsic::maximumnum);
4299 case LibFunc_cabs:
4300 case LibFunc_cabsf:
4301 case LibFunc_cabsl:
4302 return optimizeCAbs(CI, B&: Builder);
4303 case LibFunc_remquo:
4304 case LibFunc_remquof:
4305 case LibFunc_remquol:
4306 return optimizeRemquo(CI, B&: Builder);
4307 case LibFunc_nan:
4308 case LibFunc_nanf:
4309 case LibFunc_nanl:
4310 return optimizeNaN(CI);
4311 default:
4312 return nullptr;
4313 }
4314}
4315
4316Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) {
4317 Module *M = CI->getModule();
4318 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
4319
4320 // TODO: Split out the code below that operates on FP calls so that
4321 // we can all non-FP calls with the StrictFP attribute to be
4322 // optimized.
4323 if (CI->isNoBuiltin()) {
4324 // Optionally update operator new calls.
4325 return maybeOptimizeNoBuiltinOperatorNew(CI, B&: Builder);
4326 }
4327
4328 Function *Callee = CI->getCalledFunction();
4329 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4330 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4331
4332 SmallVector<OperandBundleDef, 2> OpBundles;
4333 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4334
4335 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4336 Builder.setDefaultOperandBundles(OpBundles);
4337
4338 // Command-line parameter overrides instruction attribute.
4339 // This can't be moved to optimizeFloatingPointLibCall() because it may be
4340 // used by the intrinsic optimizations.
4341 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
4342 UnsafeFPShrink = EnableUnsafeFPShrink;
4343 else if (isa<FPMathOperator>(Val: CI) && CI->isFast())
4344 UnsafeFPShrink = true;
4345
4346 // First, check for intrinsics.
4347 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI)) {
4348 if (!IsCallingConvC)
4349 return nullptr;
4350 // The FP intrinsics have corresponding constrained versions so we don't
4351 // need to check for the StrictFP attribute here.
4352 switch (II->getIntrinsicID()) {
4353 case Intrinsic::pow:
4354 return optimizePow(Pow: CI, B&: Builder);
4355 case Intrinsic::exp2:
4356 return optimizeExp2(CI, B&: Builder);
4357 case Intrinsic::log:
4358 case Intrinsic::log2:
4359 case Intrinsic::log10:
4360 return optimizeLog(Log: CI, B&: Builder);
4361 case Intrinsic::sqrt:
4362 return optimizeSqrt(CI, B&: Builder);
4363 case Intrinsic::memset:
4364 return optimizeMemSet(CI, B&: Builder);
4365 case Intrinsic::memcpy:
4366 return optimizeMemCpy(CI, B&: Builder);
4367 case Intrinsic::memmove:
4368 return optimizeMemMove(CI, B&: Builder);
4369 case Intrinsic::sin:
4370 case Intrinsic::cos:
4371 if (UnsafeFPShrink)
4372 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, /*isPrecise=*/true);
4373 return nullptr;
4374 case Intrinsic::sincos:
4375 if (UnsafeFPShrink)
4376 return optimizeSinCosDoubleFP(CI, B&: Builder);
4377 return nullptr;
4378 default:
4379 return nullptr;
4380 }
4381 }
4382
4383 // Also try to simplify calls to fortified library functions.
4384 if (Value *SimplifiedFortifiedCI =
4385 FortifiedSimplifier.optimizeCall(CI, B&: Builder))
4386 return SimplifiedFortifiedCI;
4387
4388 // Then check for known library functions.
4389 if (isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4390 // We never change the calling convention.
4391 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4392 return nullptr;
4393 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
4394 return V;
4395 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
4396 return V;
4397 switch (Func) {
4398 case LibFunc_ffs:
4399 case LibFunc_ffsl:
4400 case LibFunc_ffsll:
4401 return optimizeFFS(CI, B&: Builder);
4402 case LibFunc_fls:
4403 case LibFunc_flsl:
4404 case LibFunc_flsll:
4405 return optimizeFls(CI, B&: Builder);
4406 case LibFunc_abs:
4407 case LibFunc_labs:
4408 case LibFunc_llabs:
4409 return optimizeAbs(CI, B&: Builder);
4410 case LibFunc_isdigit:
4411 return optimizeIsDigit(CI, B&: Builder);
4412 case LibFunc_isascii:
4413 return optimizeIsAscii(CI, B&: Builder);
4414 case LibFunc_toascii:
4415 return optimizeToAscii(CI, B&: Builder);
4416 case LibFunc_atoi:
4417 case LibFunc_atol:
4418 case LibFunc_atoll:
4419 return optimizeAtoi(CI, B&: Builder);
4420 case LibFunc_strtol:
4421 case LibFunc_strtoll:
4422 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/true);
4423 case LibFunc_strtoul:
4424 case LibFunc_strtoull:
4425 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/false);
4426 case LibFunc_printf:
4427 return optimizePrintF(CI, B&: Builder);
4428 case LibFunc_sprintf:
4429 return optimizeSPrintF(CI, B&: Builder);
4430 case LibFunc_snprintf:
4431 return optimizeSnPrintF(CI, B&: Builder);
4432 case LibFunc_fprintf:
4433 return optimizeFPrintF(CI, B&: Builder);
4434 case LibFunc_fwrite:
4435 return optimizeFWrite(CI, B&: Builder);
4436 case LibFunc_fputs:
4437 return optimizeFPuts(CI, B&: Builder);
4438 case LibFunc_puts:
4439 return optimizePuts(CI, B&: Builder);
4440 case LibFunc_perror:
4441 return optimizeErrorReporting(CI, B&: Builder);
4442 case LibFunc_vfprintf:
4443 case LibFunc_fiprintf:
4444 return optimizeErrorReporting(CI, B&: Builder, StreamArg: 0);
4445 case LibFunc_exit:
4446 case LibFunc_Exit:
4447 return optimizeExit(CI);
4448 default:
4449 return nullptr;
4450 }
4451 }
4452 return nullptr;
4453}
4454
4455LibCallSimplifier::LibCallSimplifier(
4456 const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT,
4457 DomConditionCache *DC, AssumptionCache *AC, OptimizationRemarkEmitter &ORE,
4458 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
4459 function_ref<void(Instruction *, Value *)> Replacer,
4460 function_ref<void(Instruction *)> Eraser)
4461 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), DT(DT), DC(DC), AC(AC),
4462 ORE(ORE), BFI(BFI), PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
4463
4464void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
4465 // Indirect through the replacer used in this instance.
4466 Replacer(I, With);
4467}
4468
4469void LibCallSimplifier::eraseFromParent(Instruction *I) {
4470 Eraser(I);
4471}
4472
4473// TODO:
4474// Additional cases that we need to add to this file:
4475//
4476// cbrt:
4477// * cbrt(expN(X)) -> expN(x/3)
4478// * cbrt(sqrt(x)) -> pow(x,1/6)
4479// * cbrt(cbrt(x)) -> pow(x,1/9)
4480//
4481// exp, expf, expl:
4482// * exp(log(x)) -> x
4483//
4484// log, logf, logl:
4485// * log(exp(x)) -> x
4486// * log(exp(y)) -> y*log(e)
4487// * log(exp10(y)) -> y*log(10)
4488// * log(sqrt(x)) -> 0.5*log(x)
4489//
4490// pow, powf, powl:
4491// * pow(sqrt(x),y) -> pow(x,y*0.5)
4492// * pow(pow(x,y),z)-> pow(x,y*z)
4493//
4494// signbit:
4495// * signbit(cnst) -> cnst'
4496// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
4497//
4498// sqrt, sqrtf, sqrtl:
4499// * sqrt(expN(x)) -> expN(x*0.5)
4500// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
4501// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
4502//
4503
4504//===----------------------------------------------------------------------===//
4505// Fortified Library Call Optimizations
4506//===----------------------------------------------------------------------===//
4507
4508bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
4509 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
4510 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
4511 // If this function takes a flag argument, the implementation may use it to
4512 // perform extra checks. Don't fold into the non-checking variant.
4513 if (FlagOp) {
4514 ConstantInt *Flag = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *FlagOp));
4515 if (!Flag || !Flag->isZero())
4516 return false;
4517 }
4518
4519 if (SizeOp && CI->getArgOperand(i: ObjSizeOp) == CI->getArgOperand(i: *SizeOp))
4520 return true;
4521
4522 if (ConstantInt *ObjSizeCI =
4523 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: ObjSizeOp))) {
4524 if (ObjSizeCI->isMinusOne())
4525 return true;
4526 // If the object size wasn't -1 (unknown), bail out if we were asked to.
4527 if (OnlyLowerUnknownSize)
4528 return false;
4529 if (StrOp) {
4530 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: *StrOp));
4531 // If the length is 0 we don't know how long it is and so we can't
4532 // remove the check.
4533 if (Len)
4534 annotateDereferenceableBytes(CI, ArgNos: *StrOp, DereferenceableBytes: Len);
4535 else
4536 return false;
4537 return ObjSizeCI->getZExtValue() >= Len;
4538 }
4539
4540 if (SizeOp) {
4541 if (ConstantInt *SizeCI =
4542 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *SizeOp)))
4543 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4544 }
4545 }
4546 return false;
4547}
4548
4549Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4550 IRBuilderBase &B) {
4551 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4552 CallInst *NewCI =
4553 B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4554 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4555 mergeAttributesAndFlags(NewCI, Old: *CI);
4556 return CI->getArgOperand(i: 0);
4557 }
4558 return nullptr;
4559}
4560
4561Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4562 IRBuilderBase &B) {
4563 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4564 CallInst *NewCI =
4565 B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4566 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4567 mergeAttributesAndFlags(NewCI, Old: *CI);
4568 return CI->getArgOperand(i: 0);
4569 }
4570 return nullptr;
4571}
4572
4573Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4574 IRBuilderBase &B) {
4575 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4576 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
4577 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val,
4578 Size: CI->getArgOperand(i: 2), Align: Align(1));
4579 mergeAttributesAndFlags(NewCI, Old: *CI);
4580 return CI->getArgOperand(i: 0);
4581 }
4582 return nullptr;
4583}
4584
4585Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4586 IRBuilderBase &B) {
4587 const DataLayout &DL = CI->getDataLayout();
4588 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2))
4589 if (Value *Call = emitMemPCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4590 Len: CI->getArgOperand(i: 2), B, DL, TLI)) {
4591 return mergeAttributesAndFlags(NewCI: cast<CallInst>(Val: Call), Old: *CI);
4592 }
4593 return nullptr;
4594}
4595
4596Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4597 IRBuilderBase &B,
4598 LibFunc Func) {
4599 const DataLayout &DL = CI->getDataLayout();
4600 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1),
4601 *ObjSize = CI->getArgOperand(i: 2);
4602
4603 // __stpcpy_chk(x,x,...) -> x+strlen(x)
4604 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4605 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
4606 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
4607 }
4608
4609 // If a) we don't have any length information, or b) we know this will
4610 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4611 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
4612 // TODO: It might be nice to get a maximum length out of the possible
4613 // string lengths for varying.
4614 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: 1)) {
4615 if (Func == LibFunc_strcpy_chk)
4616 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
4617 else
4618 return copyFlags(Old: *CI, New: emitStpCpy(Dst, Src, B, TLI));
4619 }
4620
4621 if (OnlyLowerUnknownSize)
4622 return nullptr;
4623
4624 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4625 uint64_t Len = GetStringLength(V: Src);
4626 if (Len)
4627 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
4628 else
4629 return nullptr;
4630
4631 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4632 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4633 Value *LenV = ConstantInt::get(Ty: SizeTTy, V: Len);
4634 Value *Ret = emitMemCpyChk(Dst, Src, Len: LenV, ObjSize, B, DL, TLI);
4635 // If the function was an __stpcpy_chk, and we were able to fold it into
4636 // a __memcpy_chk, we still need to return the correct end pointer.
4637 if (Ret && Func == LibFunc_stpcpy_chk)
4638 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst,
4639 IdxList: ConstantInt::get(Ty: SizeTTy, V: Len - 1));
4640 return copyFlags(Old: *CI, New: cast<CallInst>(Val: Ret));
4641}
4642
4643Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4644 IRBuilderBase &B) {
4645 if (isFortifiedCallFoldable(CI, ObjSizeOp: 1, SizeOp: std::nullopt, StrOp: 0))
4646 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B,
4647 DL: CI->getDataLayout(), TLI));
4648 return nullptr;
4649}
4650
4651Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4652 IRBuilderBase &B,
4653 LibFunc Func) {
4654 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4655 if (Func == LibFunc_strncpy_chk)
4656 return copyFlags(Old: *CI,
4657 New: emitStrNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4658 Len: CI->getArgOperand(i: 2), B, TLI));
4659 else
4660 return copyFlags(Old: *CI,
4661 New: emitStpNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4662 Len: CI->getArgOperand(i: 2), B, TLI));
4663 }
4664
4665 return nullptr;
4666}
4667
4668Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4669 IRBuilderBase &B) {
4670 if (isFortifiedCallFoldable(CI, ObjSizeOp: 4, SizeOp: 3))
4671 return copyFlags(
4672 Old: *CI, New: emitMemCCpy(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
4673 Val: CI->getArgOperand(i: 2), Len: CI->getArgOperand(i: 3), B, TLI));
4674
4675 return nullptr;
4676}
4677
4678Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4679 IRBuilderBase &B) {
4680 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2)) {
4681 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 5));
4682 return copyFlags(Old: *CI,
4683 New: emitSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4684 Fmt: CI->getArgOperand(i: 4), Args: VariadicArgs, B, TLI));
4685 }
4686
4687 return nullptr;
4688}
4689
4690Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4691 IRBuilderBase &B) {
4692 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1)) {
4693 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 4));
4694 return copyFlags(Old: *CI,
4695 New: emitSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4696 VariadicArgs, B, TLI));
4697 }
4698
4699 return nullptr;
4700}
4701
4702Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4703 IRBuilderBase &B) {
4704 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2))
4705 return copyFlags(
4706 Old: *CI, New: emitStrCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1), B, TLI));
4707
4708 return nullptr;
4709}
4710
4711Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4712 IRBuilderBase &B) {
4713 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4714 return copyFlags(Old: *CI,
4715 New: emitStrLCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4716 Size: CI->getArgOperand(i: 2), B, TLI));
4717
4718 return nullptr;
4719}
4720
4721Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4722 IRBuilderBase &B) {
4723 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4724 return copyFlags(Old: *CI,
4725 New: emitStrNCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4726 Size: CI->getArgOperand(i: 2), B, TLI));
4727
4728 return nullptr;
4729}
4730
4731Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4732 IRBuilderBase &B) {
4733 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4734 return copyFlags(Old: *CI,
4735 New: emitStrLCpy(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4736 Size: CI->getArgOperand(i: 2), B, TLI));
4737
4738 return nullptr;
4739}
4740
4741Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4742 IRBuilderBase &B) {
4743 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2))
4744 return copyFlags(
4745 Old: *CI, New: emitVSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4746 Fmt: CI->getArgOperand(i: 4), VAList: CI->getArgOperand(i: 5), B, TLI));
4747
4748 return nullptr;
4749}
4750
4751Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4752 IRBuilderBase &B) {
4753 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1))
4754 return copyFlags(Old: *CI,
4755 New: emitVSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4756 VAList: CI->getArgOperand(i: 4), B, TLI));
4757
4758 return nullptr;
4759}
4760
4761Value *FortifiedLibCallSimplifier::optimizeCall(CallInst *CI,
4762 IRBuilderBase &Builder) {
4763 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4764 // Some clang users checked for _chk libcall availability using:
4765 // __has_builtin(__builtin___memcpy_chk)
4766 // When compiling with -fno-builtin, this is always true.
4767 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4768 // end up with fortified libcalls, which isn't acceptable in a freestanding
4769 // environment which only provides their non-fortified counterparts.
4770 //
4771 // Until we change clang and/or teach external users to check for availability
4772 // differently, disregard the "nobuiltin" attribute and TLI::has.
4773 //
4774 // PR23093.
4775
4776 Function *Callee = CI->getCalledFunction();
4777 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4778
4779 SmallVector<OperandBundleDef, 2> OpBundles;
4780 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4781
4782 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4783 Builder.setDefaultOperandBundles(OpBundles);
4784
4785 // First, check that this is a known library functions and that the prototype
4786 // is correct.
4787 LibFunc Func = TLI->getLibFunc(FDecl: *Callee);
4788 if (Func == NotLibFunc)
4789 return nullptr;
4790
4791 // We never change the calling convention.
4792 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4793 return nullptr;
4794
4795 switch (Func) {
4796 case LibFunc_memcpy_chk:
4797 return optimizeMemCpyChk(CI, B&: Builder);
4798 case LibFunc_mempcpy_chk:
4799 return optimizeMemPCpyChk(CI, B&: Builder);
4800 case LibFunc_memmove_chk:
4801 return optimizeMemMoveChk(CI, B&: Builder);
4802 case LibFunc_memset_chk:
4803 return optimizeMemSetChk(CI, B&: Builder);
4804 case LibFunc_stpcpy_chk:
4805 case LibFunc_strcpy_chk:
4806 return optimizeStrpCpyChk(CI, B&: Builder, Func);
4807 case LibFunc_strlen_chk:
4808 return optimizeStrLenChk(CI, B&: Builder);
4809 case LibFunc_stpncpy_chk:
4810 case LibFunc_strncpy_chk:
4811 return optimizeStrpNCpyChk(CI, B&: Builder, Func);
4812 case LibFunc_memccpy_chk:
4813 return optimizeMemCCpyChk(CI, B&: Builder);
4814 case LibFunc_snprintf_chk:
4815 return optimizeSNPrintfChk(CI, B&: Builder);
4816 case LibFunc_sprintf_chk:
4817 return optimizeSPrintfChk(CI, B&: Builder);
4818 case LibFunc_strcat_chk:
4819 return optimizeStrCatChk(CI, B&: Builder);
4820 case LibFunc_strlcat_chk:
4821 return optimizeStrLCat(CI, B&: Builder);
4822 case LibFunc_strncat_chk:
4823 return optimizeStrNCatChk(CI, B&: Builder);
4824 case LibFunc_strlcpy_chk:
4825 return optimizeStrLCpyChk(CI, B&: Builder);
4826 case LibFunc_vsnprintf_chk:
4827 return optimizeVSNPrintfChk(CI, B&: Builder);
4828 case LibFunc_vsprintf_chk:
4829 return optimizeVSPrintfChk(CI, B&: Builder);
4830 default:
4831 break;
4832 }
4833 return nullptr;
4834}
4835
4836FortifiedLibCallSimplifier::FortifiedLibCallSimplifier(
4837 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4838 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
4839