1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/SmallString.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/Analysis/ConstantFolding.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/AttributeMask.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/IntrinsicInst.h"
29#include "llvm/IR/Intrinsics.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/PatternMatch.h"
32#include "llvm/Support/Casting.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/KnownBits.h"
35#include "llvm/Support/KnownFPClass.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/TargetParser/Triple.h"
38#include "llvm/Transforms/Utils/BuildLibCalls.h"
39#include "llvm/Transforms/Utils/Local.h"
40#include "llvm/Transforms/Utils/SizeOpts.h"
41
42#include <cmath>
43
44using namespace llvm;
45using namespace PatternMatch;
46
47static cl::opt<bool>
48 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
49 cl::init(Val: false),
50 cl::desc("Enable unsafe double to float "
51 "shrinking for math lib calls"));
52
53// Enable conversion of operator new calls with a MemProf hot or cold hint
54// to an operator new call that takes a hot/cold hint. Off by default since
55// not all allocators currently support this extension.
56static cl::opt<bool>
57 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(Val: false),
58 cl::desc("Enable hot/cold operator new library calls"));
59static cl::opt<bool> OptimizeExistingHotColdNew(
60 "optimize-existing-hot-cold-new", cl::Hidden, cl::init(Val: false),
61 cl::desc(
62 "Enable optimization of existing hot/cold operator new library calls"));
63
64namespace {
65
66// Specialized parser to ensure the hint is an 8 bit value (we can't specify
67// uint8_t to opt<> as that is interpreted to mean that we are passing a char
68// option with a specific set of values.
69struct HotColdHintParser : public cl::parser<unsigned> {
70 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
71
72 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
73 if (Arg.getAsInteger(Radix: 0, Result&: Value))
74 return O.error(Message: "'" + Arg + "' value invalid for uint argument!");
75
76 if (Value > 255)
77 return O.error(Message: "'" + Arg + "' value must be in the range [0, 255]!");
78
79 return false;
80 }
81};
82
83} // end anonymous namespace
84
85// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
86// and 255 is the hottest. Default to 1 value away from the coldest and hottest
87// hints, so that the compiler hinted allocations are slightly less strong than
88// manually inserted hints at the two extremes.
89static cl::opt<unsigned, false, HotColdHintParser> ColdNewHintValue(
90 "cold-new-hint-value", cl::Hidden, cl::init(Val: 1),
91 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
92static cl::opt<unsigned, false, HotColdHintParser>
93 NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(Val: 128),
94 cl::desc("Value to pass to hot/cold operator new for "
95 "notcold (warm) allocation"));
96static cl::opt<unsigned, false, HotColdHintParser> HotNewHintValue(
97 "hot-new-hint-value", cl::Hidden, cl::init(Val: 254),
98 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
99
100//===----------------------------------------------------------------------===//
101// Helper Functions
102//===----------------------------------------------------------------------===//
103
104static bool ignoreCallingConv(LibFunc Func) {
105 return Func == LibFunc_abs || Func == LibFunc_labs ||
106 Func == LibFunc_llabs || Func == LibFunc_strlen;
107}
108
109/// Return true if it is only used in equality comparisons with With.
110static bool isOnlyUsedInEqualityComparison(Value *V, Value *With) {
111 for (User *U : V->users()) {
112 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
113 if (IC->isEquality() && IC->getOperand(i_nocapture: 1) == With)
114 continue;
115 // Unknown instruction.
116 return false;
117 }
118 return true;
119}
120
121static bool callHasFloatingPointArgument(const CallInst *CI) {
122 return any_of(Range: CI->operands(), P: [](const Use &OI) {
123 return OI->getType()->isFloatingPointTy();
124 });
125}
126
127static bool callHasFP128Argument(const CallInst *CI) {
128 return any_of(Range: CI->operands(), P: [](const Use &OI) {
129 return OI->getType()->isFP128Ty();
130 });
131}
132
133// Convert the entire string Str representing an integer in Base, up to
134// the terminating nul if present, to a constant according to the rules
135// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
136// return the result, otherwise null.
137// The function assumes the string is encoded in ASCII and carefully
138// avoids converting sequences (including "") that the corresponding
139// library call might fail and set errno for.
140static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
141 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
142 if (Base < 2 || Base > 36)
143 if (Base != 0)
144 // Fail for an invalid base (required by POSIX).
145 return nullptr;
146
147 // Current offset into the original string to reflect in EndPtr.
148 size_t Offset = 0;
149 // Strip leading whitespace.
150 for ( ; Offset != Str.size(); ++Offset)
151 if (!isSpace(C: (unsigned char)Str[Offset])) {
152 Str = Str.substr(Start: Offset);
153 break;
154 }
155
156 if (Str.empty())
157 // Fail for empty subject sequences (POSIX allows but doesn't require
158 // strtol[l]/strtoul[l] to fail with EINVAL).
159 return nullptr;
160
161 // Strip but remember the sign.
162 bool Negate = Str[0] == '-';
163 if (Str[0] == '-' || Str[0] == '+') {
164 Str = Str.drop_front();
165 if (Str.empty())
166 // Fail for a sign with nothing after it.
167 return nullptr;
168 ++Offset;
169 }
170
171 // Set Max to the absolute value of the minimum (for signed), or
172 // to the maximum (for unsigned) value representable in the type.
173 Type *RetTy = CI->getType();
174 unsigned NBits = RetTy->getPrimitiveSizeInBits();
175 uint64_t Max = AsSigned && Negate ? 1 : 0;
176 Max += AsSigned ? maxIntN(N: NBits) : maxUIntN(N: NBits);
177
178 // Autodetect Base if it's zero and consume the "0x" prefix.
179 if (Str.size() > 1) {
180 if (Str[0] == '0') {
181 if (toUpper(x: (unsigned char)Str[1]) == 'X') {
182 if (Str.size() == 2 || (Base && Base != 16))
183 // Fail if Base doesn't allow the "0x" prefix or for the prefix
184 // alone that implementations like BSD set errno to EINVAL for.
185 return nullptr;
186
187 Str = Str.drop_front(N: 2);
188 Offset += 2;
189 Base = 16;
190 }
191 else if (Base == 0)
192 Base = 8;
193 } else if (Base == 0)
194 Base = 10;
195 }
196 else if (Base == 0)
197 Base = 10;
198
199 // Convert the rest of the subject sequence, not including the sign,
200 // to its uint64_t representation (this assumes the source character
201 // set is ASCII).
202 uint64_t Result = 0;
203 for (unsigned i = 0; i != Str.size(); ++i) {
204 unsigned char DigVal = Str[i];
205 if (isDigit(C: DigVal))
206 DigVal = DigVal - '0';
207 else {
208 DigVal = toUpper(x: DigVal);
209 if (isAlpha(C: DigVal))
210 DigVal = DigVal - 'A' + 10;
211 else
212 return nullptr;
213 }
214
215 if (DigVal >= Base)
216 // Fail if the digit is not valid in the Base.
217 return nullptr;
218
219 // Add the digit and fail if the result is not representable in
220 // the (unsigned form of the) destination type.
221 bool VFlow;
222 Result = SaturatingMultiplyAdd(X: Result, Y: Base, A: (uint64_t)DigVal, ResultOverflowed: &VFlow);
223 if (VFlow || Result > Max)
224 return nullptr;
225 }
226
227 if (EndPtr) {
228 // Store the pointer to the end.
229 Value *Off = B.getInt64(C: Offset + Str.size());
230 Value *StrBeg = CI->getArgOperand(i: 0);
231 Value *StrEnd = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: StrBeg, IdxList: Off, Name: "endptr");
232 B.CreateStore(Val: StrEnd, Ptr: EndPtr);
233 }
234
235 if (Negate)
236 // Unsigned negation doesn't overflow.
237 Result = -Result;
238
239 return ConstantInt::get(Ty: RetTy, V: Result);
240}
241
242static bool isOnlyUsedInComparisonWithZero(Value *V) {
243 for (User *U : V->users()) {
244 if (ICmpInst *IC = dyn_cast<ICmpInst>(Val: U))
245 if (Constant *C = dyn_cast<Constant>(Val: IC->getOperand(i_nocapture: 1)))
246 if (C->isNullValue())
247 continue;
248 // Unknown instruction.
249 return false;
250 }
251 return true;
252}
253
254static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
255 const DataLayout &DL) {
256 if (!isOnlyUsedInComparisonWithZero(V: CI))
257 return false;
258
259 if (!isDereferenceableAndAlignedPointer(V: Str, Alignment: Align(1), Size: APInt(64, Len), DL))
260 return false;
261
262 if (CI->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemory))
263 return false;
264
265 return true;
266}
267
268static void annotateDereferenceableBytes(CallInst *CI,
269 ArrayRef<unsigned> ArgNos,
270 uint64_t DereferenceableBytes) {
271 const Function *F = CI->getCaller();
272 if (!F)
273 return;
274 for (unsigned ArgNo : ArgNos) {
275 uint64_t DerefBytes = DereferenceableBytes;
276 unsigned AS = CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
277 if (!llvm::NullPointerIsDefined(F, AS) ||
278 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
279 DerefBytes = std::max(a: CI->getParamDereferenceableOrNullBytes(i: ArgNo),
280 b: DereferenceableBytes);
281
282 if (CI->getParamDereferenceableBytes(i: ArgNo) < DerefBytes) {
283 CI->removeParamAttr(ArgNo, Kind: Attribute::Dereferenceable);
284 if (!llvm::NullPointerIsDefined(F, AS) ||
285 CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull))
286 CI->removeParamAttr(ArgNo, Kind: Attribute::DereferenceableOrNull);
287 CI->addParamAttr(ArgNo, Attr: Attribute::getWithDereferenceableBytes(
288 Context&: CI->getContext(), Bytes: DerefBytes));
289 }
290 }
291}
292
293static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI,
294 ArrayRef<unsigned> ArgNos) {
295 Function *F = CI->getCaller();
296 if (!F)
297 return;
298
299 for (unsigned ArgNo : ArgNos) {
300 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NoUndef))
301 CI->addParamAttr(ArgNo, Kind: Attribute::NoUndef);
302
303 if (!CI->paramHasAttr(ArgNo, Kind: Attribute::NonNull)) {
304 unsigned AS =
305 CI->getArgOperand(i: ArgNo)->getType()->getPointerAddressSpace();
306 if (llvm::NullPointerIsDefined(F, AS))
307 continue;
308 CI->addParamAttr(ArgNo, Kind: Attribute::NonNull);
309 }
310
311 annotateDereferenceableBytes(CI, ArgNos: ArgNo, DereferenceableBytes: 1);
312 }
313}
314
315static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> ArgNos,
316 Value *Size, const DataLayout &DL) {
317 if (ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size)) {
318 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
319 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: LenC->getZExtValue());
320 } else if (isKnownNonZero(V: Size, Q: DL)) {
321 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
322 const APInt *X, *Y;
323 uint64_t DerefMin = 1;
324 if (match(V: Size, P: m_Select(C: m_Value(), L: m_APInt(Res&: X), R: m_APInt(Res&: Y)))) {
325 DerefMin = std::min(a: X->getZExtValue(), b: Y->getZExtValue());
326 annotateDereferenceableBytes(CI, ArgNos, DereferenceableBytes: DerefMin);
327 }
328 }
329}
330
331// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
332// easier chaining. Calls to emit* and B.createCall should probably be wrapped
333// in this function when New is created to replace Old. Callers should take
334// care to check Old.isMustTailCall() if they aren't replacing Old directly
335// with New.
336static Value *copyFlags(const CallInst &Old, Value *New) {
337 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
338 assert(!Old.isNoTailCall() && "do not copy notail call flags");
339 if (auto *NewCI = dyn_cast_or_null<CallInst>(Val: New))
340 NewCI->setTailCallKind(Old.getTailCallKind());
341 return New;
342}
343
344static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
345 NewCI->setAttributes(AttributeList::get(
346 C&: NewCI->getContext(), Attrs: {NewCI->getAttributes(), Old.getAttributes()}));
347 NewCI->removeRetAttrs(AttrsToRemove: AttributeFuncs::typeIncompatible(
348 Ty: NewCI->getType(), AS: NewCI->getRetAttributes()));
349 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
350 NewCI->removeParamAttrs(
351 ArgNo: I, AttrsToRemove: AttributeFuncs::typeIncompatible(Ty: NewCI->getArgOperand(i: I)->getType(),
352 AS: NewCI->getParamAttributes(ArgNo: I)));
353
354 return copyFlags(Old, New: NewCI);
355}
356
357// Helper to avoid truncating the length if size_t is 32-bits.
358static StringRef substr(StringRef Str, uint64_t Len) {
359 return Len >= Str.size() ? Str : Str.substr(Start: 0, N: Len);
360}
361
362//===----------------------------------------------------------------------===//
363// String and Memory Library Call Optimizations
364//===----------------------------------------------------------------------===//
365
366Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
367 // Extract some information from the instruction
368 Value *Dst = CI->getArgOperand(i: 0);
369 Value *Src = CI->getArgOperand(i: 1);
370 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
371
372 // See if we can get the length of the input string.
373 uint64_t Len = GetStringLength(V: Src);
374 if (Len)
375 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
376 else
377 return nullptr;
378 --Len; // Unbias length.
379
380 // Handle the simple, do-nothing case: strcat(x, "") -> x
381 if (Len == 0)
382 return Dst;
383
384 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len, B));
385}
386
387Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
388 IRBuilderBase &B) {
389 // We need to find the end of the destination string. That's where the
390 // memory is to be moved to. We just generate a call to strlen.
391 Value *DstLen = emitStrLen(Ptr: Dst, B, DL, TLI);
392 if (!DstLen)
393 return nullptr;
394
395 // Now that we have the destination's length, we must index into the
396 // destination's pointer to get the actual memcpy destination (end of
397 // the string .. we're concatenating).
398 Value *CpyDst = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: DstLen, Name: "endptr");
399
400 // We have enough information to now generate the memcpy call to do the
401 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
402 B.CreateMemCpy(Dst: CpyDst, DstAlign: Align(1), Src, SrcAlign: Align(1),
403 Size: TLI->getAsSizeT(V: Len + 1, M: *B.GetInsertBlock()->getModule()));
404 return Dst;
405}
406
407Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
408 // Extract some information from the instruction.
409 Value *Dst = CI->getArgOperand(i: 0);
410 Value *Src = CI->getArgOperand(i: 1);
411 Value *Size = CI->getArgOperand(i: 2);
412 uint64_t Len;
413 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
414 if (isKnownNonZero(V: Size, Q: DL))
415 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
416
417 // We don't do anything if length is not constant.
418 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size);
419 if (LengthArg) {
420 Len = LengthArg->getZExtValue();
421 // strncat(x, c, 0) -> x
422 if (!Len)
423 return Dst;
424 } else {
425 return nullptr;
426 }
427
428 // See if we can get the length of the input string.
429 uint64_t SrcLen = GetStringLength(V: Src);
430 if (SrcLen) {
431 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
432 --SrcLen; // Unbias length.
433 } else {
434 return nullptr;
435 }
436
437 // strncat(x, "", c) -> x
438 if (SrcLen == 0)
439 return Dst;
440
441 // We don't optimize this case.
442 if (Len < SrcLen)
443 return nullptr;
444
445 // strncat(x, s, c) -> strcat(x, s)
446 // s is constant so the strcat can be optimized further.
447 return copyFlags(Old: *CI, New: emitStrLenMemCpy(Src, Dst, Len: SrcLen, B));
448}
449
450// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
451// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
452// is that either S is dereferenceable or the value of N is nonzero.
453static Value* memChrToCharCompare(CallInst *CI, Value *NBytes,
454 IRBuilderBase &B, const DataLayout &DL)
455{
456 Value *Src = CI->getArgOperand(i: 0);
457 Value *CharVal = CI->getArgOperand(i: 1);
458
459 // Fold memchr(A, C, N) == A to N && *A == C.
460 Type *CharTy = B.getInt8Ty();
461 Value *Char0 = B.CreateLoad(Ty: CharTy, Ptr: Src);
462 CharVal = B.CreateTrunc(V: CharVal, DestTy: CharTy);
463 Value *Cmp = B.CreateICmpEQ(LHS: Char0, RHS: CharVal, Name: "char0cmp");
464
465 if (NBytes) {
466 Value *Zero = ConstantInt::get(Ty: NBytes->getType(), V: 0);
467 Value *And = B.CreateICmpNE(LHS: NBytes, RHS: Zero);
468 Cmp = B.CreateLogicalAnd(Cond1: And, Cond2: Cmp);
469 }
470
471 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
472 return B.CreateSelect(C: Cmp, True: Src, False: NullPtr);
473}
474
475Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
476 Value *SrcStr = CI->getArgOperand(i: 0);
477 Value *CharVal = CI->getArgOperand(i: 1);
478 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
479
480 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
481 return memChrToCharCompare(CI, NBytes: nullptr, B, DL);
482
483 // If the second operand is non-constant, see if we can compute the length
484 // of the input string and turn this into memchr.
485 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
486 if (!CharC) {
487 uint64_t Len = GetStringLength(V: SrcStr);
488 if (Len)
489 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len);
490 else
491 return nullptr;
492
493 Function *Callee = CI->getCalledFunction();
494 FunctionType *FT = Callee->getFunctionType();
495 unsigned IntBits = TLI->getIntSize();
496 if (!FT->getParamType(i: 1)->isIntegerTy(Bitwidth: IntBits)) // memchr needs 'int'.
497 return nullptr;
498
499 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
500 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
501 return copyFlags(Old: *CI,
502 New: emitMemChr(Ptr: SrcStr, Val: CharVal, // include nul.
503 Len: ConstantInt::get(Ty: SizeTTy, V: Len), B,
504 DL, TLI));
505 }
506
507 if (CharC->isZero()) {
508 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
509 if (isOnlyUsedInEqualityComparison(V: CI, With: NullPtr))
510 // Pre-empt the transformation to strlen below and fold
511 // strchr(A, '\0') == null to false.
512 return B.CreateIntToPtr(V: B.getTrue(), DestTy: CI->getType());
513 }
514
515 // Otherwise, the character is a constant, see if the first argument is
516 // a string literal. If so, we can constant fold.
517 StringRef Str;
518 if (!getConstantStringInfo(V: SrcStr, Str)) {
519 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
520 if (Value *StrLen = emitStrLen(Ptr: SrcStr, B, DL, TLI))
521 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: StrLen, Name: "strchr");
522 return nullptr;
523 }
524
525 // Compute the offset, make sure to handle the case when we're searching for
526 // zero (a weird way to spell strlen).
527 size_t I = (0xFF & CharC->getSExtValue()) == 0
528 ? Str.size()
529 : Str.find(C: CharC->getSExtValue());
530 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
531 return Constant::getNullValue(Ty: CI->getType());
532
533 // strchr(s+n,c) -> gep(s+n+i,c)
534 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: I), Name: "strchr");
535}
536
537Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
538 Value *SrcStr = CI->getArgOperand(i: 0);
539 Value *CharVal = CI->getArgOperand(i: 1);
540 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
541 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
542
543 StringRef Str;
544 if (!getConstantStringInfo(V: SrcStr, Str)) {
545 // strrchr(s, 0) -> strchr(s, 0)
546 if (CharC && CharC->isZero())
547 return copyFlags(Old: *CI, New: emitStrChr(Ptr: SrcStr, C: '\0', B, TLI));
548 return nullptr;
549 }
550
551 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
552 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
553
554 // Try to expand strrchr to the memrchr nonstandard extension if it's
555 // available, or simply fail otherwise.
556 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
557 Value *Size = ConstantInt::get(Ty: SizeTTy, V: NBytes);
558 return copyFlags(Old: *CI, New: emitMemRChr(Ptr: SrcStr, Val: CharVal, Len: Size, B, DL, TLI));
559}
560
561Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
562 Value *Str1P = CI->getArgOperand(i: 0), *Str2P = CI->getArgOperand(i: 1);
563 if (Str1P == Str2P) // strcmp(x,x) -> 0
564 return ConstantInt::get(Ty: CI->getType(), V: 0);
565
566 StringRef Str1, Str2;
567 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
568 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
569
570 // strcmp(x, y) -> cnst (if both x and y are constant strings)
571 if (HasStr1 && HasStr2)
572 return ConstantInt::get(Ty: CI->getType(),
573 V: std::clamp(val: Str1.compare(RHS: Str2), lo: -1, hi: 1));
574
575 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
576 return B.CreateNeg(V: B.CreateZExt(
577 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
578
579 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
580 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
581 DestTy: CI->getType());
582
583 // strcmp(P, "x") -> memcmp(P, "x", 2)
584 uint64_t Len1 = GetStringLength(V: Str1P);
585 if (Len1)
586 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
587 uint64_t Len2 = GetStringLength(V: Str2P);
588 if (Len2)
589 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
590
591 if (Len1 && Len2) {
592 return copyFlags(
593 Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
594 Len: TLI->getAsSizeT(V: std::min(a: Len1, b: Len2), M: *CI->getModule()),
595 B, DL, TLI));
596 }
597
598 // strcmp to memcmp
599 if (!HasStr1 && HasStr2) {
600 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, DL))
601 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
602 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
603 B, DL, TLI));
604 } else if (HasStr1 && !HasStr2) {
605 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, DL))
606 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
607 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
608 B, DL, TLI));
609 }
610
611 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
612 return nullptr;
613}
614
615// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
616// arrays LHS and RHS and nonconstant Size.
617static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
618 Value *Size, bool StrNCmp,
619 IRBuilderBase &B, const DataLayout &DL);
620
621Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
622 Value *Str1P = CI->getArgOperand(i: 0);
623 Value *Str2P = CI->getArgOperand(i: 1);
624 Value *Size = CI->getArgOperand(i: 2);
625 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
626 return ConstantInt::get(Ty: CI->getType(), V: 0);
627
628 if (isKnownNonZero(V: Size, Q: DL))
629 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
630 // Get the length argument if it is constant.
631 uint64_t Length;
632 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Val: Size))
633 Length = LengthArg->getZExtValue();
634 else
635 return optimizeMemCmpVarSize(CI, LHS: Str1P, RHS: Str2P, Size, StrNCmp: true, B, DL);
636
637 if (Length == 0) // strncmp(x,y,0) -> 0
638 return ConstantInt::get(Ty: CI->getType(), V: 0);
639
640 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
641 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P, Len: Size, B, DL, TLI));
642
643 StringRef Str1, Str2;
644 bool HasStr1 = getConstantStringInfo(V: Str1P, Str&: Str1);
645 bool HasStr2 = getConstantStringInfo(V: Str2P, Str&: Str2);
646
647 // strncmp(x, y) -> cnst (if both x and y are constant strings)
648 if (HasStr1 && HasStr2) {
649 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
650 StringRef SubStr1 = substr(Str: Str1, Len: Length);
651 StringRef SubStr2 = substr(Str: Str2, Len: Length);
652 return ConstantInt::get(Ty: CI->getType(),
653 V: std::clamp(val: SubStr1.compare(RHS: SubStr2), lo: -1, hi: 1));
654 }
655
656 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
657 return B.CreateNeg(V: B.CreateZExt(
658 V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str2P, Name: "strcmpload"), DestTy: CI->getType()));
659
660 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
661 return B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: Str1P, Name: "strcmpload"),
662 DestTy: CI->getType());
663
664 uint64_t Len1 = GetStringLength(V: Str1P);
665 if (Len1)
666 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: Len1);
667 uint64_t Len2 = GetStringLength(V: Str2P);
668 if (Len2)
669 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len2);
670
671 // strncmp to memcmp
672 if (!HasStr1 && HasStr2) {
673 Len2 = std::min(a: Len2, b: Length);
674 if (canTransformToMemCmp(CI, Str: Str1P, Len: Len2, DL))
675 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
676 Len: TLI->getAsSizeT(V: Len2, M: *CI->getModule()),
677 B, DL, TLI));
678 } else if (HasStr1 && !HasStr2) {
679 Len1 = std::min(a: Len1, b: Length);
680 if (canTransformToMemCmp(CI, Str: Str2P, Len: Len1, DL))
681 return copyFlags(Old: *CI, New: emitMemCmp(Ptr1: Str1P, Ptr2: Str2P,
682 Len: TLI->getAsSizeT(V: Len1, M: *CI->getModule()),
683 B, DL, TLI));
684 }
685
686 return nullptr;
687}
688
689Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
690 Value *Src = CI->getArgOperand(i: 0);
691 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
692 uint64_t SrcLen = GetStringLength(V: Src);
693 if (SrcLen && Size) {
694 annotateDereferenceableBytes(CI, ArgNos: 0, DereferenceableBytes: SrcLen);
695 if (SrcLen <= Size->getZExtValue() + 1)
696 return copyFlags(Old: *CI, New: emitStrDup(Ptr: Src, B, TLI));
697 }
698
699 return nullptr;
700}
701
702Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
703 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
704 if (Dst == Src) // strcpy(x,x) -> x
705 return Src;
706
707 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
708 // See if we can get the length of the input string.
709 uint64_t Len = GetStringLength(V: Src);
710 if (Len)
711 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
712 else
713 return nullptr;
714
715 // We have enough information to now generate the memcpy call to do the
716 // copy for us. Make a memcpy to copy the nul byte with align = 1.
717 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
718 Size: TLI->getAsSizeT(V: Len, M: *CI->getModule()));
719 mergeAttributesAndFlags(NewCI, Old: *CI);
720 return Dst;
721}
722
723Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
724 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1);
725
726 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
727 if (CI->use_empty())
728 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
729
730 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
731 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
732 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
733 }
734
735 // See if we can get the length of the input string.
736 uint64_t Len = GetStringLength(V: Src);
737 if (Len)
738 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
739 else
740 return nullptr;
741
742 Value *LenV = TLI->getAsSizeT(V: Len, M: *CI->getModule());
743 Value *DstEnd = B.CreateInBoundsGEP(
744 Ty: B.getInt8Ty(), Ptr: Dst, IdxList: TLI->getAsSizeT(V: Len - 1, M: *CI->getModule()));
745
746 // We have enough information to now generate the memcpy call to do the
747 // copy for us. Make a memcpy to copy the nul byte with align = 1.
748 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: LenV);
749 mergeAttributesAndFlags(NewCI, Old: *CI);
750 return DstEnd;
751}
752
753// Optimize a call to size_t strlcpy(char*, const char*, size_t).
754
755Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
756 Value *Size = CI->getArgOperand(i: 2);
757 if (isKnownNonZero(V: Size, Q: DL))
758 // Like snprintf, the function stores into the destination only when
759 // the size argument is nonzero.
760 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
761 // The function reads the source argument regardless of Size (it returns
762 // its length).
763 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
764
765 uint64_t NBytes;
766 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
767 NBytes = SizeC->getZExtValue();
768 else
769 return nullptr;
770
771 Value *Dst = CI->getArgOperand(i: 0);
772 Value *Src = CI->getArgOperand(i: 1);
773 if (NBytes <= 1) {
774 if (NBytes == 1)
775 // For a call to strlcpy(D, S, 1) first store a nul in *D.
776 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
777
778 // Transform strlcpy(D, S, 0) to a call to strlen(S).
779 return copyFlags(Old: *CI, New: emitStrLen(Ptr: Src, B, DL, TLI));
780 }
781
782 // Try to determine the length of the source, substituting its size
783 // when it's not nul-terminated (as it's required to be) to avoid
784 // reading past its end.
785 StringRef Str;
786 if (!getConstantStringInfo(V: Src, Str, /*TrimAtNul=*/false))
787 return nullptr;
788
789 uint64_t SrcLen = Str.find(C: '\0');
790 // Set if the terminating nul should be copied by the call to memcpy
791 // below.
792 bool NulTerm = SrcLen < NBytes;
793
794 if (NulTerm)
795 // Overwrite NBytes with the number of bytes to copy, including
796 // the terminating nul.
797 NBytes = SrcLen + 1;
798 else {
799 // Set the length of the source for the function to return to its
800 // size, and cap NBytes at the same.
801 SrcLen = std::min(a: SrcLen, b: uint64_t(Str.size()));
802 NBytes = std::min(a: NBytes - 1, b: SrcLen);
803 }
804
805 if (SrcLen == 0) {
806 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
807 B.CreateStore(Val: B.getInt8(C: 0), Ptr: Dst);
808 return ConstantInt::get(Ty: CI->getType(), V: 0);
809 }
810
811 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
812 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
813 // D[N' - 1] if necessary.
814 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
815 Size: TLI->getAsSizeT(V: NBytes, M: *CI->getModule()));
816 mergeAttributesAndFlags(NewCI, Old: *CI);
817
818 if (!NulTerm) {
819 Value *EndOff = ConstantInt::get(Ty: CI->getType(), V: NBytes);
820 Value *EndPtr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: EndOff);
821 B.CreateStore(Val: B.getInt8(C: 0), Ptr: EndPtr);
822 }
823
824 // Like snprintf, strlcpy returns the number of nonzero bytes that would
825 // have been copied if the bound had been sufficiently big (which in this
826 // case is strlen(Src)).
827 return ConstantInt::get(Ty: CI->getType(), V: SrcLen);
828}
829
830// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
831// otherwise.
832Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
833 IRBuilderBase &B) {
834 Value *Dst = CI->getArgOperand(i: 0);
835 Value *Src = CI->getArgOperand(i: 1);
836 Value *Size = CI->getArgOperand(i: 2);
837
838 if (isKnownNonZero(V: Size, Q: DL)) {
839 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
840 // only when N is nonzero.
841 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
842 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 1);
843 }
844
845 // If the "bound" argument is known set N to it. Otherwise set it to
846 // UINT64_MAX and handle it later.
847 uint64_t N = UINT64_MAX;
848 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: Size))
849 N = SizeC->getZExtValue();
850
851 if (N == 0)
852 // Fold st{p,r}ncpy(D, S, 0) to D.
853 return Dst;
854
855 if (N == 1) {
856 Type *CharTy = B.getInt8Ty();
857 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "stxncpy.char0");
858 B.CreateStore(Val: CharVal, Ptr: Dst);
859 if (!RetEnd)
860 // Transform strncpy(D, S, 1) to return (*D = *S), D.
861 return Dst;
862
863 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
864 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
865 Value *Cmp = B.CreateICmpEQ(LHS: CharVal, RHS: ZeroChar, Name: "stpncpy.char0cmp");
866
867 Value *Off1 = B.getInt32(C: 1);
868 Value *EndPtr = B.CreateInBoundsGEP(Ty: CharTy, Ptr: Dst, IdxList: Off1, Name: "stpncpy.end");
869 return B.CreateSelect(C: Cmp, True: Dst, False: EndPtr, Name: "stpncpy.sel");
870 }
871
872 // If the length of the input string is known set SrcLen to it.
873 uint64_t SrcLen = GetStringLength(V: Src);
874 if (SrcLen)
875 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: SrcLen);
876 else
877 return nullptr;
878
879 --SrcLen; // Unbias length.
880
881 if (SrcLen == 0) {
882 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
883 Align MemSetAlign =
884 CI->getAttributes().getParamAttrs(ArgNo: 0).getAlignment().valueOrOne();
885 CallInst *NewCI = B.CreateMemSet(Ptr: Dst, Val: B.getInt8(C: '\0'), Size, Align: MemSetAlign);
886 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(ArgNo: 0));
887 NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
888 C&: CI->getContext(), ArgNo: 0, B: ArgAttrs));
889 copyFlags(Old: *CI, New: NewCI);
890 return Dst;
891 }
892
893 if (N > SrcLen + 1) {
894 if (N > 128)
895 // Bail if N is large or unknown.
896 return nullptr;
897
898 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
899 StringRef Str;
900 if (!getConstantStringInfo(V: Src, Str))
901 return nullptr;
902 std::string SrcStr = Str.str();
903 // Create a bigger, nul-padded array with the same length, SrcLen,
904 // as the original string.
905 SrcStr.resize(n: N, c: '\0');
906 Src = B.CreateGlobalString(Str: SrcStr, Name: "str", /*AddressSpace=*/0,
907 /*M=*/nullptr, /*AddNull=*/false);
908 }
909
910 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
911 // S and N are constant.
912 CallInst *NewCI = B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
913 Size: TLI->getAsSizeT(V: N, M: *CI->getModule()));
914 mergeAttributesAndFlags(NewCI, Old: *CI);
915 if (!RetEnd)
916 return Dst;
917
918 // stpncpy(D, S, N) returns the address of the first null in D if it writes
919 // one, otherwise D + N.
920 Value *Off = B.getInt64(C: std::min(a: SrcLen, b: N));
921 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: Off, Name: "endptr");
922}
923
924Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
925 unsigned CharSize,
926 Value *Bound) {
927 Value *Src = CI->getArgOperand(i: 0);
928 Type *CharTy = B.getIntNTy(N: CharSize);
929
930 if (isOnlyUsedInZeroEqualityComparison(CxtI: CI) &&
931 (!Bound || isKnownNonZero(V: Bound, Q: DL))) {
932 // Fold strlen:
933 // strlen(x) != 0 --> *x != 0
934 // strlen(x) == 0 --> *x == 0
935 // and likewise strnlen with constant N > 0:
936 // strnlen(x, N) != 0 --> *x != 0
937 // strnlen(x, N) == 0 --> *x == 0
938 return B.CreateZExt(V: B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "char0"),
939 DestTy: CI->getType());
940 }
941
942 if (Bound) {
943 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Val: Bound)) {
944 if (BoundCst->isZero())
945 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
946 return ConstantInt::get(Ty: CI->getType(), V: 0);
947
948 if (BoundCst->isOne()) {
949 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
950 Value *CharVal = B.CreateLoad(Ty: CharTy, Ptr: Src, Name: "strnlen.char0");
951 Value *ZeroChar = ConstantInt::get(Ty: CharTy, V: 0);
952 Value *Cmp = B.CreateICmpNE(LHS: CharVal, RHS: ZeroChar, Name: "strnlen.char0cmp");
953 return B.CreateZExt(V: Cmp, DestTy: CI->getType());
954 }
955 }
956 }
957
958 if (uint64_t Len = GetStringLength(V: Src, CharSize)) {
959 Value *LenC = ConstantInt::get(Ty: CI->getType(), V: Len - 1);
960 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
961 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
962 if (Bound)
963 return B.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: LenC, RHS: Bound);
964 return LenC;
965 }
966
967 if (Bound)
968 // Punt for strnlen for now.
969 return nullptr;
970
971 // If s is a constant pointer pointing to a string literal, we can fold
972 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
973 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
974 // We only try to simplify strlen when the pointer s points to an array
975 // of CharSize elements. Otherwise, we would need to scale the offset x before
976 // doing the subtraction. This will make the optimization more complex, and
977 // it's not very useful because calling strlen for a pointer of other types is
978 // very uncommon.
979 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: Src)) {
980 // TODO: Handle subobjects.
981 if (!isGEPBasedOnPointerToString(GEP, CharSize))
982 return nullptr;
983
984 ConstantDataArraySlice Slice;
985 if (getConstantDataArrayInfo(V: GEP->getOperand(i_nocapture: 0), Slice, ElementSize: CharSize)) {
986 uint64_t NullTermIdx;
987 if (Slice.Array == nullptr) {
988 NullTermIdx = 0;
989 } else {
990 NullTermIdx = ~((uint64_t)0);
991 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
992 if (Slice.Array->getElementAsInteger(i: I + Slice.Offset) == 0) {
993 NullTermIdx = I;
994 break;
995 }
996 }
997 // If the string does not have '\0', leave it to strlen to compute
998 // its length.
999 if (NullTermIdx == ~((uint64_t)0))
1000 return nullptr;
1001 }
1002
1003 Value *Offset = GEP->getOperand(i_nocapture: 2);
1004 KnownBits Known = computeKnownBits(V: Offset, DL, AC: nullptr, CxtI: CI, DT: nullptr);
1005 uint64_t ArrSize =
1006 cast<ArrayType>(Val: GEP->getSourceElementType())->getNumElements();
1007
1008 // If Offset is not provably in the range [0, NullTermIdx], we can still
1009 // optimize if we can prove that the program has undefined behavior when
1010 // Offset is outside that range. That is the case when GEP->getOperand(0)
1011 // is a pointer to an object whose memory extent is NullTermIdx+1.
1012 if ((Known.isNonNegative() && Known.getMaxValue().ule(RHS: NullTermIdx)) ||
1013 (isa<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)) &&
1014 NullTermIdx == ArrSize - 1)) {
1015 Offset = B.CreateSExtOrTrunc(V: Offset, DestTy: CI->getType());
1016 return B.CreateSub(LHS: ConstantInt::get(Ty: CI->getType(), V: NullTermIdx),
1017 RHS: Offset);
1018 }
1019 }
1020 }
1021
1022 // strlen(x?"foo":"bars") --> x ? 3 : 4
1023 if (SelectInst *SI = dyn_cast<SelectInst>(Val: Src)) {
1024 uint64_t LenTrue = GetStringLength(V: SI->getTrueValue(), CharSize);
1025 uint64_t LenFalse = GetStringLength(V: SI->getFalseValue(), CharSize);
1026 if (LenTrue && LenFalse) {
1027 ORE.emit(RemarkBuilder: [&]() {
1028 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1029 << "folded strlen(select) to select of constants";
1030 });
1031 return B.CreateSelect(C: SI->getCondition(),
1032 True: ConstantInt::get(Ty: CI->getType(), V: LenTrue - 1),
1033 False: ConstantInt::get(Ty: CI->getType(), V: LenFalse - 1));
1034 }
1035 }
1036
1037 return nullptr;
1038}
1039
1040Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1041 if (Value *V = optimizeStringLength(CI, B, CharSize: 8))
1042 return V;
1043 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1044 return nullptr;
1045}
1046
1047Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1048 Value *Bound = CI->getArgOperand(i: 1);
1049 if (Value *V = optimizeStringLength(CI, B, CharSize: 8, Bound))
1050 return V;
1051
1052 if (isKnownNonZero(V: Bound, Q: DL))
1053 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1054 return nullptr;
1055}
1056
1057Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1058 Module &M = *CI->getModule();
1059 unsigned WCharSize = TLI->getWCharSize(M) * 8;
1060 // We cannot perform this optimization without wchar_size metadata.
1061 if (WCharSize == 0)
1062 return nullptr;
1063
1064 return optimizeStringLength(CI, B, CharSize: WCharSize);
1065}
1066
1067Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1068 StringRef S1, S2;
1069 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1070 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1071
1072 // strpbrk(s, "") -> nullptr
1073 // strpbrk("", s) -> nullptr
1074 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1075 return Constant::getNullValue(Ty: CI->getType());
1076
1077 // Constant folding.
1078 if (HasS1 && HasS2) {
1079 size_t I = S1.find_first_of(Chars: S2);
1080 if (I == StringRef::npos) // No match.
1081 return Constant::getNullValue(Ty: CI->getType());
1082
1083 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1084 IdxList: B.getInt64(C: I), Name: "strpbrk");
1085 }
1086
1087 // strpbrk(s, "a") -> strchr(s, 'a')
1088 if (HasS2 && S2.size() == 1)
1089 return copyFlags(Old: *CI, New: emitStrChr(Ptr: CI->getArgOperand(i: 0), C: S2[0], B, TLI));
1090
1091 return nullptr;
1092}
1093
1094Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1095 Value *EndPtr = CI->getArgOperand(i: 1);
1096 if (isa<ConstantPointerNull>(Val: EndPtr)) {
1097 // With a null EndPtr, this function won't capture the main argument.
1098 // It would be readonly too, except that it still may write to errno.
1099 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
1100 CI: CaptureInfo::none()));
1101 }
1102
1103 return nullptr;
1104}
1105
1106Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1107 StringRef S1, S2;
1108 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1109 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1110
1111 // strspn(s, "") -> 0
1112 // strspn("", s) -> 0
1113 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1114 return Constant::getNullValue(Ty: CI->getType());
1115
1116 // Constant folding.
1117 if (HasS1 && HasS2) {
1118 size_t Pos = S1.find_first_not_of(Chars: S2);
1119 if (Pos == StringRef::npos)
1120 Pos = S1.size();
1121 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1122 }
1123
1124 return nullptr;
1125}
1126
1127Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1128 StringRef S1, S2;
1129 bool HasS1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: S1);
1130 bool HasS2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: S2);
1131
1132 // strcspn("", s) -> 0
1133 if (HasS1 && S1.empty())
1134 return Constant::getNullValue(Ty: CI->getType());
1135
1136 // Constant folding.
1137 if (HasS1 && HasS2) {
1138 size_t Pos = S1.find_first_of(Chars: S2);
1139 if (Pos == StringRef::npos)
1140 Pos = S1.size();
1141 return ConstantInt::get(Ty: CI->getType(), V: Pos);
1142 }
1143
1144 // strcspn(s, "") -> strlen(s)
1145 if (HasS2 && S2.empty())
1146 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B, DL, TLI));
1147
1148 return nullptr;
1149}
1150
1151Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1152 // fold strstr(x, x) -> x.
1153 if (CI->getArgOperand(i: 0) == CI->getArgOperand(i: 1))
1154 return CI->getArgOperand(i: 0);
1155
1156 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1157 if (isOnlyUsedInEqualityComparison(V: CI, With: CI->getArgOperand(i: 0))) {
1158 Value *StrLen = emitStrLen(Ptr: CI->getArgOperand(i: 1), B, DL, TLI);
1159 if (!StrLen)
1160 return nullptr;
1161 Value *StrNCmp = emitStrNCmp(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
1162 Len: StrLen, B, DL, TLI);
1163 if (!StrNCmp)
1164 return nullptr;
1165 for (User *U : llvm::make_early_inc_range(Range: CI->users())) {
1166 ICmpInst *Old = cast<ICmpInst>(Val: U);
1167 Value *Cmp =
1168 B.CreateICmp(P: Old->getPredicate(), LHS: StrNCmp,
1169 RHS: ConstantInt::getNullValue(Ty: StrNCmp->getType()), Name: "cmp");
1170 replaceAllUsesWith(I: Old, With: Cmp);
1171 }
1172 return CI;
1173 }
1174
1175 // See if either input string is a constant string.
1176 StringRef SearchStr, ToFindStr;
1177 bool HasStr1 = getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: SearchStr);
1178 bool HasStr2 = getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: ToFindStr);
1179
1180 // fold strstr(x, "") -> x.
1181 if (HasStr2 && ToFindStr.empty())
1182 return CI->getArgOperand(i: 0);
1183
1184 // If both strings are known, constant fold it.
1185 if (HasStr1 && HasStr2) {
1186 size_t Offset = SearchStr.find(Str: ToFindStr);
1187
1188 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1189 return Constant::getNullValue(Ty: CI->getType());
1190
1191 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1192 return B.CreateConstInBoundsGEP1_64(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0),
1193 Idx0: Offset, Name: "strstr");
1194 }
1195
1196 // fold strstr(x, "y") -> strchr(x, 'y').
1197 if (HasStr2 && ToFindStr.size() == 1) {
1198 return emitStrChr(Ptr: CI->getArgOperand(i: 0), C: ToFindStr[0], B, TLI);
1199 }
1200
1201 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
1202 return nullptr;
1203}
1204
1205Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1206 Value *SrcStr = CI->getArgOperand(i: 0);
1207 Value *Size = CI->getArgOperand(i: 2);
1208 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1209 Value *CharVal = CI->getArgOperand(i: 1);
1210 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1211 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1212
1213 if (LenC) {
1214 if (LenC->isZero())
1215 // Fold memrchr(x, y, 0) --> null.
1216 return NullPtr;
1217
1218 if (LenC->isOne()) {
1219 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1220 // constant or otherwise.
1221 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memrchr.char0");
1222 // Slice off the character's high end bits.
1223 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1224 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memrchr.char0cmp");
1225 return B.CreateSelect(C: Cmp, True: SrcStr, False: NullPtr, Name: "memrchr.sel");
1226 }
1227 }
1228
1229 StringRef Str;
1230 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1231 return nullptr;
1232
1233 if (Str.size() == 0)
1234 // If the array is empty fold memrchr(A, C, N) to null for any value
1235 // of C and N on the basis that the only valid value of N is zero
1236 // (otherwise the call is undefined).
1237 return NullPtr;
1238
1239 uint64_t EndOff = UINT64_MAX;
1240 if (LenC) {
1241 EndOff = LenC->getZExtValue();
1242 if (Str.size() < EndOff)
1243 // Punt out-of-bounds accesses to sanitizers and/or libc.
1244 return nullptr;
1245 }
1246
1247 if (ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal)) {
1248 // Fold memrchr(S, C, N) for a constant C.
1249 size_t Pos = Str.rfind(C: CharC->getZExtValue(), From: EndOff);
1250 if (Pos == StringRef::npos)
1251 // When the character is not in the source array fold the result
1252 // to null regardless of Size.
1253 return NullPtr;
1254
1255 if (LenC)
1256 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1257 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos));
1258
1259 if (Str.find(C: Str[Pos]) == Pos) {
1260 // When there is just a single occurrence of C in S, i.e., the one
1261 // in Str[Pos], fold
1262 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1263 // for nonconstant N.
1264 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1265 Name: "memrchr.cmp");
1266 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr,
1267 IdxList: B.getInt64(C: Pos), Name: "memrchr.ptr_plus");
1268 return B.CreateSelect(C: Cmp, True: NullPtr, False: SrcPlus, Name: "memrchr.sel");
1269 }
1270 }
1271
1272 // Truncate the string to search at most EndOff characters.
1273 Str = Str.substr(Start: 0, N: EndOff);
1274 if (Str.find_first_not_of(C: Str[0]) != StringRef::npos)
1275 return nullptr;
1276
1277 // If the source array consists of all equal characters, then for any
1278 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1279 // N != 0 && *S == C ? S + N - 1 : null
1280 Type *SizeTy = Size->getType();
1281 Type *Int8Ty = B.getInt8Ty();
1282 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1283 // Slice off the sought character's high end bits.
1284 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1285 Value *CEqS0 = B.CreateICmpEQ(LHS: ConstantInt::get(Ty: Int8Ty, V: Str[0]), RHS: CharVal);
1286 Value *And = B.CreateLogicalAnd(Cond1: NNeZ, Cond2: CEqS0);
1287 Value *SizeM1 = B.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
1288 Value *SrcPlus =
1289 B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: SrcStr, IdxList: SizeM1, Name: "memrchr.ptr_plus");
1290 return B.CreateSelect(C: And, True: SrcPlus, False: NullPtr, Name: "memrchr.sel");
1291}
1292
1293Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1294 Value *SrcStr = CI->getArgOperand(i: 0);
1295 Value *Size = CI->getArgOperand(i: 2);
1296
1297 if (isKnownNonZero(V: Size, Q: DL)) {
1298 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
1299 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1300 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1301 }
1302
1303 Value *CharVal = CI->getArgOperand(i: 1);
1304 ConstantInt *CharC = dyn_cast<ConstantInt>(Val: CharVal);
1305 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1306 Value *NullPtr = Constant::getNullValue(Ty: CI->getType());
1307
1308 // memchr(x, y, 0) -> null
1309 if (LenC) {
1310 if (LenC->isZero())
1311 return NullPtr;
1312
1313 if (LenC->isOne()) {
1314 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1315 // constant or otherwise.
1316 Value *Val = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: SrcStr, Name: "memchr.char0");
1317 // Slice off the character's high end bits.
1318 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1319 Value *Cmp = B.CreateICmpEQ(LHS: Val, RHS: CharVal, Name: "memchr.char0cmp");
1320 return B.CreateSelect(C: Cmp, True: SrcStr, False: NullPtr, Name: "memchr.sel");
1321 }
1322 }
1323
1324 StringRef Str;
1325 if (!getConstantStringInfo(V: SrcStr, Str, /*TrimAtNul=*/false))
1326 return nullptr;
1327
1328 if (CharC) {
1329 size_t Pos = Str.find(C: CharC->getZExtValue());
1330 if (Pos == StringRef::npos)
1331 // When the character is not in the source array fold the result
1332 // to null regardless of Size.
1333 return NullPtr;
1334
1335 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1336 // When the constant Size is less than or equal to the character
1337 // position also fold the result to null.
1338 Value *Cmp = B.CreateICmpULE(LHS: Size, RHS: ConstantInt::get(Ty: Size->getType(), V: Pos),
1339 Name: "memchr.cmp");
1340 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: B.getInt64(C: Pos),
1341 Name: "memchr.ptr");
1342 return B.CreateSelect(C: Cmp, True: NullPtr, False: SrcPlus);
1343 }
1344
1345 if (Str.size() == 0)
1346 // If the array is empty fold memchr(A, C, N) to null for any value
1347 // of C and N on the basis that the only valid value of N is zero
1348 // (otherwise the call is undefined).
1349 return NullPtr;
1350
1351 if (LenC)
1352 Str = substr(Str, Len: LenC->getZExtValue());
1353
1354 size_t Pos = Str.find_first_not_of(C: Str[0]);
1355 if (Pos == StringRef::npos
1356 || Str.find_first_not_of(C: Str[Pos], From: Pos) == StringRef::npos) {
1357 // If the source array consists of at most two consecutive sequences
1358 // of the same characters, then for any C and N (whether in bounds or
1359 // not), fold memchr(S, C, N) to
1360 // N != 0 && *S == C ? S : null
1361 // or for the two sequences to:
1362 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1363 // ^Sel2 ^Sel1 are denoted above.
1364 // The latter makes it also possible to fold strchr() calls with strings
1365 // of the same characters.
1366 Type *SizeTy = Size->getType();
1367 Type *Int8Ty = B.getInt8Ty();
1368
1369 // Slice off the sought character's high end bits.
1370 CharVal = B.CreateTrunc(V: CharVal, DestTy: Int8Ty);
1371
1372 Value *Sel1 = NullPtr;
1373 if (Pos != StringRef::npos) {
1374 // Handle two consecutive sequences of the same characters.
1375 Value *PosVal = ConstantInt::get(Ty: SizeTy, V: Pos);
1376 Value *StrPos = ConstantInt::get(Ty: Int8Ty, V: Str[Pos]);
1377 Value *CEqSPos = B.CreateICmpEQ(LHS: CharVal, RHS: StrPos);
1378 Value *NGtPos = B.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: Size, RHS: PosVal);
1379 Value *And = B.CreateAnd(LHS: CEqSPos, RHS: NGtPos);
1380 Value *SrcPlus = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: SrcStr, IdxList: PosVal);
1381 Sel1 = B.CreateSelect(C: And, True: SrcPlus, False: NullPtr, Name: "memchr.sel1");
1382 }
1383
1384 Value *Str0 = ConstantInt::get(Ty: Int8Ty, V: Str[0]);
1385 Value *CEqS0 = B.CreateICmpEQ(LHS: Str0, RHS: CharVal);
1386 Value *NNeZ = B.CreateICmpNE(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
1387 Value *And = B.CreateAnd(LHS: NNeZ, RHS: CEqS0);
1388 return B.CreateSelect(C: And, True: SrcStr, False: Sel1, Name: "memchr.sel2");
1389 }
1390
1391 if (!LenC) {
1392 if (isOnlyUsedInEqualityComparison(V: CI, With: SrcStr))
1393 // S is dereferenceable so it's safe to load from it and fold
1394 // memchr(S, C, N) == S to N && *S == C for any C and N.
1395 // TODO: This is safe even for nonconstant S.
1396 return memChrToCharCompare(CI, NBytes: Size, B, DL);
1397
1398 // From now on we need a constant length and constant array.
1399 return nullptr;
1400 }
1401
1402 bool OptForSize = llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
1403 QueryType: PGSOQueryType::IRPass);
1404
1405 // If the char is variable but the input str and length are not we can turn
1406 // this memchr call into a simple bit field test. Of course this only works
1407 // when the return value is only checked against null.
1408 //
1409 // It would be really nice to reuse switch lowering here but we can't change
1410 // the CFG at this point.
1411 //
1412 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1413 // != 0
1414 // after bounds check.
1415 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CxtI: CI))
1416 return nullptr;
1417
1418 unsigned char Max =
1419 *std::max_element(first: reinterpret_cast<const unsigned char *>(Str.begin()),
1420 last: reinterpret_cast<const unsigned char *>(Str.end()));
1421
1422 // Make sure the bit field we're about to create fits in a register on the
1423 // target.
1424 // FIXME: On a 64 bit architecture this prevents us from using the
1425 // interesting range of alpha ascii chars. We could do better by emitting
1426 // two bitfields or shifting the range by 64 if no lower chars are used.
1427 if (!DL.fitsInLegalInteger(Width: Max + 1)) {
1428 // Build chain of ORs
1429 // Transform:
1430 // memchr("abcd", C, 4) != nullptr
1431 // to:
1432 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1433 std::string SortedStr = Str.str();
1434 llvm::sort(C&: SortedStr);
1435 // Compute the number of of non-contiguous ranges.
1436 unsigned NonContRanges = 1;
1437 for (size_t i = 1; i < SortedStr.size(); ++i) {
1438 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1439 NonContRanges++;
1440 }
1441 }
1442
1443 // Restrict this optimization to profitable cases with one or two range
1444 // checks.
1445 if (NonContRanges > 2)
1446 return nullptr;
1447
1448 // Slice off the character's high end bits.
1449 CharVal = B.CreateTrunc(V: CharVal, DestTy: B.getInt8Ty());
1450
1451 SmallVector<Value *> CharCompares;
1452 for (unsigned char C : SortedStr)
1453 CharCompares.push_back(Elt: B.CreateICmpEQ(LHS: CharVal, RHS: B.getInt8(C)));
1454
1455 return B.CreateIntToPtr(V: B.CreateOr(Ops: CharCompares), DestTy: CI->getType());
1456 }
1457
1458 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1459 // creating unnecessary illegal types.
1460 unsigned char Width = NextPowerOf2(A: std::max(a: (unsigned char)7, b: Max));
1461
1462 // Now build the bit field.
1463 APInt Bitfield(Width, 0);
1464 for (char C : Str)
1465 Bitfield.setBit((unsigned char)C);
1466 Value *BitfieldC = B.getInt(AI: Bitfield);
1467
1468 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1469 Value *C = B.CreateZExtOrTrunc(V: CharVal, DestTy: BitfieldC->getType());
1470 C = B.CreateAnd(LHS: C, RHS: B.getIntN(N: Width, C: 0xFF));
1471
1472 // First check that the bit field access is within bounds.
1473 Value *Bounds = B.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: C, RHS: B.getIntN(N: Width, C: Width),
1474 Name: "memchr.bounds");
1475
1476 // Create code that checks if the given bit is set in the field.
1477 Value *Shl = B.CreateShl(LHS: B.getIntN(N: Width, C: 1ULL), RHS: C);
1478 Value *Bits = B.CreateIsNotNull(Arg: B.CreateAnd(LHS: Shl, RHS: BitfieldC), Name: "memchr.bits");
1479
1480 // Finally merge both checks and cast to pointer type. The inttoptr
1481 // implicitly zexts the i1 to intptr type.
1482 return B.CreateIntToPtr(V: B.CreateLogicalAnd(Cond1: Bounds, Cond2: Bits, Name: "memchr"),
1483 DestTy: CI->getType());
1484}
1485
1486// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1487// arrays LHS and RHS and nonconstant Size.
1488static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
1489 Value *Size, bool StrNCmp,
1490 IRBuilderBase &B, const DataLayout &DL) {
1491 if (LHS == RHS) // memcmp(s,s,x) -> 0
1492 return Constant::getNullValue(Ty: CI->getType());
1493
1494 StringRef LStr, RStr;
1495 if (!getConstantStringInfo(V: LHS, Str&: LStr, /*TrimAtNul=*/false) ||
1496 !getConstantStringInfo(V: RHS, Str&: RStr, /*TrimAtNul=*/false))
1497 return nullptr;
1498
1499 // If the contents of both constant arrays are known, fold a call to
1500 // memcmp(A, B, N) to
1501 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1502 // where Pos is the first mismatch between A and B, determined below.
1503
1504 uint64_t Pos = 0;
1505 Value *Zero = ConstantInt::get(Ty: CI->getType(), V: 0);
1506 for (uint64_t MinSize = std::min(a: LStr.size(), b: RStr.size()); ; ++Pos) {
1507 if (Pos == MinSize ||
1508 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1509 // One array is a leading part of the other of equal or greater
1510 // size, or for strncmp, the arrays are equal strings.
1511 // Fold the result to zero. Size is assumed to be in bounds, since
1512 // otherwise the call would be undefined.
1513 return Zero;
1514 }
1515
1516 if (LStr[Pos] != RStr[Pos])
1517 break;
1518 }
1519
1520 // Normalize the result.
1521 typedef unsigned char UChar;
1522 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1523 Value *MaxSize = ConstantInt::get(Ty: Size->getType(), V: Pos);
1524 Value *Cmp = B.CreateICmp(P: ICmpInst::ICMP_ULE, LHS: Size, RHS: MaxSize);
1525 Value *Res = ConstantInt::get(Ty: CI->getType(), V: IRes);
1526 return B.CreateSelect(C: Cmp, True: Zero, False: Res);
1527}
1528
1529// Optimize a memcmp call CI with constant size Len.
1530static Value *optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS,
1531 uint64_t Len, IRBuilderBase &B,
1532 const DataLayout &DL) {
1533 if (Len == 0) // memcmp(s1,s2,0) -> 0
1534 return Constant::getNullValue(Ty: CI->getType());
1535
1536 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1537 if (Len == 1) {
1538 Value *LHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: LHS, Name: "lhsc"),
1539 DestTy: CI->getType(), Name: "lhsv");
1540 Value *RHSV = B.CreateZExt(V: B.CreateLoad(Ty: B.getInt8Ty(), Ptr: RHS, Name: "rhsc"),
1541 DestTy: CI->getType(), Name: "rhsv");
1542 return B.CreateSub(LHS: LHSV, RHS: RHSV, Name: "chardiff");
1543 }
1544
1545 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1546 // TODO: The case where both inputs are constants does not need to be limited
1547 // to legal integers or equality comparison. See block below this.
1548 if (DL.isLegalInteger(Width: Len * 8) && isOnlyUsedInZeroEqualityComparison(CxtI: CI)) {
1549 IntegerType *IntType = IntegerType::get(C&: CI->getContext(), NumBits: Len * 8);
1550 Align PrefAlignment = DL.getPrefTypeAlign(Ty: IntType);
1551
1552 // First, see if we can fold either argument to a constant.
1553 Value *LHSV = nullptr;
1554 if (auto *LHSC = dyn_cast<Constant>(Val: LHS))
1555 LHSV = ConstantFoldLoadFromConstPtr(C: LHSC, Ty: IntType, DL);
1556
1557 Value *RHSV = nullptr;
1558 if (auto *RHSC = dyn_cast<Constant>(Val: RHS))
1559 RHSV = ConstantFoldLoadFromConstPtr(C: RHSC, Ty: IntType, DL);
1560
1561 // Don't generate unaligned loads. If either source is constant data,
1562 // alignment doesn't matter for that source because there is no load.
1563 if ((LHSV || getKnownAlignment(V: LHS, DL, CxtI: CI) >= PrefAlignment) &&
1564 (RHSV || getKnownAlignment(V: RHS, DL, CxtI: CI) >= PrefAlignment)) {
1565 if (!LHSV)
1566 LHSV = B.CreateLoad(Ty: IntType, Ptr: LHS, Name: "lhsv");
1567 if (!RHSV)
1568 RHSV = B.CreateLoad(Ty: IntType, Ptr: RHS, Name: "rhsv");
1569 return B.CreateZExt(V: B.CreateICmpNE(LHS: LHSV, RHS: RHSV), DestTy: CI->getType(), Name: "memcmp");
1570 }
1571 }
1572
1573 return nullptr;
1574}
1575
1576// Most simplifications for memcmp also apply to bcmp.
1577Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1578 IRBuilderBase &B) {
1579 Value *LHS = CI->getArgOperand(i: 0), *RHS = CI->getArgOperand(i: 1);
1580 Value *Size = CI->getArgOperand(i: 2);
1581
1582 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1583
1584 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, StrNCmp: false, B, DL))
1585 return Res;
1586
1587 // Handle constant Size.
1588 ConstantInt *LenC = dyn_cast<ConstantInt>(Val: Size);
1589 if (!LenC)
1590 return nullptr;
1591
1592 return optimizeMemCmpConstantSize(CI, LHS, RHS, Len: LenC->getZExtValue(), B, DL);
1593}
1594
1595Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1596 Module *M = CI->getModule();
1597 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1598 return V;
1599
1600 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1601 // bcmp can be more efficient than memcmp because it only has to know that
1602 // there is a difference, not how different one is to the other.
1603 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_bcmp) &&
1604 isOnlyUsedInZeroEqualityComparison(CxtI: CI)) {
1605 Value *LHS = CI->getArgOperand(i: 0);
1606 Value *RHS = CI->getArgOperand(i: 1);
1607 Value *Size = CI->getArgOperand(i: 2);
1608 return copyFlags(Old: *CI, New: emitBCmp(Ptr1: LHS, Ptr2: RHS, Len: Size, B, DL, TLI));
1609 }
1610
1611 return nullptr;
1612}
1613
1614Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1615 return optimizeMemCmpBCmpCommon(CI, B);
1616}
1617
1618Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1619 Value *Size = CI->getArgOperand(i: 2);
1620 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1621 if (isa<IntrinsicInst>(Val: CI))
1622 return nullptr;
1623
1624 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1625 CallInst *NewCI = B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1626 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1627 mergeAttributesAndFlags(NewCI, Old: *CI);
1628 return CI->getArgOperand(i: 0);
1629}
1630
1631Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1632 Value *Dst = CI->getArgOperand(i: 0);
1633 Value *Src = CI->getArgOperand(i: 1);
1634 ConstantInt *StopChar = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
1635 ConstantInt *N = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
1636 StringRef SrcStr;
1637 if (CI->use_empty() && Dst == Src)
1638 return Dst;
1639 // memccpy(d, s, c, 0) -> nullptr
1640 if (N) {
1641 if (N->isNullValue())
1642 return Constant::getNullValue(Ty: CI->getType());
1643 if (!getConstantStringInfo(V: Src, Str&: SrcStr, /*TrimAtNul=*/false) ||
1644 // TODO: Handle zeroinitializer.
1645 !StopChar)
1646 return nullptr;
1647 } else {
1648 return nullptr;
1649 }
1650
1651 // Wrap arg 'c' of type int to char
1652 size_t Pos = SrcStr.find(C: StopChar->getSExtValue() & 0xFF);
1653 if (Pos == StringRef::npos) {
1654 if (N->getZExtValue() <= SrcStr.size()) {
1655 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1),
1656 Size: CI->getArgOperand(i: 3)));
1657 return Constant::getNullValue(Ty: CI->getType());
1658 }
1659 return nullptr;
1660 }
1661
1662 Value *NewN =
1663 ConstantInt::get(Ty: N->getType(), V: std::min(a: uint64_t(Pos + 1), b: N->getZExtValue()));
1664 // memccpy -> llvm.memcpy
1665 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst, DstAlign: Align(1), Src, SrcAlign: Align(1), Size: NewN));
1666 return Pos + 1 <= N->getZExtValue()
1667 ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: NewN)
1668 : Constant::getNullValue(Ty: CI->getType());
1669}
1670
1671Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1672 Value *Dst = CI->getArgOperand(i: 0);
1673 Value *N = CI->getArgOperand(i: 2);
1674 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1675 CallInst *NewCI =
1676 B.CreateMemCpy(Dst, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size: N);
1677 // Propagate attributes, but memcpy has no return value, so make sure that
1678 // any return attributes are compliant.
1679 // TODO: Attach return value attributes to the 1st operand to preserve them?
1680 mergeAttributesAndFlags(NewCI, Old: *CI);
1681 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: N);
1682}
1683
1684Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1685 Value *Size = CI->getArgOperand(i: 2);
1686 annotateNonNullAndDereferenceable(CI, ArgNos: {0, 1}, Size, DL);
1687 if (isa<IntrinsicInst>(Val: CI))
1688 return nullptr;
1689
1690 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1691 CallInst *NewCI = B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1),
1692 Src: CI->getArgOperand(i: 1), SrcAlign: Align(1), Size);
1693 mergeAttributesAndFlags(NewCI, Old: *CI);
1694 return CI->getArgOperand(i: 0);
1695}
1696
1697Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1698 Value *Size = CI->getArgOperand(i: 2);
1699 annotateNonNullAndDereferenceable(CI, ArgNos: 0, Size, DL);
1700 if (isa<IntrinsicInst>(Val: CI))
1701 return nullptr;
1702
1703 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1704 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
1705 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val, Size, Align: Align(1));
1706 mergeAttributesAndFlags(NewCI, Old: *CI);
1707 return CI->getArgOperand(i: 0);
1708}
1709
1710Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1711 if (isa<ConstantPointerNull>(Val: CI->getArgOperand(i: 0)))
1712 return copyFlags(Old: *CI, New: emitMalloc(Num: CI->getArgOperand(i: 1), B, DL, TLI));
1713
1714 return nullptr;
1715}
1716
1717// When enabled, replace operator new() calls marked with a hot or cold memprof
1718// attribute with an operator new() call that takes a __hot_cold_t parameter.
1719// Currently this is supported by the open source version of tcmalloc, see:
1720// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1721Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1722 LibFunc &Func) {
1723 if (!OptimizeHotColdNew)
1724 return nullptr;
1725
1726 uint8_t HotCold;
1727 if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold")
1728 HotCold = ColdNewHintValue;
1729 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() ==
1730 "notcold")
1731 HotCold = NotColdNewHintValue;
1732 else if (CI->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "hot")
1733 HotCold = HotNewHintValue;
1734 else
1735 return nullptr;
1736
1737 // For calls that already pass a hot/cold hint, only update the hint if
1738 // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
1739 // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
1740 // Note that in cases where we decide it is "notcold", it might be slightly
1741 // better to replace the hinted call with a non hinted call, to avoid the
1742 // extra parameter and the if condition check of the hint value in the
1743 // allocator. This can be considered in the future.
1744 switch (Func) {
1745 case LibFunc_Znwm12__hot_cold_t:
1746 if (OptimizeExistingHotColdNew)
1747 return emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1748 NewFunc: LibFunc_Znwm12__hot_cold_t, HotCold);
1749 break;
1750 case LibFunc_Znwm:
1751 if (HotCold != NotColdNewHintValue)
1752 return emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1753 NewFunc: LibFunc_Znwm12__hot_cold_t, HotCold);
1754 break;
1755 case LibFunc_Znam12__hot_cold_t:
1756 if (OptimizeExistingHotColdNew)
1757 return emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1758 NewFunc: LibFunc_Znam12__hot_cold_t, HotCold);
1759 break;
1760 case LibFunc_Znam:
1761 if (HotCold != NotColdNewHintValue)
1762 return emitHotColdNew(Num: CI->getArgOperand(i: 0), B, TLI,
1763 NewFunc: LibFunc_Znam12__hot_cold_t, HotCold);
1764 break;
1765 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1766 if (OptimizeExistingHotColdNew)
1767 return emitHotColdNewNoThrow(
1768 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1769 NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
1770 break;
1771 case LibFunc_ZnwmRKSt9nothrow_t:
1772 if (HotCold != NotColdNewHintValue)
1773 return emitHotColdNewNoThrow(
1774 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1775 NewFunc: LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
1776 break;
1777 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1778 if (OptimizeExistingHotColdNew)
1779 return emitHotColdNewNoThrow(
1780 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1781 NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
1782 break;
1783 case LibFunc_ZnamRKSt9nothrow_t:
1784 if (HotCold != NotColdNewHintValue)
1785 return emitHotColdNewNoThrow(
1786 Num: CI->getArgOperand(i: 0), NoThrow: CI->getArgOperand(i: 1), B, TLI,
1787 NewFunc: LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
1788 break;
1789 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1790 if (OptimizeExistingHotColdNew)
1791 return emitHotColdNewAligned(
1792 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1793 NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
1794 break;
1795 case LibFunc_ZnwmSt11align_val_t:
1796 if (HotCold != NotColdNewHintValue)
1797 return emitHotColdNewAligned(
1798 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1799 NewFunc: LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
1800 break;
1801 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1802 if (OptimizeExistingHotColdNew)
1803 return emitHotColdNewAligned(
1804 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1805 NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
1806 break;
1807 case LibFunc_ZnamSt11align_val_t:
1808 if (HotCold != NotColdNewHintValue)
1809 return emitHotColdNewAligned(
1810 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1811 NewFunc: LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
1812 break;
1813 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1814 if (OptimizeExistingHotColdNew)
1815 return emitHotColdNewAlignedNoThrow(
1816 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1817 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1818 HotCold);
1819 break;
1820 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1821 if (HotCold != NotColdNewHintValue)
1822 return emitHotColdNewAlignedNoThrow(
1823 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1824 TLI, NewFunc: LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1825 HotCold);
1826 break;
1827 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1828 if (OptimizeExistingHotColdNew)
1829 return emitHotColdNewAlignedNoThrow(
1830 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1831 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1832 HotCold);
1833 break;
1834 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1835 if (HotCold != NotColdNewHintValue)
1836 return emitHotColdNewAlignedNoThrow(
1837 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), NoThrow: CI->getArgOperand(i: 2), B,
1838 TLI, NewFunc: LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1839 HotCold);
1840 break;
1841 case LibFunc_size_returning_new:
1842 if (HotCold != NotColdNewHintValue)
1843 return emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
1844 NewFunc: LibFunc_size_returning_new_hot_cold,
1845 HotCold);
1846 break;
1847 case LibFunc_size_returning_new_hot_cold:
1848 if (OptimizeExistingHotColdNew)
1849 return emitHotColdSizeReturningNew(Num: CI->getArgOperand(i: 0), B, TLI,
1850 NewFunc: LibFunc_size_returning_new_hot_cold,
1851 HotCold);
1852 break;
1853 case LibFunc_size_returning_new_aligned:
1854 if (HotCold != NotColdNewHintValue)
1855 return emitHotColdSizeReturningNewAligned(
1856 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1857 NewFunc: LibFunc_size_returning_new_aligned_hot_cold, HotCold);
1858 break;
1859 case LibFunc_size_returning_new_aligned_hot_cold:
1860 if (OptimizeExistingHotColdNew)
1861 return emitHotColdSizeReturningNewAligned(
1862 Num: CI->getArgOperand(i: 0), Align: CI->getArgOperand(i: 1), B, TLI,
1863 NewFunc: LibFunc_size_returning_new_aligned_hot_cold, HotCold);
1864 break;
1865 default:
1866 return nullptr;
1867 }
1868 return nullptr;
1869}
1870
1871//===----------------------------------------------------------------------===//
1872// Math Library Optimizations
1873//===----------------------------------------------------------------------===//
1874
1875// Replace a libcall \p CI with a call to intrinsic \p IID
1876static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
1877 Intrinsic::ID IID) {
1878 CallInst *NewCall = B.CreateUnaryIntrinsic(ID: IID, V: CI->getArgOperand(i: 0), FMFSource: CI);
1879 NewCall->takeName(V: CI);
1880 return copyFlags(Old: *CI, New: NewCall);
1881}
1882
1883/// Return a variant of Val with float type.
1884/// Currently this works in two cases: If Val is an FPExtension of a float
1885/// value to something bigger, simply return the operand.
1886/// If Val is a ConstantFP but can be converted to a float ConstantFP without
1887/// loss of precision do so.
1888static Value *valueHasFloatPrecision(Value *Val) {
1889 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
1890 Value *Op = Cast->getOperand(i_nocapture: 0);
1891 if (Op->getType()->isFloatTy())
1892 return Op;
1893 }
1894 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
1895 APFloat F = Const->getValueAPF();
1896 bool losesInfo;
1897 (void)F.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
1898 losesInfo: &losesInfo);
1899 if (!losesInfo)
1900 return ConstantFP::get(Context&: Const->getContext(), V: F);
1901 }
1902 return nullptr;
1903}
1904
1905/// Shrink double -> float functions.
1906static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
1907 bool isBinary, const TargetLibraryInfo *TLI,
1908 bool isPrecise = false) {
1909 Function *CalleeFn = CI->getCalledFunction();
1910 if (!CI->getType()->isDoubleTy() || !CalleeFn)
1911 return nullptr;
1912
1913 // If not all the uses of the function are converted to float, then bail out.
1914 // This matters if the precision of the result is more important than the
1915 // precision of the arguments.
1916 if (isPrecise)
1917 for (User *U : CI->users()) {
1918 FPTruncInst *Cast = dyn_cast<FPTruncInst>(Val: U);
1919 if (!Cast || !Cast->getType()->isFloatTy())
1920 return nullptr;
1921 }
1922
1923 // If this is something like 'g((double) float)', convert to 'gf(float)'.
1924 Value *V[2];
1925 V[0] = valueHasFloatPrecision(Val: CI->getArgOperand(i: 0));
1926 V[1] = isBinary ? valueHasFloatPrecision(Val: CI->getArgOperand(i: 1)) : nullptr;
1927 if (!V[0] || (isBinary && !V[1]))
1928 return nullptr;
1929
1930 // If call isn't an intrinsic, check that it isn't within a function with the
1931 // same name as the float version of this call, otherwise the result is an
1932 // infinite loop. For example, from MinGW-w64:
1933 //
1934 // float expf(float val) { return (float) exp((double) val); }
1935 StringRef CalleeName = CalleeFn->getName();
1936 bool IsIntrinsic = CalleeFn->isIntrinsic();
1937 if (!IsIntrinsic) {
1938 StringRef CallerName = CI->getFunction()->getName();
1939 if (CallerName.ends_with(Suffix: 'f') &&
1940 CallerName.size() == (CalleeName.size() + 1) &&
1941 CallerName.starts_with(Prefix: CalleeName))
1942 return nullptr;
1943 }
1944
1945 // Propagate the math semantics from the current function to the new function.
1946 IRBuilderBase::FastMathFlagGuard Guard(B);
1947 B.setFastMathFlags(CI->getFastMathFlags());
1948
1949 // g((double) float) -> (double) gf(float)
1950 Value *R;
1951 if (IsIntrinsic) {
1952 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
1953 R = isBinary ? B.CreateIntrinsic(ID: IID, Types: B.getFloatTy(), Args: V)
1954 : B.CreateIntrinsic(ID: IID, Types: B.getFloatTy(), Args: V[0]);
1955 } else {
1956 AttributeList CalleeAttrs = CalleeFn->getAttributes();
1957 R = isBinary ? emitBinaryFloatFnCall(Op1: V[0], Op2: V[1], TLI, Name: CalleeName, B,
1958 Attrs: CalleeAttrs)
1959 : emitUnaryFloatFnCall(Op: V[0], TLI, Name: CalleeName, B, Attrs: CalleeAttrs);
1960 }
1961 return B.CreateFPExt(V: R, DestTy: B.getDoubleTy());
1962}
1963
1964/// Shrink double -> float for unary functions.
1965static Value *optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B,
1966 const TargetLibraryInfo *TLI,
1967 bool isPrecise = false) {
1968 return optimizeDoubleFP(CI, B, isBinary: false, TLI, isPrecise);
1969}
1970
1971/// Shrink double -> float for binary functions.
1972static Value *optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B,
1973 const TargetLibraryInfo *TLI,
1974 bool isPrecise = false) {
1975 return optimizeDoubleFP(CI, B, isBinary: true, TLI, isPrecise);
1976}
1977
1978// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
1979Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
1980 Value *Real, *Imag;
1981
1982 if (CI->arg_size() == 1) {
1983
1984 if (!CI->isFast())
1985 return nullptr;
1986
1987 Value *Op = CI->getArgOperand(i: 0);
1988 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
1989
1990 Real = B.CreateExtractValue(Agg: Op, Idxs: 0, Name: "real");
1991 Imag = B.CreateExtractValue(Agg: Op, Idxs: 1, Name: "imag");
1992
1993 } else {
1994 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
1995
1996 Real = CI->getArgOperand(i: 0);
1997 Imag = CI->getArgOperand(i: 1);
1998
1999 // if real or imaginary part is zero, simplify to abs(cimag(z))
2000 // or abs(creal(z))
2001 Value *AbsOp = nullptr;
2002 if (ConstantFP *ConstReal = dyn_cast<ConstantFP>(Val: Real)) {
2003 if (ConstReal->isZero())
2004 AbsOp = Imag;
2005
2006 } else if (ConstantFP *ConstImag = dyn_cast<ConstantFP>(Val: Imag)) {
2007 if (ConstImag->isZero())
2008 AbsOp = Real;
2009 }
2010
2011 if (AbsOp)
2012 return copyFlags(
2013 Old: *CI, New: B.CreateUnaryIntrinsic(ID: Intrinsic::fabs, V: AbsOp, FMFSource: CI, Name: "cabs"));
2014
2015 if (!CI->isFast())
2016 return nullptr;
2017 }
2018
2019 // Propagate fast-math flags from the existing call to new instructions.
2020 Value *RealReal = B.CreateFMulFMF(L: Real, R: Real, FMFSource: CI);
2021 Value *ImagImag = B.CreateFMulFMF(L: Imag, R: Imag, FMFSource: CI);
2022 return copyFlags(
2023 Old: *CI, New: B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt,
2024 V: B.CreateFAddFMF(L: RealReal, R: ImagImag, FMFSource: CI), FMFSource: CI,
2025 Name: "cabs"));
2026}
2027
2028// Return a properly extended integer (DstWidth bits wide) if the operation is
2029// an itofp.
2030static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
2031 if (isa<SIToFPInst>(Val: I2F) || isa<UIToFPInst>(Val: I2F)) {
2032 Value *Op = cast<Instruction>(Val: I2F)->getOperand(i: 0);
2033 // Make sure that the exponent fits inside an "int" of size DstWidth,
2034 // thus avoiding any range issues that FP has not.
2035 unsigned BitWidth = Op->getType()->getScalarSizeInBits();
2036 if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(Val: I2F))) {
2037 Type *IntTy = Op->getType()->getWithNewBitWidth(NewBitWidth: DstWidth);
2038 return isa<SIToFPInst>(Val: I2F) ? B.CreateSExt(V: Op, DestTy: IntTy)
2039 : B.CreateZExt(V: Op, DestTy: IntTy);
2040 }
2041 }
2042
2043 return nullptr;
2044}
2045
2046/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
2047/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
2048/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
2049Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
2050 Module *M = Pow->getModule();
2051 Value *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2052 Type *Ty = Pow->getType();
2053 bool Ignored;
2054
2055 // Evaluate special cases related to a nested function as the base.
2056
2057 // pow(exp(x), y) -> exp(x * y)
2058 // pow(exp2(x), y) -> exp2(x * y)
2059 // If exp{,2}() is used only once, it is better to fold two transcendental
2060 // math functions into one. If used again, exp{,2}() would still have to be
2061 // called with the original argument, then keep both original transcendental
2062 // functions. However, this transformation is only safe with fully relaxed
2063 // math semantics, since, besides rounding differences, it changes overflow
2064 // and underflow behavior quite dramatically. For example:
2065 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
2066 // Whereas:
2067 // exp(1000 * 0.001) = exp(1)
2068 // TODO: Loosen the requirement for fully relaxed math semantics.
2069 // TODO: Handle exp10() when more targets have it available.
2070 CallInst *BaseFn = dyn_cast<CallInst>(Val: Base);
2071 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
2072 LibFunc LibFn;
2073
2074 Function *CalleeFn = BaseFn->getCalledFunction();
2075 if (CalleeFn && TLI->getLibFunc(funcName: CalleeFn->getName(), F&: LibFn) &&
2076 isLibFuncEmittable(M, TLI, TheLibFunc: LibFn)) {
2077 StringRef ExpName;
2078 Intrinsic::ID ID;
2079 Value *ExpFn;
2080 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2081
2082 switch (LibFn) {
2083 default:
2084 return nullptr;
2085 case LibFunc_expf:
2086 case LibFunc_exp:
2087 case LibFunc_expl:
2088 ExpName = TLI->getName(F: LibFunc_exp);
2089 ID = Intrinsic::exp;
2090 LibFnFloat = LibFunc_expf;
2091 LibFnDouble = LibFunc_exp;
2092 LibFnLongDouble = LibFunc_expl;
2093 break;
2094 case LibFunc_exp2f:
2095 case LibFunc_exp2:
2096 case LibFunc_exp2l:
2097 ExpName = TLI->getName(F: LibFunc_exp2);
2098 ID = Intrinsic::exp2;
2099 LibFnFloat = LibFunc_exp2f;
2100 LibFnDouble = LibFunc_exp2;
2101 LibFnLongDouble = LibFunc_exp2l;
2102 break;
2103 }
2104
2105 // Create new exp{,2}() with the product as its argument.
2106 Value *FMul = B.CreateFMul(L: BaseFn->getArgOperand(i: 0), R: Expo, Name: "mul");
2107 ExpFn = BaseFn->doesNotAccessMemory()
2108 ? B.CreateUnaryIntrinsic(ID, V: FMul, FMFSource: nullptr, Name: ExpName)
2109 : emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFnDouble, FloatFn: LibFnFloat,
2110 LongDoubleFn: LibFnLongDouble, B,
2111 Attrs: BaseFn->getAttributes());
2112
2113 // Since the new exp{,2}() is different from the original one, dead code
2114 // elimination cannot be trusted to remove it, since it may have side
2115 // effects (e.g., errno). When the only consumer for the original
2116 // exp{,2}() is pow(), then it has to be explicitly erased.
2117 substituteInParent(I: BaseFn, With: ExpFn);
2118 return ExpFn;
2119 }
2120 }
2121
2122 // Evaluate special cases related to a constant base.
2123
2124 const APFloat *BaseF;
2125 if (!match(V: Base, P: m_APFloat(Res&: BaseF)))
2126 return nullptr;
2127
2128 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2129
2130 const bool UseIntrinsic = Pow->doesNotAccessMemory();
2131
2132 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2133 if ((UseIntrinsic || !Ty->isVectorTy()) && BaseF->isExactlyValue(V: 2.0) &&
2134 (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo)) &&
2135 (UseIntrinsic ||
2136 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2137
2138 // TODO: Shouldn't really need to depend on getIntToFPVal for intrinsic. Can
2139 // just directly use the original integer type.
2140 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize())) {
2141 Constant *One = ConstantFP::get(Ty, V: 1.0);
2142
2143 if (UseIntrinsic) {
2144 return copyFlags(Old: *Pow, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2145 Types: {Ty, ExpoI->getType()},
2146 Args: {One, ExpoI}, FMFSource: Pow, Name: "exp2"));
2147 }
2148
2149 return copyFlags(Old: *Pow, New: emitBinaryFloatFnCall(
2150 Op1: One, Op2: ExpoI, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2151 LongDoubleFn: LibFunc_ldexpl, B, Attrs: NoAttrs));
2152 }
2153 }
2154
2155 // pow(2.0 ** n, x) -> exp2(n * x)
2156 if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f, LongDoubleFn: LibFunc_exp2l)) {
2157 APFloat BaseR = APFloat(1.0);
2158 BaseR.convert(ToSemantics: BaseF->getSemantics(), RM: APFloat::rmTowardZero, losesInfo: &Ignored);
2159 BaseR = BaseR / *BaseF;
2160 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2161 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2162 APSInt NI(64, false);
2163 if ((IsInteger || IsReciprocal) &&
2164 NF->convertToInteger(Result&: NI, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2165 APFloat::opOK &&
2166 NI > 1 && NI.isPowerOf2()) {
2167 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2168 Value *FMul = B.CreateFMul(L: Expo, R: ConstantFP::get(Ty, V: N), Name: "mul");
2169 if (Pow->doesNotAccessMemory())
2170 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, V: FMul,
2171 FMFSource: nullptr, Name: "exp2"));
2172 else
2173 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2174 FloatFn: LibFunc_exp2f,
2175 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2176 }
2177 }
2178
2179 // pow(10.0, x) -> exp10(x)
2180 if (BaseF->isExactlyValue(V: 10.0) &&
2181 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp10, FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l)) {
2182
2183 if (Pow->doesNotAccessMemory()) {
2184 CallInst *NewExp10 =
2185 B.CreateIntrinsic(ID: Intrinsic::exp10, Types: {Ty}, Args: {Expo}, FMFSource: Pow, Name: "exp10");
2186 return copyFlags(Old: *Pow, New: NewExp10);
2187 }
2188
2189 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: Expo, TLI, DoubleFn: LibFunc_exp10,
2190 FloatFn: LibFunc_exp10f, LongDoubleFn: LibFunc_exp10l,
2191 B, Attrs: NoAttrs));
2192 }
2193
2194 // pow(x, y) -> exp2(log2(x) * y)
2195 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2196 !BaseF->isNegative()) {
2197 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2198 // Luckily optimizePow has already handled the x == 1 case.
2199 assert(!match(Base, m_FPOne()) &&
2200 "pow(1.0, y) should have been simplified earlier!");
2201
2202 Value *Log = nullptr;
2203 if (Ty->isFloatTy())
2204 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToFloat()));
2205 else if (Ty->isDoubleTy())
2206 Log = ConstantFP::get(Ty, V: std::log2(x: BaseF->convertToDouble()));
2207
2208 if (Log) {
2209 Value *FMul = B.CreateFMul(L: Log, R: Expo, Name: "mul");
2210 if (Pow->doesNotAccessMemory())
2211 return copyFlags(Old: *Pow, New: B.CreateUnaryIntrinsic(ID: Intrinsic::exp2, V: FMul,
2212 FMFSource: nullptr, Name: "exp2"));
2213 else if (hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_exp2, FloatFn: LibFunc_exp2f,
2214 LongDoubleFn: LibFunc_exp2l))
2215 return copyFlags(Old: *Pow, New: emitUnaryFloatFnCall(Op: FMul, TLI, DoubleFn: LibFunc_exp2,
2216 FloatFn: LibFunc_exp2f,
2217 LongDoubleFn: LibFunc_exp2l, B, Attrs: NoAttrs));
2218 }
2219 }
2220
2221 return nullptr;
2222}
2223
2224static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2225 Module *M, IRBuilderBase &B,
2226 const TargetLibraryInfo *TLI) {
2227 // If errno is never set, then use the intrinsic for sqrt().
2228 if (NoErrno)
2229 return B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, V, FMFSource: nullptr, Name: "sqrt");
2230
2231 // Otherwise, use the libcall for sqrt().
2232 if (hasFloatFn(M, TLI, Ty: V->getType(), DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2233 LongDoubleFn: LibFunc_sqrtl))
2234 // TODO: We also should check that the target can in fact lower the sqrt()
2235 // libcall. We currently have no way to ask this question, so we ask if
2236 // the target has a sqrt() libcall, which is not exactly the same.
2237 return emitUnaryFloatFnCall(Op: V, TLI, DoubleFn: LibFunc_sqrt, FloatFn: LibFunc_sqrtf,
2238 LongDoubleFn: LibFunc_sqrtl, B, Attrs);
2239
2240 return nullptr;
2241}
2242
2243/// Use square root in place of pow(x, +/-0.5).
2244Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2245 Value *Sqrt, *Base = Pow->getArgOperand(i: 0), *Expo = Pow->getArgOperand(i: 1);
2246 Module *Mod = Pow->getModule();
2247 Type *Ty = Pow->getType();
2248
2249 const APFloat *ExpoF;
2250 if (!match(V: Expo, P: m_APFloat(Res&: ExpoF)) ||
2251 (!ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)))
2252 return nullptr;
2253
2254 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2255 // so that requires fast-math-flags (afn or reassoc).
2256 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2257 return nullptr;
2258
2259 // If we have a pow() library call (accesses memory) and we can't guarantee
2260 // that the base is not an infinity, give up:
2261 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2262 // errno), but sqrt(-Inf) is required by various standards to set errno.
2263 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2264 !isKnownNeverInfinity(
2265 V: Base, SQ: SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
2266 return nullptr;
2267
2268 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M: Mod, B,
2269 TLI);
2270 if (!Sqrt)
2271 return nullptr;
2272
2273 // Handle signed zero base by expanding to fabs(sqrt(x)).
2274 if (!Pow->hasNoSignedZeros())
2275 Sqrt = B.CreateUnaryIntrinsic(ID: Intrinsic::fabs, V: Sqrt, FMFSource: nullptr, Name: "abs");
2276
2277 Sqrt = copyFlags(Old: *Pow, New: Sqrt);
2278
2279 // Handle non finite base by expanding to
2280 // (x == -infinity ? +infinity : sqrt(x)).
2281 if (!Pow->hasNoInfs()) {
2282 Value *PosInf = ConstantFP::getInfinity(Ty),
2283 *NegInf = ConstantFP::getInfinity(Ty, Negative: true);
2284 Value *FCmp = B.CreateFCmpOEQ(LHS: Base, RHS: NegInf, Name: "isinf");
2285 Sqrt = B.CreateSelect(C: FCmp, True: PosInf, False: Sqrt);
2286 }
2287
2288 // If the exponent is negative, then get the reciprocal.
2289 if (ExpoF->isNegative())
2290 Sqrt = B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Sqrt, Name: "reciprocal");
2291
2292 return Sqrt;
2293}
2294
2295static Value *createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M,
2296 IRBuilderBase &B) {
2297 Value *Args[] = {Base, Expo};
2298 Type *Types[] = {Base->getType(), Expo->getType()};
2299 return B.CreateIntrinsic(ID: Intrinsic::powi, Types, Args);
2300}
2301
2302Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2303 Value *Base = Pow->getArgOperand(i: 0);
2304 Value *Expo = Pow->getArgOperand(i: 1);
2305 Function *Callee = Pow->getCalledFunction();
2306 StringRef Name = Callee->getName();
2307 Type *Ty = Pow->getType();
2308 Module *M = Pow->getModule();
2309 bool AllowApprox = Pow->hasApproxFunc();
2310 bool Ignored;
2311
2312 // Propagate the math semantics from the call to any created instructions.
2313 IRBuilderBase::FastMathFlagGuard Guard(B);
2314 B.setFastMathFlags(Pow->getFastMathFlags());
2315 // Evaluate special cases related to the base.
2316
2317 // pow(1.0, x) -> 1.0
2318 if (match(V: Base, P: m_FPOne()))
2319 return Base;
2320
2321 if (Value *Exp = replacePowWithExp(Pow, B))
2322 return Exp;
2323
2324 // Evaluate special cases related to the exponent.
2325
2326 // pow(x, -1.0) -> 1.0 / x
2327 if (match(V: Expo, P: m_SpecificFP(V: -1.0)))
2328 return B.CreateFDiv(L: ConstantFP::get(Ty, V: 1.0), R: Base, Name: "reciprocal");
2329
2330 // pow(x, +/-0.0) -> 1.0
2331 if (match(V: Expo, P: m_AnyZeroFP()))
2332 return ConstantFP::get(Ty, V: 1.0);
2333
2334 // pow(x, 1.0) -> x
2335 if (match(V: Expo, P: m_FPOne()))
2336 return Base;
2337
2338 // pow(x, 2.0) -> x * x
2339 if (match(V: Expo, P: m_SpecificFP(V: 2.0)))
2340 return B.CreateFMul(L: Base, R: Base, Name: "square");
2341
2342 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2343 return Sqrt;
2344
2345 // If we can approximate pow:
2346 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2347 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2348 const APFloat *ExpoF;
2349 if (AllowApprox && match(V: Expo, P: m_APFloat(Res&: ExpoF)) &&
2350 !ExpoF->isExactlyValue(V: 0.5) && !ExpoF->isExactlyValue(V: -0.5)) {
2351 APFloat ExpoA(abs(X: *ExpoF));
2352 APFloat ExpoI(*ExpoF);
2353 Value *Sqrt = nullptr;
2354 if (!ExpoA.isInteger()) {
2355 APFloat Expo2 = ExpoA;
2356 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2357 // is no floating point exception and the result is an integer, then
2358 // ExpoA == integer + 0.5
2359 if (Expo2.add(RHS: ExpoA, RM: APFloat::rmNearestTiesToEven) != APFloat::opOK)
2360 return nullptr;
2361
2362 if (!Expo2.isInteger())
2363 return nullptr;
2364
2365 if (ExpoI.roundToIntegral(RM: APFloat::rmTowardNegative) !=
2366 APFloat::opInexact)
2367 return nullptr;
2368 if (!ExpoI.isInteger())
2369 return nullptr;
2370 ExpoF = &ExpoI;
2371
2372 Sqrt = getSqrtCall(V: Base, Attrs: AttributeList(), NoErrno: Pow->doesNotAccessMemory(), M,
2373 B, TLI);
2374 if (!Sqrt)
2375 return nullptr;
2376 }
2377
2378 // 0.5 fraction is now optionally handled.
2379 // Do pow -> powi for remaining integer exponent
2380 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2381 if (ExpoF->isInteger() &&
2382 ExpoF->convertToInteger(Result&: IntExpo, RM: APFloat::rmTowardZero, IsExact: &Ignored) ==
2383 APFloat::opOK) {
2384 Value *PowI = copyFlags(
2385 Old: *Pow,
2386 New: createPowWithIntegerExponent(
2387 Base, Expo: ConstantInt::get(Ty: B.getIntNTy(N: TLI->getIntSize()), V: IntExpo),
2388 M, B));
2389
2390 if (PowI && Sqrt)
2391 return B.CreateFMul(L: PowI, R: Sqrt);
2392
2393 return PowI;
2394 }
2395 }
2396
2397 // powf(x, itofp(y)) -> powi(x, y)
2398 if (AllowApprox && (isa<SIToFPInst>(Val: Expo) || isa<UIToFPInst>(Val: Expo))) {
2399 if (Value *ExpoI = getIntToFPVal(I2F: Expo, B, DstWidth: TLI->getIntSize()))
2400 return copyFlags(Old: *Pow, New: createPowWithIntegerExponent(Base, Expo: ExpoI, M, B));
2401 }
2402
2403 // Shrink pow() to powf() if the arguments are single precision,
2404 // unless the result is expected to be double precision.
2405 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_pow) &&
2406 hasFloatVersion(M, FuncName: Name)) {
2407 if (Value *Shrunk = optimizeBinaryDoubleFP(CI: Pow, B, TLI, isPrecise: true))
2408 return Shrunk;
2409 }
2410
2411 return nullptr;
2412}
2413
2414Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2415 Module *M = CI->getModule();
2416 Function *Callee = CI->getCalledFunction();
2417 StringRef Name = Callee->getName();
2418 Value *Ret = nullptr;
2419 if (UnsafeFPShrink && Name == TLI->getName(F: LibFunc_exp2) &&
2420 hasFloatVersion(M, FuncName: Name))
2421 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2422
2423 // If we have an llvm.exp2 intrinsic, emit the llvm.ldexp intrinsic. If we
2424 // have the libcall, emit the libcall.
2425 //
2426 // TODO: In principle we should be able to just always use the intrinsic for
2427 // any doesNotAccessMemory callsite.
2428
2429 const bool UseIntrinsic = Callee->isIntrinsic();
2430 // Bail out for vectors because the code below only expects scalars.
2431 Type *Ty = CI->getType();
2432 if (!UseIntrinsic && Ty->isVectorTy())
2433 return Ret;
2434
2435 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2436 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2437 Value *Op = CI->getArgOperand(i: 0);
2438 if ((isa<SIToFPInst>(Val: Op) || isa<UIToFPInst>(Val: Op)) &&
2439 (UseIntrinsic ||
2440 hasFloatFn(M, TLI, Ty, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf, LongDoubleFn: LibFunc_ldexpl))) {
2441 if (Value *Exp = getIntToFPVal(I2F: Op, B, DstWidth: TLI->getIntSize())) {
2442 Constant *One = ConstantFP::get(Ty, V: 1.0);
2443
2444 if (UseIntrinsic) {
2445 return copyFlags(Old: *CI, New: B.CreateIntrinsic(ID: Intrinsic::ldexp,
2446 Types: {Ty, Exp->getType()},
2447 Args: {One, Exp}, FMFSource: CI));
2448 }
2449
2450 IRBuilderBase::FastMathFlagGuard Guard(B);
2451 B.setFastMathFlags(CI->getFastMathFlags());
2452 return copyFlags(Old: *CI, New: emitBinaryFloatFnCall(
2453 Op1: One, Op2: Exp, TLI, DoubleFn: LibFunc_ldexp, FloatFn: LibFunc_ldexpf,
2454 LongDoubleFn: LibFunc_ldexpl, B, Attrs: AttributeList()));
2455 }
2456 }
2457
2458 return Ret;
2459}
2460
2461Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B) {
2462 Module *M = CI->getModule();
2463
2464 // If we can shrink the call to a float function rather than a double
2465 // function, do that first.
2466 Function *Callee = CI->getCalledFunction();
2467 StringRef Name = Callee->getName();
2468 if ((Name == "fmin" || Name == "fmax") && hasFloatVersion(M, FuncName: Name))
2469 if (Value *Ret = optimizeBinaryDoubleFP(CI, B, TLI))
2470 return Ret;
2471
2472 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2473 // the intrinsics for improved optimization (for example, vectorization).
2474 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2475 // From the C standard draft WG14/N1256:
2476 // "Ideally, fmax would be sensitive to the sign of zero, for example
2477 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2478 // might be impractical."
2479 FastMathFlags FMF = CI->getFastMathFlags();
2480 FMF.setNoSignedZeros();
2481
2482 Intrinsic::ID IID = Callee->getName().starts_with(Prefix: "fmin") ? Intrinsic::minnum
2483 : Intrinsic::maxnum;
2484 return copyFlags(Old: *CI, New: B.CreateBinaryIntrinsic(ID: IID, LHS: CI->getArgOperand(i: 0),
2485 RHS: CI->getArgOperand(i: 1), FMFSource: FMF));
2486}
2487
2488Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2489 Function *LogFn = Log->getCalledFunction();
2490 StringRef LogNm = LogFn->getName();
2491 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2492 Module *Mod = Log->getModule();
2493 Type *Ty = Log->getType();
2494
2495 if (UnsafeFPShrink && hasFloatVersion(M: Mod, FuncName: LogNm))
2496 if (Value *Ret = optimizeUnaryDoubleFP(CI: Log, B, TLI, isPrecise: true))
2497 return Ret;
2498
2499 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2500
2501 // This is only applicable to log(), log2(), log10().
2502 if (TLI->getLibFunc(funcName: LogNm, F&: LogLb)) {
2503 switch (LogLb) {
2504 case LibFunc_logf:
2505 LogID = Intrinsic::log;
2506 ExpLb = LibFunc_expf;
2507 Exp2Lb = LibFunc_exp2f;
2508 Exp10Lb = LibFunc_exp10f;
2509 PowLb = LibFunc_powf;
2510 break;
2511 case LibFunc_log:
2512 LogID = Intrinsic::log;
2513 ExpLb = LibFunc_exp;
2514 Exp2Lb = LibFunc_exp2;
2515 Exp10Lb = LibFunc_exp10;
2516 PowLb = LibFunc_pow;
2517 break;
2518 case LibFunc_logl:
2519 LogID = Intrinsic::log;
2520 ExpLb = LibFunc_expl;
2521 Exp2Lb = LibFunc_exp2l;
2522 Exp10Lb = LibFunc_exp10l;
2523 PowLb = LibFunc_powl;
2524 break;
2525 case LibFunc_log2f:
2526 LogID = Intrinsic::log2;
2527 ExpLb = LibFunc_expf;
2528 Exp2Lb = LibFunc_exp2f;
2529 Exp10Lb = LibFunc_exp10f;
2530 PowLb = LibFunc_powf;
2531 break;
2532 case LibFunc_log2:
2533 LogID = Intrinsic::log2;
2534 ExpLb = LibFunc_exp;
2535 Exp2Lb = LibFunc_exp2;
2536 Exp10Lb = LibFunc_exp10;
2537 PowLb = LibFunc_pow;
2538 break;
2539 case LibFunc_log2l:
2540 LogID = Intrinsic::log2;
2541 ExpLb = LibFunc_expl;
2542 Exp2Lb = LibFunc_exp2l;
2543 Exp10Lb = LibFunc_exp10l;
2544 PowLb = LibFunc_powl;
2545 break;
2546 case LibFunc_log10f:
2547 LogID = Intrinsic::log10;
2548 ExpLb = LibFunc_expf;
2549 Exp2Lb = LibFunc_exp2f;
2550 Exp10Lb = LibFunc_exp10f;
2551 PowLb = LibFunc_powf;
2552 break;
2553 case LibFunc_log10:
2554 LogID = Intrinsic::log10;
2555 ExpLb = LibFunc_exp;
2556 Exp2Lb = LibFunc_exp2;
2557 Exp10Lb = LibFunc_exp10;
2558 PowLb = LibFunc_pow;
2559 break;
2560 case LibFunc_log10l:
2561 LogID = Intrinsic::log10;
2562 ExpLb = LibFunc_expl;
2563 Exp2Lb = LibFunc_exp2l;
2564 Exp10Lb = LibFunc_exp10l;
2565 PowLb = LibFunc_powl;
2566 break;
2567 default:
2568 return nullptr;
2569 }
2570
2571 // Convert libcall to intrinsic if the value is known > 0.
2572 bool IsKnownNoErrno = Log->hasNoNaNs() && Log->hasNoInfs();
2573 if (!IsKnownNoErrno) {
2574 SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
2575 KnownFPClass Known = computeKnownFPClass(
2576 V: Log->getOperand(i_nocapture: 0),
2577 InterestedClasses: KnownFPClass::OrderedLessThanZeroMask | fcSubnormal, SQ);
2578 Function *F = Log->getParent()->getParent();
2579 const fltSemantics &FltSem = Ty->getScalarType()->getFltSemantics();
2580 IsKnownNoErrno =
2581 Known.cannotBeOrderedLessThanZero() &&
2582 Known.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
2583 }
2584 if (IsKnownNoErrno) {
2585 auto *NewLog = B.CreateUnaryIntrinsic(ID: LogID, V: Log->getArgOperand(i: 0), FMFSource: Log);
2586 NewLog->copyMetadata(SrcInst: *Log);
2587 return copyFlags(Old: *Log, New: NewLog);
2588 }
2589 } else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2590 LogID == Intrinsic::log10) {
2591 if (Ty->getScalarType()->isFloatTy()) {
2592 ExpLb = LibFunc_expf;
2593 Exp2Lb = LibFunc_exp2f;
2594 Exp10Lb = LibFunc_exp10f;
2595 PowLb = LibFunc_powf;
2596 } else if (Ty->getScalarType()->isDoubleTy()) {
2597 ExpLb = LibFunc_exp;
2598 Exp2Lb = LibFunc_exp2;
2599 Exp10Lb = LibFunc_exp10;
2600 PowLb = LibFunc_pow;
2601 } else
2602 return nullptr;
2603 } else
2604 return nullptr;
2605
2606 // The earlier call must also be 'fast' in order to do these transforms.
2607 CallInst *Arg = dyn_cast<CallInst>(Val: Log->getArgOperand(i: 0));
2608 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2609 return nullptr;
2610
2611 IRBuilderBase::FastMathFlagGuard Guard(B);
2612 B.setFastMathFlags(FastMathFlags::getFast());
2613
2614 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2615 LibFunc ArgLb = NotLibFunc;
2616 TLI->getLibFunc(CB: *Arg, F&: ArgLb);
2617
2618 // log(pow(x,y)) -> y*log(x)
2619 AttributeList NoAttrs;
2620 if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2621 Value *LogX =
2622 Log->doesNotAccessMemory()
2623 ? B.CreateUnaryIntrinsic(ID: LogID, V: Arg->getOperand(i_nocapture: 0), FMFSource: nullptr, Name: "log")
2624 : emitUnaryFloatFnCall(Op: Arg->getOperand(i_nocapture: 0), TLI, Name: LogNm, B, Attrs: NoAttrs);
2625 Value *Y = Arg->getArgOperand(i: 1);
2626 // Cast exponent to FP if integer.
2627 if (ArgID == Intrinsic::powi)
2628 Y = B.CreateSIToFP(V: Y, DestTy: Ty, Name: "cast");
2629 Value *MulY = B.CreateFMul(L: Y, R: LogX, Name: "mul");
2630 // Since pow() may have side effects, e.g. errno,
2631 // dead code elimination may not be trusted to remove it.
2632 substituteInParent(I: Arg, With: MulY);
2633 return MulY;
2634 }
2635
2636 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2637 // TODO: There is no exp10() intrinsic yet.
2638 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2639 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
2640 Constant *Eul;
2641 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2642 // FIXME: Add more precise value of e for long double.
2643 Eul = ConstantFP::get(Ty: Log->getType(), V: numbers::e);
2644 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2645 Eul = ConstantFP::get(Ty: Log->getType(), V: 2.0);
2646 else
2647 Eul = ConstantFP::get(Ty: Log->getType(), V: 10.0);
2648 Value *LogE = Log->doesNotAccessMemory()
2649 ? B.CreateUnaryIntrinsic(ID: LogID, V: Eul, FMFSource: nullptr, Name: "log")
2650 : emitUnaryFloatFnCall(Op: Eul, TLI, Name: LogNm, B, Attrs: NoAttrs);
2651 Value *MulY = B.CreateFMul(L: Arg->getArgOperand(i: 0), R: LogE, Name: "mul");
2652 // Since exp() may have side effects, e.g. errno,
2653 // dead code elimination may not be trusted to remove it.
2654 substituteInParent(I: Arg, With: MulY);
2655 return MulY;
2656 }
2657
2658 return nullptr;
2659}
2660
2661// sqrt(exp(X)) -> exp(X * 0.5)
2662Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2663 if (!CI->hasAllowReassoc())
2664 return nullptr;
2665
2666 Function *SqrtFn = CI->getCalledFunction();
2667 CallInst *Arg = dyn_cast<CallInst>(Val: CI->getArgOperand(i: 0));
2668 if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2669 return nullptr;
2670 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2671 LibFunc ArgLb = NotLibFunc;
2672 TLI->getLibFunc(CB: *Arg, F&: ArgLb);
2673
2674 LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2675
2676 if (TLI->getLibFunc(funcName: SqrtFn->getName(), F&: SqrtLb))
2677 switch (SqrtLb) {
2678 case LibFunc_sqrtf:
2679 ExpLb = LibFunc_expf;
2680 Exp2Lb = LibFunc_exp2f;
2681 Exp10Lb = LibFunc_exp10f;
2682 break;
2683 case LibFunc_sqrt:
2684 ExpLb = LibFunc_exp;
2685 Exp2Lb = LibFunc_exp2;
2686 Exp10Lb = LibFunc_exp10;
2687 break;
2688 case LibFunc_sqrtl:
2689 ExpLb = LibFunc_expl;
2690 Exp2Lb = LibFunc_exp2l;
2691 Exp10Lb = LibFunc_exp10l;
2692 break;
2693 default:
2694 return nullptr;
2695 }
2696 else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2697 if (CI->getType()->getScalarType()->isFloatTy()) {
2698 ExpLb = LibFunc_expf;
2699 Exp2Lb = LibFunc_exp2f;
2700 Exp10Lb = LibFunc_exp10f;
2701 } else if (CI->getType()->getScalarType()->isDoubleTy()) {
2702 ExpLb = LibFunc_exp;
2703 Exp2Lb = LibFunc_exp2;
2704 Exp10Lb = LibFunc_exp10;
2705 } else
2706 return nullptr;
2707 } else
2708 return nullptr;
2709
2710 if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2711 ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2712 return nullptr;
2713
2714 IRBuilderBase::InsertPointGuard Guard(B);
2715 B.SetInsertPoint(Arg);
2716 auto *ExpOperand = Arg->getOperand(i_nocapture: 0);
2717 auto *FMul =
2718 B.CreateFMulFMF(L: ExpOperand, R: ConstantFP::get(Ty: ExpOperand->getType(), V: 0.5),
2719 FMFSource: CI, Name: "merged.sqrt");
2720
2721 Arg->setOperand(i_nocapture: 0, Val_nocapture: FMul);
2722 return Arg;
2723}
2724
2725Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2726 Module *M = CI->getModule();
2727 Function *Callee = CI->getCalledFunction();
2728 Value *Ret = nullptr;
2729 // TODO: Once we have a way (other than checking for the existince of the
2730 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2731 // condition below.
2732 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_sqrtf) &&
2733 (Callee->getName() == "sqrt" ||
2734 Callee->getIntrinsicID() == Intrinsic::sqrt))
2735 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2736
2737 if (Value *Opt = mergeSqrtToExp(CI, B))
2738 return Opt;
2739
2740 if (!CI->isFast())
2741 return Ret;
2742
2743 Instruction *I = dyn_cast<Instruction>(Val: CI->getArgOperand(i: 0));
2744 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2745 return Ret;
2746
2747 // We're looking for a repeated factor in a multiplication tree,
2748 // so we can do this fold: sqrt(x * x) -> fabs(x);
2749 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2750 Value *Op0 = I->getOperand(i: 0);
2751 Value *Op1 = I->getOperand(i: 1);
2752 Value *RepeatOp = nullptr;
2753 Value *OtherOp = nullptr;
2754 if (Op0 == Op1) {
2755 // Simple match: the operands of the multiply are identical.
2756 RepeatOp = Op0;
2757 } else {
2758 // Look for a more complicated pattern: one of the operands is itself
2759 // a multiply, so search for a common factor in that multiply.
2760 // Note: We don't bother looking any deeper than this first level or for
2761 // variations of this pattern because instcombine's visitFMUL and/or the
2762 // reassociation pass should give us this form.
2763 Value *MulOp;
2764 if (match(V: Op0, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2765 cast<Instruction>(Val: Op0)->isFast()) {
2766 // Pattern: sqrt((x * x) * z)
2767 RepeatOp = MulOp;
2768 OtherOp = Op1;
2769 } else if (match(V: Op1, P: m_FMul(L: m_Value(V&: MulOp), R: m_Deferred(V: MulOp))) &&
2770 cast<Instruction>(Val: Op1)->isFast()) {
2771 // Pattern: sqrt(z * (x * x))
2772 RepeatOp = MulOp;
2773 OtherOp = Op0;
2774 }
2775 }
2776 if (!RepeatOp)
2777 return Ret;
2778
2779 // Fast math flags for any created instructions should match the sqrt
2780 // and multiply.
2781
2782 // If we found a repeated factor, hoist it out of the square root and
2783 // replace it with the fabs of that factor.
2784 Value *FabsCall =
2785 B.CreateUnaryIntrinsic(ID: Intrinsic::fabs, V: RepeatOp, FMFSource: I, Name: "fabs");
2786 if (OtherOp) {
2787 // If we found a non-repeated factor, we still need to get its square
2788 // root. We then multiply that by the value that was simplified out
2789 // of the square root calculation.
2790 Value *SqrtCall =
2791 B.CreateUnaryIntrinsic(ID: Intrinsic::sqrt, V: OtherOp, FMFSource: I, Name: "sqrt");
2792 return copyFlags(Old: *CI, New: B.CreateFMulFMF(L: FabsCall, R: SqrtCall, FMFSource: I));
2793 }
2794 return copyFlags(Old: *CI, New: FabsCall);
2795}
2796
2797Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
2798
2799 // fmod(x,y) can set errno if y == 0 or x == +/-inf, and returns Nan in those
2800 // case. If we know those do not happen, then we can convert the fmod into
2801 // frem.
2802 bool IsNoNan = CI->hasNoNaNs();
2803 if (!IsNoNan) {
2804 SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
2805 KnownFPClass Known0 = computeKnownFPClass(V: CI->getOperand(i_nocapture: 0), InterestedClasses: fcInf, SQ);
2806 if (Known0.isKnownNeverInfinity()) {
2807 KnownFPClass Known1 =
2808 computeKnownFPClass(V: CI->getOperand(i_nocapture: 1), InterestedClasses: fcZero | fcSubnormal, SQ);
2809 Function *F = CI->getParent()->getParent();
2810 const fltSemantics &FltSem =
2811 CI->getType()->getScalarType()->getFltSemantics();
2812 IsNoNan = Known1.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem));
2813 }
2814 }
2815
2816 if (IsNoNan) {
2817 Value *FRem = B.CreateFRemFMF(L: CI->getOperand(i_nocapture: 0), R: CI->getOperand(i_nocapture: 1), FMFSource: CI);
2818 if (auto *FRemI = dyn_cast<Instruction>(Val: FRem))
2819 FRemI->setHasNoNaNs(true);
2820 return FRem;
2821 }
2822 return nullptr;
2823}
2824
2825Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
2826 IRBuilderBase &B) {
2827 Module *M = CI->getModule();
2828 Function *Callee = CI->getCalledFunction();
2829 Value *Ret = nullptr;
2830 StringRef Name = Callee->getName();
2831 if (UnsafeFPShrink &&
2832 (Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
2833 Name == "asinh") &&
2834 hasFloatVersion(M, FuncName: Name))
2835 Ret = optimizeUnaryDoubleFP(CI, B, TLI, isPrecise: true);
2836
2837 Value *Op1 = CI->getArgOperand(i: 0);
2838 auto *OpC = dyn_cast<CallInst>(Val: Op1);
2839 if (!OpC)
2840 return Ret;
2841
2842 // Both calls must be 'fast' in order to remove them.
2843 if (!CI->isFast() || !OpC->isFast())
2844 return Ret;
2845
2846 // tan(atan(x)) -> x
2847 // atanh(tanh(x)) -> x
2848 // sinh(asinh(x)) -> x
2849 // asinh(sinh(x)) -> x
2850 // cosh(acosh(x)) -> x
2851 LibFunc Func;
2852 Function *F = OpC->getCalledFunction();
2853 if (F && TLI->getLibFunc(funcName: F->getName(), F&: Func) &&
2854 isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
2855 LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
2856 .Case(S: "tan", Value: LibFunc_atan)
2857 .Case(S: "atanh", Value: LibFunc_tanh)
2858 .Case(S: "sinh", Value: LibFunc_asinh)
2859 .Case(S: "cosh", Value: LibFunc_acosh)
2860 .Case(S: "tanf", Value: LibFunc_atanf)
2861 .Case(S: "atanhf", Value: LibFunc_tanhf)
2862 .Case(S: "sinhf", Value: LibFunc_asinhf)
2863 .Case(S: "coshf", Value: LibFunc_acoshf)
2864 .Case(S: "tanl", Value: LibFunc_atanl)
2865 .Case(S: "atanhl", Value: LibFunc_tanhl)
2866 .Case(S: "sinhl", Value: LibFunc_asinhl)
2867 .Case(S: "coshl", Value: LibFunc_acoshl)
2868 .Case(S: "asinh", Value: LibFunc_sinh)
2869 .Case(S: "asinhf", Value: LibFunc_sinhf)
2870 .Case(S: "asinhl", Value: LibFunc_sinhl)
2871 .Default(Value: NumLibFuncs); // Used as error value
2872 if (Func == inverseFunc)
2873 Ret = OpC->getArgOperand(i: 0);
2874 }
2875 return Ret;
2876}
2877
2878static bool isTrigLibCall(CallInst *CI) {
2879 // We can only hope to do anything useful if we can ignore things like errno
2880 // and floating-point exceptions.
2881 // We already checked the prototype.
2882 return CI->doesNotThrow() && CI->doesNotAccessMemory();
2883}
2884
2885static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
2886 bool UseFloat, Value *&Sin, Value *&Cos,
2887 Value *&SinCos, const TargetLibraryInfo *TLI) {
2888 Module *M = OrigCallee->getParent();
2889 Type *ArgTy = Arg->getType();
2890 Type *ResTy;
2891 StringRef Name;
2892
2893 Triple T(OrigCallee->getParent()->getTargetTriple());
2894 if (UseFloat) {
2895 Name = "__sincospif_stret";
2896
2897 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
2898 // x86_64 can't use {float, float} since that would be returned in both
2899 // xmm0 and xmm1, which isn't what a real struct would do.
2900 ResTy = T.getArch() == Triple::x86_64
2901 ? static_cast<Type *>(FixedVectorType::get(ElementType: ArgTy, NumElts: 2))
2902 : static_cast<Type *>(StructType::get(elt1: ArgTy, elts: ArgTy));
2903 } else {
2904 Name = "__sincospi_stret";
2905 ResTy = StructType::get(elt1: ArgTy, elts: ArgTy);
2906 }
2907
2908 if (!isLibFuncEmittable(M, TLI, Name))
2909 return false;
2910 LibFunc TheLibFunc;
2911 TLI->getLibFunc(funcName: Name, F&: TheLibFunc);
2912 FunctionCallee Callee = getOrInsertLibFunc(
2913 M, TLI: *TLI, TheLibFunc, AttributeList: OrigCallee->getAttributes(), RetTy: ResTy, Args: ArgTy);
2914
2915 if (Instruction *ArgInst = dyn_cast<Instruction>(Val: Arg)) {
2916 // If the argument is an instruction, it must dominate all uses so put our
2917 // sincos call there.
2918 B.SetInsertPoint(TheBB: ArgInst->getParent(), IP: ++ArgInst->getIterator());
2919 } else {
2920 // Otherwise (e.g. for a constant) the beginning of the function is as
2921 // good a place as any.
2922 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
2923 B.SetInsertPoint(TheBB: &EntryBB, IP: EntryBB.begin());
2924 }
2925
2926 SinCos = B.CreateCall(Callee, Args: Arg, Name: "sincospi");
2927
2928 if (SinCos->getType()->isStructTy()) {
2929 Sin = B.CreateExtractValue(Agg: SinCos, Idxs: 0, Name: "sinpi");
2930 Cos = B.CreateExtractValue(Agg: SinCos, Idxs: 1, Name: "cospi");
2931 } else {
2932 Sin = B.CreateExtractElement(Vec: SinCos, Idx: ConstantInt::get(Ty: B.getInt32Ty(), V: 0),
2933 Name: "sinpi");
2934 Cos = B.CreateExtractElement(Vec: SinCos, Idx: ConstantInt::get(Ty: B.getInt32Ty(), V: 1),
2935 Name: "cospi");
2936 }
2937
2938 return true;
2939}
2940
2941static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
2942 IRBuilderBase &B) {
2943 Value *X;
2944 Value *Src = CI->getArgOperand(i: 0);
2945
2946 if (match(V: Src, P: m_OneUse(SubPattern: m_FNeg(X: m_Value(V&: X))))) {
2947 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X});
2948 Call->copyFastMathFlags(I: CI);
2949 auto *CallInst = copyFlags(Old: *CI, New: Call);
2950 if (IsEven) {
2951 // Even function: f(-x) = f(x)
2952 return CallInst;
2953 }
2954 // Odd function: f(-x) = -f(x)
2955 return B.CreateFNegFMF(V: CallInst, FMFSource: CI);
2956 }
2957
2958 // Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
2959 if (IsEven && (match(V: Src, P: m_FAbs(Op0: m_Value(V&: X))) ||
2960 match(V: Src, P: m_CopySign(Op0: m_Value(V&: X), Op1: m_Value())))) {
2961 auto *Call = B.CreateCall(Callee: CI->getCalledFunction(), Args: {X});
2962 Call->copyFastMathFlags(I: CI);
2963 return copyFlags(Old: *CI, New: Call);
2964 }
2965
2966 return nullptr;
2967}
2968
2969Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
2970 IRBuilderBase &B) {
2971 switch (Func) {
2972 case LibFunc_cos:
2973 case LibFunc_cosf:
2974 case LibFunc_cosl:
2975 return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
2976
2977 case LibFunc_sin:
2978 case LibFunc_sinf:
2979 case LibFunc_sinl:
2980
2981 case LibFunc_tan:
2982 case LibFunc_tanf:
2983 case LibFunc_tanl:
2984
2985 case LibFunc_erf:
2986 case LibFunc_erff:
2987 case LibFunc_erfl:
2988 return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
2989
2990 default:
2991 return nullptr;
2992 }
2993}
2994
2995Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
2996 // Make sure the prototype is as expected, otherwise the rest of the
2997 // function is probably invalid and likely to abort.
2998 if (!isTrigLibCall(CI))
2999 return nullptr;
3000
3001 Value *Arg = CI->getArgOperand(i: 0);
3002 if (isa<ConstantData>(Val: Arg))
3003 return nullptr;
3004
3005 SmallVector<CallInst *, 1> SinCalls;
3006 SmallVector<CallInst *, 1> CosCalls;
3007 SmallVector<CallInst *, 1> SinCosCalls;
3008
3009 bool IsFloat = Arg->getType()->isFloatTy();
3010
3011 // Look for all compatible sinpi, cospi and sincospi calls with the same
3012 // argument. If there are enough (in some sense) we can make the
3013 // substitution.
3014 Function *F = CI->getFunction();
3015 for (User *U : Arg->users())
3016 classifyArgUse(Val: U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
3017
3018 // It's only worthwhile if both sinpi and cospi are actually used.
3019 if (SinCalls.empty() || CosCalls.empty())
3020 return nullptr;
3021
3022 Value *Sin, *Cos, *SinCos;
3023 if (!insertSinCosCall(B, OrigCallee: CI->getCalledFunction(), Arg, UseFloat: IsFloat, Sin, Cos,
3024 SinCos, TLI))
3025 return nullptr;
3026
3027 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
3028 Value *Res) {
3029 for (CallInst *C : Calls)
3030 replaceAllUsesWith(I: C, With: Res);
3031 };
3032
3033 replaceTrigInsts(SinCalls, Sin);
3034 replaceTrigInsts(CosCalls, Cos);
3035 replaceTrigInsts(SinCosCalls, SinCos);
3036
3037 return IsSin ? Sin : Cos;
3038}
3039
3040void LibCallSimplifier::classifyArgUse(
3041 Value *Val, Function *F, bool IsFloat,
3042 SmallVectorImpl<CallInst *> &SinCalls,
3043 SmallVectorImpl<CallInst *> &CosCalls,
3044 SmallVectorImpl<CallInst *> &SinCosCalls) {
3045 auto *CI = dyn_cast<CallInst>(Val);
3046 if (!CI || CI->use_empty())
3047 return;
3048
3049 // Don't consider calls in other functions.
3050 if (CI->getFunction() != F)
3051 return;
3052
3053 Module *M = CI->getModule();
3054 Function *Callee = CI->getCalledFunction();
3055 LibFunc Func;
3056 if (!Callee || !TLI->getLibFunc(FDecl: *Callee, F&: Func) ||
3057 !isLibFuncEmittable(M, TLI, TheLibFunc: Func) ||
3058 !isTrigLibCall(CI))
3059 return;
3060
3061 if (IsFloat) {
3062 if (Func == LibFunc_sinpif)
3063 SinCalls.push_back(Elt: CI);
3064 else if (Func == LibFunc_cospif)
3065 CosCalls.push_back(Elt: CI);
3066 else if (Func == LibFunc_sincospif_stret)
3067 SinCosCalls.push_back(Elt: CI);
3068 } else {
3069 if (Func == LibFunc_sinpi)
3070 SinCalls.push_back(Elt: CI);
3071 else if (Func == LibFunc_cospi)
3072 CosCalls.push_back(Elt: CI);
3073 else if (Func == LibFunc_sincospi_stret)
3074 SinCosCalls.push_back(Elt: CI);
3075 }
3076}
3077
3078/// Constant folds remquo
3079Value *LibCallSimplifier::optimizeRemquo(CallInst *CI, IRBuilderBase &B) {
3080 const APFloat *X, *Y;
3081 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3082 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3083 return nullptr;
3084
3085 APFloat::opStatus Status;
3086 APFloat Quot = *X;
3087 Status = Quot.divide(RHS: *Y, RM: APFloat::rmNearestTiesToEven);
3088 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3089 return nullptr;
3090 APFloat Rem = *X;
3091 if (Rem.remainder(RHS: *Y) != APFloat::opOK)
3092 return nullptr;
3093
3094 // TODO: We can only keep at least the three of the last bits of x/y
3095 unsigned IntBW = TLI->getIntSize();
3096 APSInt QuotInt(IntBW, /*isUnsigned=*/false);
3097 bool IsExact;
3098 Status =
3099 Quot.convertToInteger(Result&: QuotInt, RM: APFloat::rmNearestTiesToEven, IsExact: &IsExact);
3100 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3101 return nullptr;
3102
3103 B.CreateAlignedStore(
3104 Val: ConstantInt::get(Ty: B.getIntNTy(N: IntBW), V: QuotInt.getExtValue()),
3105 Ptr: CI->getArgOperand(i: 2), Align: CI->getParamAlign(ArgNo: 2));
3106 return ConstantFP::get(Ty: CI->getType(), V: Rem);
3107}
3108
3109/// Constant folds fdim
3110Value *LibCallSimplifier::optimizeFdim(CallInst *CI, IRBuilderBase &B) {
3111 // Cannot perform the fold unless the call has attribute memory(none)
3112 if (!CI->doesNotAccessMemory())
3113 return nullptr;
3114
3115 // TODO : Handle undef values
3116 // Propagate poison if any
3117 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 0)))
3118 return CI->getArgOperand(i: 0);
3119 if (isa<PoisonValue>(Val: CI->getArgOperand(i: 1)))
3120 return CI->getArgOperand(i: 1);
3121
3122 const APFloat *X, *Y;
3123 // Check if both values are constants
3124 if (!match(V: CI->getArgOperand(i: 0), P: m_APFloat(Res&: X)) ||
3125 !match(V: CI->getArgOperand(i: 1), P: m_APFloat(Res&: Y)))
3126 return nullptr;
3127
3128 APFloat Difference = *X;
3129 Difference.subtract(RHS: *Y, RM: RoundingMode::NearestTiesToEven);
3130
3131 APFloat MaxVal =
3132 maximum(A: Difference, B: APFloat::getZero(Sem: CI->getType()->getFltSemantics()));
3133 return ConstantFP::get(Ty: CI->getType(), V: MaxVal);
3134}
3135
3136//===----------------------------------------------------------------------===//
3137// Integer Library Call Optimizations
3138//===----------------------------------------------------------------------===//
3139
3140Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
3141 // All variants of ffs return int which need not be 32 bits wide.
3142 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
3143 Type *RetType = CI->getType();
3144 Value *Op = CI->getArgOperand(i: 0);
3145 Type *ArgType = Op->getType();
3146 Value *V = B.CreateIntrinsic(ID: Intrinsic::cttz, Types: {ArgType}, Args: {Op, B.getTrue()},
3147 FMFSource: nullptr, Name: "cttz");
3148 V = B.CreateAdd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: 1));
3149 V = B.CreateIntCast(V, DestTy: RetType, isSigned: false);
3150
3151 Value *Cond = B.CreateICmpNE(LHS: Op, RHS: Constant::getNullValue(Ty: ArgType));
3152 return B.CreateSelect(C: Cond, True: V, False: ConstantInt::get(Ty: RetType, V: 0));
3153}
3154
3155Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
3156 // All variants of fls return int which need not be 32 bits wide.
3157 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
3158 Value *Op = CI->getArgOperand(i: 0);
3159 Type *ArgType = Op->getType();
3160 Value *V = B.CreateIntrinsic(ID: Intrinsic::ctlz, Types: {ArgType}, Args: {Op, B.getFalse()},
3161 FMFSource: nullptr, Name: "ctlz");
3162 V = B.CreateSub(LHS: ConstantInt::get(Ty: V->getType(), V: ArgType->getIntegerBitWidth()),
3163 RHS: V);
3164 return B.CreateIntCast(V, DestTy: CI->getType(), isSigned: false);
3165}
3166
3167Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
3168 // abs(x) -> x <s 0 ? -x : x
3169 // The negation has 'nsw' because abs of INT_MIN is undefined.
3170 Value *X = CI->getArgOperand(i: 0);
3171 Value *IsNeg = B.CreateIsNeg(Arg: X);
3172 Value *NegX = B.CreateNSWNeg(V: X, Name: "neg");
3173 return B.CreateSelect(C: IsNeg, True: NegX, False: X);
3174}
3175
3176Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
3177 // isdigit(c) -> (c-'0') <u 10
3178 Value *Op = CI->getArgOperand(i: 0);
3179 Type *ArgType = Op->getType();
3180 Op = B.CreateSub(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: '0'), Name: "isdigittmp");
3181 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 10), Name: "isdigit");
3182 return B.CreateZExt(V: Op, DestTy: CI->getType());
3183}
3184
3185Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
3186 // isascii(c) -> c <u 128
3187 Value *Op = CI->getArgOperand(i: 0);
3188 Type *ArgType = Op->getType();
3189 Op = B.CreateICmpULT(LHS: Op, RHS: ConstantInt::get(Ty: ArgType, V: 128), Name: "isascii");
3190 return B.CreateZExt(V: Op, DestTy: CI->getType());
3191}
3192
3193Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
3194 // toascii(c) -> c & 0x7f
3195 return B.CreateAnd(LHS: CI->getArgOperand(i: 0),
3196 RHS: ConstantInt::get(Ty: CI->getType(), V: 0x7F));
3197}
3198
3199// Fold calls to atoi, atol, and atoll.
3200Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
3201 StringRef Str;
3202 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3203 return nullptr;
3204
3205 return convertStrToInt(CI, Str, EndPtr: nullptr, Base: 10, /*AsSigned=*/true, B);
3206}
3207
3208// Fold calls to strtol, strtoll, strtoul, and strtoull.
3209Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
3210 bool AsSigned) {
3211 Value *EndPtr = CI->getArgOperand(i: 1);
3212 if (isa<ConstantPointerNull>(Val: EndPtr)) {
3213 // With a null EndPtr, this function won't capture the main argument.
3214 // It would be readonly too, except that it still may write to errno.
3215 CI->addParamAttr(ArgNo: 0, Attr: Attribute::getWithCaptureInfo(Context&: CI->getContext(),
3216 CI: CaptureInfo::none()));
3217 EndPtr = nullptr;
3218 } else if (!isKnownNonZero(V: EndPtr, Q: DL))
3219 return nullptr;
3220
3221 StringRef Str;
3222 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str))
3223 return nullptr;
3224
3225 if (ConstantInt *CInt = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2))) {
3226 return convertStrToInt(CI, Str, EndPtr, Base: CInt->getSExtValue(), AsSigned, B);
3227 }
3228
3229 return nullptr;
3230}
3231
3232//===----------------------------------------------------------------------===//
3233// Formatting and IO Library Call Optimizations
3234//===----------------------------------------------------------------------===//
3235
3236static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
3237
3238Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3239 int StreamArg) {
3240 Function *Callee = CI->getCalledFunction();
3241 // Error reporting calls should be cold, mark them as such.
3242 // This applies even to non-builtin calls: it is only a hint and applies to
3243 // functions that the frontend might not understand as builtins.
3244
3245 // This heuristic was suggested in:
3246 // Improving Static Branch Prediction in a Compiler
3247 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3248 // Proceedings of PACT'98, Oct. 1998, IEEE
3249 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
3250 isReportingError(Callee, CI, StreamArg)) {
3251 CI->addFnAttr(Kind: Attribute::Cold);
3252 }
3253
3254 return nullptr;
3255}
3256
3257static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3258 if (!Callee || !Callee->isDeclaration())
3259 return false;
3260
3261 if (StreamArg < 0)
3262 return true;
3263
3264 // These functions might be considered cold, but only if their stream
3265 // argument is stderr.
3266
3267 if (StreamArg >= (int)CI->arg_size())
3268 return false;
3269 LoadInst *LI = dyn_cast<LoadInst>(Val: CI->getArgOperand(i: StreamArg));
3270 if (!LI)
3271 return false;
3272 GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: LI->getPointerOperand());
3273 if (!GV || !GV->isDeclaration())
3274 return false;
3275 return GV->getName() == "stderr";
3276}
3277
3278Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3279 // Check for a fixed format string.
3280 StringRef FormatStr;
3281 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: FormatStr))
3282 return nullptr;
3283
3284 // Empty format string -> noop.
3285 if (FormatStr.empty()) // Tolerate printf's declared void.
3286 return CI->use_empty() ? (Value *)CI : ConstantInt::get(Ty: CI->getType(), V: 0);
3287
3288 // Do not do any of the following transformations if the printf return value
3289 // is used, in general the printf return value is not compatible with either
3290 // putchar() or puts().
3291 if (!CI->use_empty())
3292 return nullptr;
3293
3294 Type *IntTy = CI->getType();
3295 // printf("x") -> putchar('x'), even for "%" and "%%".
3296 if (FormatStr.size() == 1 || FormatStr == "%%") {
3297 // Convert the character to unsigned char before passing it to putchar
3298 // to avoid host-specific sign extension in the IR. Putchar converts
3299 // it to unsigned char regardless.
3300 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)FormatStr[0]);
3301 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3302 }
3303
3304 // Try to remove call or emit putchar/puts.
3305 if (FormatStr == "%s" && CI->arg_size() > 1) {
3306 StringRef OperandStr;
3307 if (!getConstantStringInfo(V: CI->getOperand(i_nocapture: 1), Str&: OperandStr))
3308 return nullptr;
3309 // printf("%s", "") --> NOP
3310 if (OperandStr.empty())
3311 return (Value *)CI;
3312 // printf("%s", "a") --> putchar('a')
3313 if (OperandStr.size() == 1) {
3314 // Convert the character to unsigned char before passing it to putchar
3315 // to avoid host-specific sign extension in the IR. Putchar converts
3316 // it to unsigned char regardless.
3317 Value *IntChar = ConstantInt::get(Ty: IntTy, V: (unsigned char)OperandStr[0]);
3318 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3319 }
3320 // printf("%s", str"\n") --> puts(str)
3321 if (OperandStr.back() == '\n') {
3322 OperandStr = OperandStr.drop_back();
3323 Value *GV = B.CreateGlobalString(Str: OperandStr, Name: "str");
3324 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3325 }
3326 return nullptr;
3327 }
3328
3329 // printf("foo\n") --> puts("foo")
3330 if (FormatStr.back() == '\n' &&
3331 !FormatStr.contains(C: '%')) { // No format characters.
3332 // Create a string literal with no \n on it. We expect the constant merge
3333 // pass to be run after this pass, to merge duplicate strings.
3334 FormatStr = FormatStr.drop_back();
3335 Value *GV = B.CreateGlobalString(Str: FormatStr, Name: "str");
3336 return copyFlags(Old: *CI, New: emitPutS(Str: GV, B, TLI));
3337 }
3338
3339 // Optimize specific format strings.
3340 // printf("%c", chr) --> putchar(chr)
3341 if (FormatStr == "%c" && CI->arg_size() > 1 &&
3342 CI->getArgOperand(i: 1)->getType()->isIntegerTy()) {
3343 // Convert the argument to the type expected by putchar, i.e., int, which
3344 // need not be 32 bits wide but which is the same as printf's return type.
3345 Value *IntChar = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: IntTy, isSigned: false);
3346 return copyFlags(Old: *CI, New: emitPutChar(Char: IntChar, B, TLI));
3347 }
3348
3349 // printf("%s\n", str) --> puts(str)
3350 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3351 CI->getArgOperand(i: 1)->getType()->isPointerTy())
3352 return copyFlags(Old: *CI, New: emitPutS(Str: CI->getArgOperand(i: 1), B, TLI));
3353 return nullptr;
3354}
3355
3356Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3357
3358 Module *M = CI->getModule();
3359 Function *Callee = CI->getCalledFunction();
3360 FunctionType *FT = Callee->getFunctionType();
3361 if (Value *V = optimizePrintFString(CI, B)) {
3362 return V;
3363 }
3364
3365 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3366
3367 // printf(format, ...) -> iprintf(format, ...) if no floating point
3368 // arguments.
3369 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_iprintf) &&
3370 !callHasFloatingPointArgument(CI)) {
3371 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_iprintf, T: FT,
3372 AttributeList: Callee->getAttributes());
3373 CallInst *New = cast<CallInst>(Val: CI->clone());
3374 New->setCalledFunction(IPrintFFn);
3375 B.Insert(I: New);
3376 return New;
3377 }
3378
3379 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3380 // arguments.
3381 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_printf) &&
3382 !callHasFP128Argument(CI)) {
3383 auto SmallPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_printf, T: FT,
3384 AttributeList: Callee->getAttributes());
3385 CallInst *New = cast<CallInst>(Val: CI->clone());
3386 New->setCalledFunction(SmallPrintFFn);
3387 B.Insert(I: New);
3388 return New;
3389 }
3390
3391 return nullptr;
3392}
3393
3394Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3395 IRBuilderBase &B) {
3396 // Check for a fixed format string.
3397 StringRef FormatStr;
3398 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3399 return nullptr;
3400
3401 // If we just have a format string (nothing else crazy) transform it.
3402 Value *Dest = CI->getArgOperand(i: 0);
3403 if (CI->arg_size() == 2) {
3404 // Make sure there's no % in the constant array. We could try to handle
3405 // %% -> % in the future if we cared.
3406 if (FormatStr.contains(C: '%'))
3407 return nullptr; // we found a format specifier, bail out.
3408
3409 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3410 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 1), SrcAlign: Align(1),
3411 // Copy the null byte.
3412 Size: TLI->getAsSizeT(V: FormatStr.size() + 1, M: *CI->getModule()));
3413 return ConstantInt::get(Ty: CI->getType(), V: FormatStr.size());
3414 }
3415
3416 // The remaining optimizations require the format string to be "%s" or "%c"
3417 // and have an extra operand.
3418 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3419 return nullptr;
3420
3421 // Decode the second character of the format string.
3422 if (FormatStr[1] == 'c') {
3423 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3424 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3425 return nullptr;
3426 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 2), DestTy: B.getInt8Ty(), Name: "char");
3427 Value *Ptr = Dest;
3428 B.CreateStore(Val: V, Ptr);
3429 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3430 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3431
3432 return ConstantInt::get(Ty: CI->getType(), V: 1);
3433 }
3434
3435 if (FormatStr[1] == 's') {
3436 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3437 // strlen(str)+1)
3438 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3439 return nullptr;
3440
3441 if (CI->use_empty())
3442 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3443 return copyFlags(Old: *CI, New: emitStrCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI));
3444
3445 uint64_t SrcLen = GetStringLength(V: CI->getArgOperand(i: 2));
3446 if (SrcLen) {
3447 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1),
3448 Size: TLI->getAsSizeT(V: SrcLen, M: *CI->getModule()));
3449 // Returns total number of characters written without null-character.
3450 return ConstantInt::get(Ty: CI->getType(), V: SrcLen - 1);
3451 } else if (Value *V = emitStpCpy(Dst: Dest, Src: CI->getArgOperand(i: 2), B, TLI)) {
3452 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3453 Value *PtrDiff = B.CreatePtrDiff(ElemTy: B.getInt8Ty(), LHS: V, RHS: Dest);
3454 return B.CreateIntCast(V: PtrDiff, DestTy: CI->getType(), isSigned: false);
3455 }
3456
3457 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3458 QueryType: PGSOQueryType::IRPass))
3459 return nullptr;
3460
3461 Value *Len = emitStrLen(Ptr: CI->getArgOperand(i: 2), B, DL, TLI);
3462 if (!Len)
3463 return nullptr;
3464 Value *IncLen =
3465 B.CreateAdd(LHS: Len, RHS: ConstantInt::get(Ty: Len->getType(), V: 1), Name: "leninc");
3466 B.CreateMemCpy(Dst: Dest, DstAlign: Align(1), Src: CI->getArgOperand(i: 2), SrcAlign: Align(1), Size: IncLen);
3467
3468 // The sprintf result is the unincremented number of bytes in the string.
3469 return B.CreateIntCast(V: Len, DestTy: CI->getType(), isSigned: false);
3470 }
3471 return nullptr;
3472}
3473
3474Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3475 Module *M = CI->getModule();
3476 Function *Callee = CI->getCalledFunction();
3477 FunctionType *FT = Callee->getFunctionType();
3478 if (Value *V = optimizeSPrintFString(CI, B)) {
3479 return V;
3480 }
3481
3482 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: {0, 1});
3483
3484 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3485 // point arguments.
3486 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_siprintf) &&
3487 !callHasFloatingPointArgument(CI)) {
3488 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_siprintf,
3489 T: FT, AttributeList: Callee->getAttributes());
3490 CallInst *New = cast<CallInst>(Val: CI->clone());
3491 New->setCalledFunction(SIPrintFFn);
3492 B.Insert(I: New);
3493 return New;
3494 }
3495
3496 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3497 // floating point arguments.
3498 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_sprintf) &&
3499 !callHasFP128Argument(CI)) {
3500 auto SmallSPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_sprintf, T: FT,
3501 AttributeList: Callee->getAttributes());
3502 CallInst *New = cast<CallInst>(Val: CI->clone());
3503 New->setCalledFunction(SmallSPrintFFn);
3504 B.Insert(I: New);
3505 return New;
3506 }
3507
3508 return nullptr;
3509}
3510
3511// Transform an snprintf call CI with the bound N to format the string Str
3512// either to a call to memcpy, or to single character a store, or to nothing,
3513// and fold the result to a constant. A nonnull StrArg refers to the string
3514// argument being formatted. Otherwise the call is one with N < 2 and
3515// the "%c" directive to format a single character.
3516Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3517 StringRef Str, uint64_t N,
3518 IRBuilderBase &B) {
3519 assert(StrArg || (N < 2 && Str.size() == 1));
3520
3521 unsigned IntBits = TLI->getIntSize();
3522 uint64_t IntMax = maxIntN(N: IntBits);
3523 if (Str.size() > IntMax)
3524 // Bail if the string is longer than INT_MAX. POSIX requires
3525 // implementations to set errno to EOVERFLOW in this case, in
3526 // addition to when N is larger than that (checked by the caller).
3527 return nullptr;
3528
3529 Value *StrLen = ConstantInt::get(Ty: CI->getType(), V: Str.size());
3530 if (N == 0)
3531 return StrLen;
3532
3533 // Set to the number of bytes to copy fron StrArg which is also
3534 // the offset of the terinating nul.
3535 uint64_t NCopy;
3536 if (N > Str.size())
3537 // Copy the full string, including the terminating nul (which must
3538 // be present regardless of the bound).
3539 NCopy = Str.size() + 1;
3540 else
3541 NCopy = N - 1;
3542
3543 Value *DstArg = CI->getArgOperand(i: 0);
3544 if (NCopy && StrArg)
3545 // Transform the call to lvm.memcpy(dst, fmt, N).
3546 copyFlags(Old: *CI, New: B.CreateMemCpy(Dst: DstArg, DstAlign: Align(1), Src: StrArg, SrcAlign: Align(1),
3547 Size: TLI->getAsSizeT(V: NCopy, M: *CI->getModule())));
3548
3549 if (N > Str.size())
3550 // Return early when the whole format string, including the final nul,
3551 // has been copied.
3552 return StrLen;
3553
3554 // Otherwise, when truncating the string append a terminating nul.
3555 Type *Int8Ty = B.getInt8Ty();
3556 Value *NulOff = B.getIntN(N: IntBits, C: NCopy);
3557 Value *DstEnd = B.CreateInBoundsGEP(Ty: Int8Ty, Ptr: DstArg, IdxList: NulOff, Name: "endptr");
3558 B.CreateStore(Val: ConstantInt::get(Ty: Int8Ty, V: 0), Ptr: DstEnd);
3559 return StrLen;
3560}
3561
3562Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3563 IRBuilderBase &B) {
3564 // Check for size
3565 ConstantInt *Size = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3566 if (!Size)
3567 return nullptr;
3568
3569 uint64_t N = Size->getZExtValue();
3570 uint64_t IntMax = maxIntN(N: TLI->getIntSize());
3571 if (N > IntMax)
3572 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3573 // to set errno to EOVERFLOW in this case.
3574 return nullptr;
3575
3576 Value *DstArg = CI->getArgOperand(i: 0);
3577 Value *FmtArg = CI->getArgOperand(i: 2);
3578
3579 // Check for a fixed format string.
3580 StringRef FormatStr;
3581 if (!getConstantStringInfo(V: FmtArg, Str&: FormatStr))
3582 return nullptr;
3583
3584 // If we just have a format string (nothing else crazy) transform it.
3585 if (CI->arg_size() == 3) {
3586 if (FormatStr.contains(C: '%'))
3587 // Bail if the format string contains a directive and there are
3588 // no arguments. We could handle "%%" in the future.
3589 return nullptr;
3590
3591 return emitSnPrintfMemCpy(CI, StrArg: FmtArg, Str: FormatStr, N, B);
3592 }
3593
3594 // The remaining optimizations require the format string to be "%s" or "%c"
3595 // and have an extra operand.
3596 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3597 return nullptr;
3598
3599 // Decode the second character of the format string.
3600 if (FormatStr[1] == 'c') {
3601 if (N <= 1) {
3602 // Use an arbitary string of length 1 to transform the call into
3603 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3604 // to one.
3605 StringRef CharStr("*");
3606 return emitSnPrintfMemCpy(CI, StrArg: nullptr, Str: CharStr, N, B);
3607 }
3608
3609 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3610 if (!CI->getArgOperand(i: 3)->getType()->isIntegerTy())
3611 return nullptr;
3612 Value *V = B.CreateTrunc(V: CI->getArgOperand(i: 3), DestTy: B.getInt8Ty(), Name: "char");
3613 Value *Ptr = DstArg;
3614 B.CreateStore(Val: V, Ptr);
3615 Ptr = B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr, IdxList: B.getInt32(C: 1), Name: "nul");
3616 B.CreateStore(Val: B.getInt8(C: 0), Ptr);
3617 return ConstantInt::get(Ty: CI->getType(), V: 1);
3618 }
3619
3620 if (FormatStr[1] != 's')
3621 return nullptr;
3622
3623 Value *StrArg = CI->getArgOperand(i: 3);
3624 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3625 StringRef Str;
3626 if (!getConstantStringInfo(V: StrArg, Str))
3627 return nullptr;
3628
3629 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3630}
3631
3632Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3633 if (Value *V = optimizeSnPrintFString(CI, B)) {
3634 return V;
3635 }
3636
3637 if (isKnownNonZero(V: CI->getOperand(i_nocapture: 1), Q: DL))
3638 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3639 return nullptr;
3640}
3641
3642Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3643 IRBuilderBase &B) {
3644 optimizeErrorReporting(CI, B, StreamArg: 0);
3645
3646 // All the optimizations depend on the format string.
3647 StringRef FormatStr;
3648 if (!getConstantStringInfo(V: CI->getArgOperand(i: 1), Str&: FormatStr))
3649 return nullptr;
3650
3651 // Do not do any of the following transformations if the fprintf return
3652 // value is used, in general the fprintf return value is not compatible
3653 // with fwrite(), fputc() or fputs().
3654 if (!CI->use_empty())
3655 return nullptr;
3656
3657 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3658 if (CI->arg_size() == 2) {
3659 // Could handle %% -> % if we cared.
3660 if (FormatStr.contains(C: '%'))
3661 return nullptr; // We found a format specifier.
3662
3663 return copyFlags(
3664 Old: *CI, New: emitFWrite(Ptr: CI->getArgOperand(i: 1),
3665 Size: TLI->getAsSizeT(V: FormatStr.size(), M: *CI->getModule()),
3666 File: CI->getArgOperand(i: 0), B, DL, TLI));
3667 }
3668
3669 // The remaining optimizations require the format string to be "%s" or "%c"
3670 // and have an extra operand.
3671 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3672 return nullptr;
3673
3674 // Decode the second character of the format string.
3675 if (FormatStr[1] == 'c') {
3676 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3677 if (!CI->getArgOperand(i: 2)->getType()->isIntegerTy())
3678 return nullptr;
3679 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3680 Value *V = B.CreateIntCast(V: CI->getArgOperand(i: 2), DestTy: IntTy, /*isSigned*/ true,
3681 Name: "chari");
3682 return copyFlags(Old: *CI, New: emitFPutC(Char: V, File: CI->getArgOperand(i: 0), B, TLI));
3683 }
3684
3685 if (FormatStr[1] == 's') {
3686 // fprintf(F, "%s", str) --> fputs(str, F)
3687 if (!CI->getArgOperand(i: 2)->getType()->isPointerTy())
3688 return nullptr;
3689 return copyFlags(
3690 Old: *CI, New: emitFPutS(Str: CI->getArgOperand(i: 2), File: CI->getArgOperand(i: 0), B, TLI));
3691 }
3692 return nullptr;
3693}
3694
3695Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3696 Module *M = CI->getModule();
3697 Function *Callee = CI->getCalledFunction();
3698 FunctionType *FT = Callee->getFunctionType();
3699 if (Value *V = optimizeFPrintFString(CI, B)) {
3700 return V;
3701 }
3702
3703 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3704 // floating point arguments.
3705 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_fiprintf) &&
3706 !callHasFloatingPointArgument(CI)) {
3707 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_fiprintf,
3708 T: FT, AttributeList: Callee->getAttributes());
3709 CallInst *New = cast<CallInst>(Val: CI->clone());
3710 New->setCalledFunction(FIPrintFFn);
3711 B.Insert(I: New);
3712 return New;
3713 }
3714
3715 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3716 // 128-bit floating point arguments.
3717 if (isLibFuncEmittable(M, TLI, TheLibFunc: LibFunc_small_fprintf) &&
3718 !callHasFP128Argument(CI)) {
3719 auto SmallFPrintFFn =
3720 getOrInsertLibFunc(M, TLI: *TLI, TheLibFunc: LibFunc_small_fprintf, T: FT,
3721 AttributeList: Callee->getAttributes());
3722 CallInst *New = cast<CallInst>(Val: CI->clone());
3723 New->setCalledFunction(SmallFPrintFFn);
3724 B.Insert(I: New);
3725 return New;
3726 }
3727
3728 return nullptr;
3729}
3730
3731Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3732 optimizeErrorReporting(CI, B, StreamArg: 3);
3733
3734 // Get the element size and count.
3735 ConstantInt *SizeC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 1));
3736 ConstantInt *CountC = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
3737 if (SizeC && CountC) {
3738 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3739
3740 // If this is writing zero records, remove the call (it's a noop).
3741 if (Bytes == 0)
3742 return ConstantInt::get(Ty: CI->getType(), V: 0);
3743
3744 // If this is writing one byte, turn it into fputc.
3745 // This optimisation is only valid, if the return value is unused.
3746 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
3747 Value *Char = B.CreateLoad(Ty: B.getInt8Ty(), Ptr: CI->getArgOperand(i: 0), Name: "char");
3748 Type *IntTy = B.getIntNTy(N: TLI->getIntSize());
3749 Value *Cast = B.CreateIntCast(V: Char, DestTy: IntTy, /*isSigned*/ true, Name: "chari");
3750 Value *NewCI = emitFPutC(Char: Cast, File: CI->getArgOperand(i: 3), B, TLI);
3751 return NewCI ? ConstantInt::get(Ty: CI->getType(), V: 1) : nullptr;
3752 }
3753 }
3754
3755 return nullptr;
3756}
3757
3758Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
3759 optimizeErrorReporting(CI, B, StreamArg: 1);
3760
3761 // Don't rewrite fputs to fwrite when optimising for size because fwrite
3762 // requires more arguments and thus extra MOVs are required.
3763 if (llvm::shouldOptimizeForSize(BB: CI->getParent(), PSI, BFI,
3764 QueryType: PGSOQueryType::IRPass))
3765 return nullptr;
3766
3767 // We can't optimize if return value is used.
3768 if (!CI->use_empty())
3769 return nullptr;
3770
3771 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
3772 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: 0));
3773 if (!Len)
3774 return nullptr;
3775
3776 // Known to have no uses (see above).
3777 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
3778 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
3779 return copyFlags(
3780 Old: *CI,
3781 New: emitFWrite(Ptr: CI->getArgOperand(i: 0),
3782 Size: ConstantInt::get(Ty: SizeTTy, V: Len - 1),
3783 File: CI->getArgOperand(i: 1), B, DL, TLI));
3784}
3785
3786Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
3787 annotateNonNullNoUndefBasedOnAccess(CI, ArgNos: 0);
3788 if (!CI->use_empty())
3789 return nullptr;
3790
3791 // Check for a constant string.
3792 // puts("") -> putchar('\n')
3793 StringRef Str;
3794 if (getConstantStringInfo(V: CI->getArgOperand(i: 0), Str) && Str.empty()) {
3795 // putchar takes an argument of the same type as puts returns, i.e.,
3796 // int, which need not be 32 bits wide.
3797 Type *IntTy = CI->getType();
3798 return copyFlags(Old: *CI, New: emitPutChar(Char: ConstantInt::get(Ty: IntTy, V: '\n'), B, TLI));
3799 }
3800
3801 return nullptr;
3802}
3803
3804Value *LibCallSimplifier::optimizeExit(CallInst *CI) {
3805
3806 // Mark 'exit' as cold if its not exit(0) (success).
3807 const APInt *C;
3808 if (!CI->hasFnAttr(Kind: Attribute::Cold) &&
3809 match(V: CI->getArgOperand(i: 0), P: m_APInt(Res&: C)) && !C->isZero()) {
3810 CI->addFnAttr(Kind: Attribute::Cold);
3811 }
3812 return nullptr;
3813}
3814
3815Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
3816 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
3817 return copyFlags(Old: *CI, New: B.CreateMemMove(Dst: CI->getArgOperand(i: 1), DstAlign: Align(1),
3818 Src: CI->getArgOperand(i: 0), SrcAlign: Align(1),
3819 Size: CI->getArgOperand(i: 2)));
3820}
3821
3822bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
3823 SmallString<20> FloatFuncName = FuncName;
3824 FloatFuncName += 'f';
3825 return isLibFuncEmittable(M, TLI, Name: FloatFuncName);
3826}
3827
3828Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
3829 IRBuilderBase &Builder) {
3830 Module *M = CI->getModule();
3831 LibFunc Func;
3832 Function *Callee = CI->getCalledFunction();
3833
3834 // Check for string/memory library functions.
3835 if (TLI->getLibFunc(FDecl: *Callee, F&: Func) && isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
3836 // Make sure we never change the calling convention.
3837 assert(
3838 (ignoreCallingConv(Func) ||
3839 TargetLibraryInfoImpl::isCallingConvCCompatible(CI)) &&
3840 "Optimizing string/memory libcall would change the calling convention");
3841 switch (Func) {
3842 case LibFunc_strcat:
3843 return optimizeStrCat(CI, B&: Builder);
3844 case LibFunc_strncat:
3845 return optimizeStrNCat(CI, B&: Builder);
3846 case LibFunc_strchr:
3847 return optimizeStrChr(CI, B&: Builder);
3848 case LibFunc_strrchr:
3849 return optimizeStrRChr(CI, B&: Builder);
3850 case LibFunc_strcmp:
3851 return optimizeStrCmp(CI, B&: Builder);
3852 case LibFunc_strncmp:
3853 return optimizeStrNCmp(CI, B&: Builder);
3854 case LibFunc_strcpy:
3855 return optimizeStrCpy(CI, B&: Builder);
3856 case LibFunc_stpcpy:
3857 return optimizeStpCpy(CI, B&: Builder);
3858 case LibFunc_strlcpy:
3859 return optimizeStrLCpy(CI, B&: Builder);
3860 case LibFunc_stpncpy:
3861 return optimizeStringNCpy(CI, /*RetEnd=*/true, B&: Builder);
3862 case LibFunc_strncpy:
3863 return optimizeStringNCpy(CI, /*RetEnd=*/false, B&: Builder);
3864 case LibFunc_strlen:
3865 return optimizeStrLen(CI, B&: Builder);
3866 case LibFunc_strnlen:
3867 return optimizeStrNLen(CI, B&: Builder);
3868 case LibFunc_strpbrk:
3869 return optimizeStrPBrk(CI, B&: Builder);
3870 case LibFunc_strndup:
3871 return optimizeStrNDup(CI, B&: Builder);
3872 case LibFunc_strtol:
3873 case LibFunc_strtod:
3874 case LibFunc_strtof:
3875 case LibFunc_strtoul:
3876 case LibFunc_strtoll:
3877 case LibFunc_strtold:
3878 case LibFunc_strtoull:
3879 return optimizeStrTo(CI, B&: Builder);
3880 case LibFunc_strspn:
3881 return optimizeStrSpn(CI, B&: Builder);
3882 case LibFunc_strcspn:
3883 return optimizeStrCSpn(CI, B&: Builder);
3884 case LibFunc_strstr:
3885 return optimizeStrStr(CI, B&: Builder);
3886 case LibFunc_memchr:
3887 return optimizeMemChr(CI, B&: Builder);
3888 case LibFunc_memrchr:
3889 return optimizeMemRChr(CI, B&: Builder);
3890 case LibFunc_bcmp:
3891 return optimizeBCmp(CI, B&: Builder);
3892 case LibFunc_memcmp:
3893 return optimizeMemCmp(CI, B&: Builder);
3894 case LibFunc_memcpy:
3895 return optimizeMemCpy(CI, B&: Builder);
3896 case LibFunc_memccpy:
3897 return optimizeMemCCpy(CI, B&: Builder);
3898 case LibFunc_mempcpy:
3899 return optimizeMemPCpy(CI, B&: Builder);
3900 case LibFunc_memmove:
3901 return optimizeMemMove(CI, B&: Builder);
3902 case LibFunc_memset:
3903 return optimizeMemSet(CI, B&: Builder);
3904 case LibFunc_realloc:
3905 return optimizeRealloc(CI, B&: Builder);
3906 case LibFunc_wcslen:
3907 return optimizeWcslen(CI, B&: Builder);
3908 case LibFunc_bcopy:
3909 return optimizeBCopy(CI, B&: Builder);
3910 case LibFunc_Znwm:
3911 case LibFunc_ZnwmRKSt9nothrow_t:
3912 case LibFunc_ZnwmSt11align_val_t:
3913 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
3914 case LibFunc_Znam:
3915 case LibFunc_ZnamRKSt9nothrow_t:
3916 case LibFunc_ZnamSt11align_val_t:
3917 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
3918 case LibFunc_Znwm12__hot_cold_t:
3919 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
3920 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
3921 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
3922 case LibFunc_Znam12__hot_cold_t:
3923 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
3924 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
3925 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
3926 case LibFunc_size_returning_new:
3927 case LibFunc_size_returning_new_hot_cold:
3928 case LibFunc_size_returning_new_aligned:
3929 case LibFunc_size_returning_new_aligned_hot_cold:
3930 return optimizeNew(CI, B&: Builder, Func);
3931 default:
3932 break;
3933 }
3934 }
3935 return nullptr;
3936}
3937
3938/// Constant folding nan/nanf/nanl.
3939static Value *optimizeNaN(CallInst *CI) {
3940 StringRef CharSeq;
3941 if (!getConstantStringInfo(V: CI->getArgOperand(i: 0), Str&: CharSeq))
3942 return nullptr;
3943
3944 APInt Fill;
3945 // Treat empty strings as if they were zero.
3946 if (CharSeq.empty())
3947 Fill = APInt(32, 0);
3948 else if (CharSeq.getAsInteger(Radix: 0, Result&: Fill))
3949 return nullptr;
3950
3951 return ConstantFP::getQNaN(Ty: CI->getType(), /*Negative=*/false, Payload: &Fill);
3952}
3953
3954Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
3955 LibFunc Func,
3956 IRBuilderBase &Builder) {
3957 const Module *M = CI->getModule();
3958
3959 // Don't optimize calls that require strict floating point semantics.
3960 if (CI->isStrictFP())
3961 return nullptr;
3962
3963 if (Value *V = optimizeSymmetric(CI, Func, B&: Builder))
3964 return V;
3965
3966 switch (Func) {
3967 case LibFunc_sinpif:
3968 case LibFunc_sinpi:
3969 return optimizeSinCosPi(CI, /*IsSin*/true, B&: Builder);
3970 case LibFunc_cospif:
3971 case LibFunc_cospi:
3972 return optimizeSinCosPi(CI, /*IsSin*/false, B&: Builder);
3973 case LibFunc_powf:
3974 case LibFunc_pow:
3975 case LibFunc_powl:
3976 return optimizePow(Pow: CI, B&: Builder);
3977 case LibFunc_exp2l:
3978 case LibFunc_exp2:
3979 case LibFunc_exp2f:
3980 return optimizeExp2(CI, B&: Builder);
3981 case LibFunc_fabsf:
3982 case LibFunc_fabs:
3983 case LibFunc_fabsl:
3984 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::fabs);
3985 case LibFunc_sqrtf:
3986 case LibFunc_sqrt:
3987 case LibFunc_sqrtl:
3988 return optimizeSqrt(CI, B&: Builder);
3989 case LibFunc_fmod:
3990 case LibFunc_fmodf:
3991 case LibFunc_fmodl:
3992 return optimizeFMod(CI, B&: Builder);
3993 case LibFunc_logf:
3994 case LibFunc_log:
3995 case LibFunc_logl:
3996 case LibFunc_log10f:
3997 case LibFunc_log10:
3998 case LibFunc_log10l:
3999 case LibFunc_log1pf:
4000 case LibFunc_log1p:
4001 case LibFunc_log1pl:
4002 case LibFunc_log2f:
4003 case LibFunc_log2:
4004 case LibFunc_log2l:
4005 case LibFunc_logbf:
4006 case LibFunc_logb:
4007 case LibFunc_logbl:
4008 return optimizeLog(Log: CI, B&: Builder);
4009 case LibFunc_tan:
4010 case LibFunc_tanf:
4011 case LibFunc_tanl:
4012 case LibFunc_sinh:
4013 case LibFunc_sinhf:
4014 case LibFunc_sinhl:
4015 case LibFunc_asinh:
4016 case LibFunc_asinhf:
4017 case LibFunc_asinhl:
4018 case LibFunc_cosh:
4019 case LibFunc_coshf:
4020 case LibFunc_coshl:
4021 case LibFunc_atanh:
4022 case LibFunc_atanhf:
4023 case LibFunc_atanhl:
4024 return optimizeTrigInversionPairs(CI, B&: Builder);
4025 case LibFunc_ceil:
4026 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::ceil);
4027 case LibFunc_floor:
4028 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::floor);
4029 case LibFunc_round:
4030 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::round);
4031 case LibFunc_roundeven:
4032 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::roundeven);
4033 case LibFunc_nearbyint:
4034 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::nearbyint);
4035 case LibFunc_rint:
4036 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::rint);
4037 case LibFunc_trunc:
4038 return replaceUnaryCall(CI, B&: Builder, IID: Intrinsic::trunc);
4039 case LibFunc_acos:
4040 case LibFunc_acosh:
4041 case LibFunc_asin:
4042 case LibFunc_atan:
4043 case LibFunc_cbrt:
4044 case LibFunc_exp:
4045 case LibFunc_exp10:
4046 case LibFunc_expm1:
4047 case LibFunc_cos:
4048 case LibFunc_sin:
4049 case LibFunc_tanh:
4050 if (UnsafeFPShrink && hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4051 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, isPrecise: true);
4052 return nullptr;
4053 case LibFunc_copysign:
4054 if (hasFloatVersion(M, FuncName: CI->getCalledFunction()->getName()))
4055 return optimizeBinaryDoubleFP(CI, B&: Builder, TLI);
4056 return nullptr;
4057 case LibFunc_fdim:
4058 case LibFunc_fdimf:
4059 case LibFunc_fdiml:
4060 return optimizeFdim(CI, B&: Builder);
4061 case LibFunc_fminf:
4062 case LibFunc_fmin:
4063 case LibFunc_fminl:
4064 case LibFunc_fmaxf:
4065 case LibFunc_fmax:
4066 case LibFunc_fmaxl:
4067 return optimizeFMinFMax(CI, B&: Builder);
4068 case LibFunc_cabs:
4069 case LibFunc_cabsf:
4070 case LibFunc_cabsl:
4071 return optimizeCAbs(CI, B&: Builder);
4072 case LibFunc_remquo:
4073 case LibFunc_remquof:
4074 case LibFunc_remquol:
4075 return optimizeRemquo(CI, B&: Builder);
4076 case LibFunc_nan:
4077 case LibFunc_nanf:
4078 case LibFunc_nanl:
4079 return optimizeNaN(CI);
4080 default:
4081 return nullptr;
4082 }
4083}
4084
4085Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) {
4086 Module *M = CI->getModule();
4087 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
4088
4089 // TODO: Split out the code below that operates on FP calls so that
4090 // we can all non-FP calls with the StrictFP attribute to be
4091 // optimized.
4092 if (CI->isNoBuiltin())
4093 return nullptr;
4094
4095 LibFunc Func;
4096 Function *Callee = CI->getCalledFunction();
4097 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4098
4099 SmallVector<OperandBundleDef, 2> OpBundles;
4100 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4101
4102 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4103 Builder.setDefaultOperandBundles(OpBundles);
4104
4105 // Command-line parameter overrides instruction attribute.
4106 // This can't be moved to optimizeFloatingPointLibCall() because it may be
4107 // used by the intrinsic optimizations.
4108 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
4109 UnsafeFPShrink = EnableUnsafeFPShrink;
4110 else if (isa<FPMathOperator>(Val: CI) && CI->isFast())
4111 UnsafeFPShrink = true;
4112
4113 // First, check for intrinsics.
4114 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI)) {
4115 if (!IsCallingConvC)
4116 return nullptr;
4117 // The FP intrinsics have corresponding constrained versions so we don't
4118 // need to check for the StrictFP attribute here.
4119 switch (II->getIntrinsicID()) {
4120 case Intrinsic::pow:
4121 return optimizePow(Pow: CI, B&: Builder);
4122 case Intrinsic::exp2:
4123 return optimizeExp2(CI, B&: Builder);
4124 case Intrinsic::log:
4125 case Intrinsic::log2:
4126 case Intrinsic::log10:
4127 return optimizeLog(Log: CI, B&: Builder);
4128 case Intrinsic::sqrt:
4129 return optimizeSqrt(CI, B&: Builder);
4130 case Intrinsic::memset:
4131 return optimizeMemSet(CI, B&: Builder);
4132 case Intrinsic::memcpy:
4133 return optimizeMemCpy(CI, B&: Builder);
4134 case Intrinsic::memmove:
4135 return optimizeMemMove(CI, B&: Builder);
4136 case Intrinsic::sin:
4137 case Intrinsic::cos:
4138 if (UnsafeFPShrink)
4139 return optimizeUnaryDoubleFP(CI, B&: Builder, TLI, /*isPrecise=*/true);
4140 return nullptr;
4141 default:
4142 return nullptr;
4143 }
4144 }
4145
4146 // Also try to simplify calls to fortified library functions.
4147 if (Value *SimplifiedFortifiedCI =
4148 FortifiedSimplifier.optimizeCall(CI, B&: Builder))
4149 return SimplifiedFortifiedCI;
4150
4151 // Then check for known library functions.
4152 if (TLI->getLibFunc(FDecl: *Callee, F&: Func) && isLibFuncEmittable(M, TLI, TheLibFunc: Func)) {
4153 // We never change the calling convention.
4154 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4155 return nullptr;
4156 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
4157 return V;
4158 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
4159 return V;
4160 switch (Func) {
4161 case LibFunc_ffs:
4162 case LibFunc_ffsl:
4163 case LibFunc_ffsll:
4164 return optimizeFFS(CI, B&: Builder);
4165 case LibFunc_fls:
4166 case LibFunc_flsl:
4167 case LibFunc_flsll:
4168 return optimizeFls(CI, B&: Builder);
4169 case LibFunc_abs:
4170 case LibFunc_labs:
4171 case LibFunc_llabs:
4172 return optimizeAbs(CI, B&: Builder);
4173 case LibFunc_isdigit:
4174 return optimizeIsDigit(CI, B&: Builder);
4175 case LibFunc_isascii:
4176 return optimizeIsAscii(CI, B&: Builder);
4177 case LibFunc_toascii:
4178 return optimizeToAscii(CI, B&: Builder);
4179 case LibFunc_atoi:
4180 case LibFunc_atol:
4181 case LibFunc_atoll:
4182 return optimizeAtoi(CI, B&: Builder);
4183 case LibFunc_strtol:
4184 case LibFunc_strtoll:
4185 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/true);
4186 case LibFunc_strtoul:
4187 case LibFunc_strtoull:
4188 return optimizeStrToInt(CI, B&: Builder, /*AsSigned=*/false);
4189 case LibFunc_printf:
4190 return optimizePrintF(CI, B&: Builder);
4191 case LibFunc_sprintf:
4192 return optimizeSPrintF(CI, B&: Builder);
4193 case LibFunc_snprintf:
4194 return optimizeSnPrintF(CI, B&: Builder);
4195 case LibFunc_fprintf:
4196 return optimizeFPrintF(CI, B&: Builder);
4197 case LibFunc_fwrite:
4198 return optimizeFWrite(CI, B&: Builder);
4199 case LibFunc_fputs:
4200 return optimizeFPuts(CI, B&: Builder);
4201 case LibFunc_puts:
4202 return optimizePuts(CI, B&: Builder);
4203 case LibFunc_perror:
4204 return optimizeErrorReporting(CI, B&: Builder);
4205 case LibFunc_vfprintf:
4206 case LibFunc_fiprintf:
4207 return optimizeErrorReporting(CI, B&: Builder, StreamArg: 0);
4208 case LibFunc_exit:
4209 case LibFunc_Exit:
4210 return optimizeExit(CI);
4211 default:
4212 return nullptr;
4213 }
4214 }
4215 return nullptr;
4216}
4217
4218LibCallSimplifier::LibCallSimplifier(
4219 const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT,
4220 DomConditionCache *DC, AssumptionCache *AC, OptimizationRemarkEmitter &ORE,
4221 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
4222 function_ref<void(Instruction *, Value *)> Replacer,
4223 function_ref<void(Instruction *)> Eraser)
4224 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), DT(DT), DC(DC), AC(AC),
4225 ORE(ORE), BFI(BFI), PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
4226
4227void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
4228 // Indirect through the replacer used in this instance.
4229 Replacer(I, With);
4230}
4231
4232void LibCallSimplifier::eraseFromParent(Instruction *I) {
4233 Eraser(I);
4234}
4235
4236// TODO:
4237// Additional cases that we need to add to this file:
4238//
4239// cbrt:
4240// * cbrt(expN(X)) -> expN(x/3)
4241// * cbrt(sqrt(x)) -> pow(x,1/6)
4242// * cbrt(cbrt(x)) -> pow(x,1/9)
4243//
4244// exp, expf, expl:
4245// * exp(log(x)) -> x
4246//
4247// log, logf, logl:
4248// * log(exp(x)) -> x
4249// * log(exp(y)) -> y*log(e)
4250// * log(exp10(y)) -> y*log(10)
4251// * log(sqrt(x)) -> 0.5*log(x)
4252//
4253// pow, powf, powl:
4254// * pow(sqrt(x),y) -> pow(x,y*0.5)
4255// * pow(pow(x,y),z)-> pow(x,y*z)
4256//
4257// signbit:
4258// * signbit(cnst) -> cnst'
4259// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
4260//
4261// sqrt, sqrtf, sqrtl:
4262// * sqrt(expN(x)) -> expN(x*0.5)
4263// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
4264// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
4265//
4266
4267//===----------------------------------------------------------------------===//
4268// Fortified Library Call Optimizations
4269//===----------------------------------------------------------------------===//
4270
4271bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
4272 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
4273 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
4274 // If this function takes a flag argument, the implementation may use it to
4275 // perform extra checks. Don't fold into the non-checking variant.
4276 if (FlagOp) {
4277 ConstantInt *Flag = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *FlagOp));
4278 if (!Flag || !Flag->isZero())
4279 return false;
4280 }
4281
4282 if (SizeOp && CI->getArgOperand(i: ObjSizeOp) == CI->getArgOperand(i: *SizeOp))
4283 return true;
4284
4285 if (ConstantInt *ObjSizeCI =
4286 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: ObjSizeOp))) {
4287 if (ObjSizeCI->isMinusOne())
4288 return true;
4289 // If the object size wasn't -1 (unknown), bail out if we were asked to.
4290 if (OnlyLowerUnknownSize)
4291 return false;
4292 if (StrOp) {
4293 uint64_t Len = GetStringLength(V: CI->getArgOperand(i: *StrOp));
4294 // If the length is 0 we don't know how long it is and so we can't
4295 // remove the check.
4296 if (Len)
4297 annotateDereferenceableBytes(CI, ArgNos: *StrOp, DereferenceableBytes: Len);
4298 else
4299 return false;
4300 return ObjSizeCI->getZExtValue() >= Len;
4301 }
4302
4303 if (SizeOp) {
4304 if (ConstantInt *SizeCI =
4305 dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: *SizeOp)))
4306 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4307 }
4308 }
4309 return false;
4310}
4311
4312Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4313 IRBuilderBase &B) {
4314 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4315 CallInst *NewCI =
4316 B.CreateMemCpy(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4317 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4318 mergeAttributesAndFlags(NewCI, Old: *CI);
4319 return CI->getArgOperand(i: 0);
4320 }
4321 return nullptr;
4322}
4323
4324Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4325 IRBuilderBase &B) {
4326 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4327 CallInst *NewCI =
4328 B.CreateMemMove(Dst: CI->getArgOperand(i: 0), DstAlign: Align(1), Src: CI->getArgOperand(i: 1),
4329 SrcAlign: Align(1), Size: CI->getArgOperand(i: 2));
4330 mergeAttributesAndFlags(NewCI, Old: *CI);
4331 return CI->getArgOperand(i: 0);
4332 }
4333 return nullptr;
4334}
4335
4336Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4337 IRBuilderBase &B) {
4338 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4339 Value *Val = B.CreateIntCast(V: CI->getArgOperand(i: 1), DestTy: B.getInt8Ty(), isSigned: false);
4340 CallInst *NewCI = B.CreateMemSet(Ptr: CI->getArgOperand(i: 0), Val,
4341 Size: CI->getArgOperand(i: 2), Align: Align(1));
4342 mergeAttributesAndFlags(NewCI, Old: *CI);
4343 return CI->getArgOperand(i: 0);
4344 }
4345 return nullptr;
4346}
4347
4348Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4349 IRBuilderBase &B) {
4350 const DataLayout &DL = CI->getDataLayout();
4351 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2))
4352 if (Value *Call = emitMemPCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4353 Len: CI->getArgOperand(i: 2), B, DL, TLI)) {
4354 return mergeAttributesAndFlags(NewCI: cast<CallInst>(Val: Call), Old: *CI);
4355 }
4356 return nullptr;
4357}
4358
4359Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4360 IRBuilderBase &B,
4361 LibFunc Func) {
4362 const DataLayout &DL = CI->getDataLayout();
4363 Value *Dst = CI->getArgOperand(i: 0), *Src = CI->getArgOperand(i: 1),
4364 *ObjSize = CI->getArgOperand(i: 2);
4365
4366 // __stpcpy_chk(x,x,...) -> x+strlen(x)
4367 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4368 Value *StrLen = emitStrLen(Ptr: Src, B, DL, TLI);
4369 return StrLen ? B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst, IdxList: StrLen) : nullptr;
4370 }
4371
4372 // If a) we don't have any length information, or b) we know this will
4373 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4374 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
4375 // TODO: It might be nice to get a maximum length out of the possible
4376 // string lengths for varying.
4377 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: 1)) {
4378 if (Func == LibFunc_strcpy_chk)
4379 return copyFlags(Old: *CI, New: emitStrCpy(Dst, Src, B, TLI));
4380 else
4381 return copyFlags(Old: *CI, New: emitStpCpy(Dst, Src, B, TLI));
4382 }
4383
4384 if (OnlyLowerUnknownSize)
4385 return nullptr;
4386
4387 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4388 uint64_t Len = GetStringLength(V: Src);
4389 if (Len)
4390 annotateDereferenceableBytes(CI, ArgNos: 1, DereferenceableBytes: Len);
4391 else
4392 return nullptr;
4393
4394 unsigned SizeTBits = TLI->getSizeTSize(M: *CI->getModule());
4395 Type *SizeTTy = IntegerType::get(C&: CI->getContext(), NumBits: SizeTBits);
4396 Value *LenV = ConstantInt::get(Ty: SizeTTy, V: Len);
4397 Value *Ret = emitMemCpyChk(Dst, Src, Len: LenV, ObjSize, B, DL, TLI);
4398 // If the function was an __stpcpy_chk, and we were able to fold it into
4399 // a __memcpy_chk, we still need to return the correct end pointer.
4400 if (Ret && Func == LibFunc_stpcpy_chk)
4401 return B.CreateInBoundsGEP(Ty: B.getInt8Ty(), Ptr: Dst,
4402 IdxList: ConstantInt::get(Ty: SizeTTy, V: Len - 1));
4403 return copyFlags(Old: *CI, New: cast<CallInst>(Val: Ret));
4404}
4405
4406Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4407 IRBuilderBase &B) {
4408 if (isFortifiedCallFoldable(CI, ObjSizeOp: 1, SizeOp: std::nullopt, StrOp: 0))
4409 return copyFlags(Old: *CI, New: emitStrLen(Ptr: CI->getArgOperand(i: 0), B,
4410 DL: CI->getDataLayout(), TLI));
4411 return nullptr;
4412}
4413
4414Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4415 IRBuilderBase &B,
4416 LibFunc Func) {
4417 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 2)) {
4418 if (Func == LibFunc_strncpy_chk)
4419 return copyFlags(Old: *CI,
4420 New: emitStrNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4421 Len: CI->getArgOperand(i: 2), B, TLI));
4422 else
4423 return copyFlags(Old: *CI,
4424 New: emitStpNCpy(Dst: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4425 Len: CI->getArgOperand(i: 2), B, TLI));
4426 }
4427
4428 return nullptr;
4429}
4430
4431Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4432 IRBuilderBase &B) {
4433 if (isFortifiedCallFoldable(CI, ObjSizeOp: 4, SizeOp: 3))
4434 return copyFlags(
4435 Old: *CI, New: emitMemCCpy(Ptr1: CI->getArgOperand(i: 0), Ptr2: CI->getArgOperand(i: 1),
4436 Val: CI->getArgOperand(i: 2), Len: CI->getArgOperand(i: 3), B, TLI));
4437
4438 return nullptr;
4439}
4440
4441Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4442 IRBuilderBase &B) {
4443 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2)) {
4444 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 5));
4445 return copyFlags(Old: *CI,
4446 New: emitSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4447 Fmt: CI->getArgOperand(i: 4), Args: VariadicArgs, B, TLI));
4448 }
4449
4450 return nullptr;
4451}
4452
4453Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4454 IRBuilderBase &B) {
4455 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1)) {
4456 SmallVector<Value *, 8> VariadicArgs(drop_begin(RangeOrContainer: CI->args(), N: 4));
4457 return copyFlags(Old: *CI,
4458 New: emitSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4459 VariadicArgs, B, TLI));
4460 }
4461
4462 return nullptr;
4463}
4464
4465Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4466 IRBuilderBase &B) {
4467 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2))
4468 return copyFlags(
4469 Old: *CI, New: emitStrCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1), B, TLI));
4470
4471 return nullptr;
4472}
4473
4474Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4475 IRBuilderBase &B) {
4476 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4477 return copyFlags(Old: *CI,
4478 New: emitStrLCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4479 Size: CI->getArgOperand(i: 2), B, TLI));
4480
4481 return nullptr;
4482}
4483
4484Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4485 IRBuilderBase &B) {
4486 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4487 return copyFlags(Old: *CI,
4488 New: emitStrNCat(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4489 Size: CI->getArgOperand(i: 2), B, TLI));
4490
4491 return nullptr;
4492}
4493
4494Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4495 IRBuilderBase &B) {
4496 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3))
4497 return copyFlags(Old: *CI,
4498 New: emitStrLCpy(Dest: CI->getArgOperand(i: 0), Src: CI->getArgOperand(i: 1),
4499 Size: CI->getArgOperand(i: 2), B, TLI));
4500
4501 return nullptr;
4502}
4503
4504Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4505 IRBuilderBase &B) {
4506 if (isFortifiedCallFoldable(CI, ObjSizeOp: 3, SizeOp: 1, StrOp: std::nullopt, FlagOp: 2))
4507 return copyFlags(
4508 Old: *CI, New: emitVSNPrintf(Dest: CI->getArgOperand(i: 0), Size: CI->getArgOperand(i: 1),
4509 Fmt: CI->getArgOperand(i: 4), VAList: CI->getArgOperand(i: 5), B, TLI));
4510
4511 return nullptr;
4512}
4513
4514Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4515 IRBuilderBase &B) {
4516 if (isFortifiedCallFoldable(CI, ObjSizeOp: 2, SizeOp: std::nullopt, StrOp: std::nullopt, FlagOp: 1))
4517 return copyFlags(Old: *CI,
4518 New: emitVSPrintf(Dest: CI->getArgOperand(i: 0), Fmt: CI->getArgOperand(i: 3),
4519 VAList: CI->getArgOperand(i: 4), B, TLI));
4520
4521 return nullptr;
4522}
4523
4524Value *FortifiedLibCallSimplifier::optimizeCall(CallInst *CI,
4525 IRBuilderBase &Builder) {
4526 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4527 // Some clang users checked for _chk libcall availability using:
4528 // __has_builtin(__builtin___memcpy_chk)
4529 // When compiling with -fno-builtin, this is always true.
4530 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4531 // end up with fortified libcalls, which isn't acceptable in a freestanding
4532 // environment which only provides their non-fortified counterparts.
4533 //
4534 // Until we change clang and/or teach external users to check for availability
4535 // differently, disregard the "nobuiltin" attribute and TLI::has.
4536 //
4537 // PR23093.
4538
4539 LibFunc Func;
4540 Function *Callee = CI->getCalledFunction();
4541 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4542
4543 SmallVector<OperandBundleDef, 2> OpBundles;
4544 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
4545
4546 IRBuilderBase::OperandBundlesGuard Guard(Builder);
4547 Builder.setDefaultOperandBundles(OpBundles);
4548
4549 // First, check that this is a known library functions and that the prototype
4550 // is correct.
4551 if (!TLI->getLibFunc(FDecl: *Callee, F&: Func))
4552 return nullptr;
4553
4554 // We never change the calling convention.
4555 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4556 return nullptr;
4557
4558 switch (Func) {
4559 case LibFunc_memcpy_chk:
4560 return optimizeMemCpyChk(CI, B&: Builder);
4561 case LibFunc_mempcpy_chk:
4562 return optimizeMemPCpyChk(CI, B&: Builder);
4563 case LibFunc_memmove_chk:
4564 return optimizeMemMoveChk(CI, B&: Builder);
4565 case LibFunc_memset_chk:
4566 return optimizeMemSetChk(CI, B&: Builder);
4567 case LibFunc_stpcpy_chk:
4568 case LibFunc_strcpy_chk:
4569 return optimizeStrpCpyChk(CI, B&: Builder, Func);
4570 case LibFunc_strlen_chk:
4571 return optimizeStrLenChk(CI, B&: Builder);
4572 case LibFunc_stpncpy_chk:
4573 case LibFunc_strncpy_chk:
4574 return optimizeStrpNCpyChk(CI, B&: Builder, Func);
4575 case LibFunc_memccpy_chk:
4576 return optimizeMemCCpyChk(CI, B&: Builder);
4577 case LibFunc_snprintf_chk:
4578 return optimizeSNPrintfChk(CI, B&: Builder);
4579 case LibFunc_sprintf_chk:
4580 return optimizeSPrintfChk(CI, B&: Builder);
4581 case LibFunc_strcat_chk:
4582 return optimizeStrCatChk(CI, B&: Builder);
4583 case LibFunc_strlcat_chk:
4584 return optimizeStrLCat(CI, B&: Builder);
4585 case LibFunc_strncat_chk:
4586 return optimizeStrNCatChk(CI, B&: Builder);
4587 case LibFunc_strlcpy_chk:
4588 return optimizeStrLCpyChk(CI, B&: Builder);
4589 case LibFunc_vsnprintf_chk:
4590 return optimizeVSNPrintfChk(CI, B&: Builder);
4591 case LibFunc_vsprintf_chk:
4592 return optimizeVSPrintfChk(CI, B&: Builder);
4593 default:
4594 break;
4595 }
4596 return nullptr;
4597}
4598
4599FortifiedLibCallSimplifier::FortifiedLibCallSimplifier(
4600 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4601 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
4602